1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 // Dummy labels for just measuring the code size
1796 Label dummy_slow_path;
1797 Label dummy_continuation;
1798 Label dummy_guard;
1799 Label* slow_path = &dummy_slow_path;
1800 Label* continuation = &dummy_continuation;
1801 Label* guard = &dummy_guard;
1802 if (!Compile::current()->output()->in_scratch_emit_size()) {
1803 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1804 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1805 Compile::current()->output()->add_stub(stub);
1806 slow_path = &stub->entry();
1807 continuation = &stub->continuation();
1808 guard = &stub->guard();
1809 }
1810 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1811 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1812 }
1813
1814 if (VerifyStackAtCalls) {
1815 Unimplemented();
1816 }
1817
1818 C->output()->set_frame_complete(__ offset());
1819
1820 if (C->has_mach_constant_base_node()) {
1821 // NOTE: We set the table base offset here because users might be
1822 // emitted before MachConstantBaseNode.
1823 ConstantTable& constant_table = C->output()->constant_table();
1824 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1825 }
1826 }
1827
1828 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1829 {
1830 return MachNode::size(ra_); // too many variables; just compute it
1831 // the hard way
1832 }
1833
1834 int MachPrologNode::reloc() const
1835 {
1836 return 0;
1837 }
1838
1839 //=============================================================================
1840
1841 #ifndef PRODUCT
1842 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1843 Compile* C = ra_->C;
1844 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1845
1846 st->print("# pop frame %d\n\t",framesize);
1847
1848 if (framesize == 0) {
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1851 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1852 st->print("add sp, sp, #%d\n\t", framesize);
1853 } else {
1854 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1855 st->print("add sp, sp, rscratch1\n\t");
1856 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1857 }
1858 if (VM_Version::use_rop_protection()) {
1859 st->print("autiaz\n\t");
1860 st->print("ldr zr, [lr]\n\t");
1861 }
1862
1863 if (do_polling() && C->is_method_compilation()) {
1864 st->print("# test polling word\n\t");
1865 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1866 st->print("cmp sp, rscratch1\n\t");
1867 st->print("bhi #slow_path");
1868 }
1869 }
1870 #endif
1871
1872 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 __ remove_frame(framesize);
1877
1878 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1879 __ reserved_stack_check();
1880 }
1881
1882 if (do_polling() && C->is_method_compilation()) {
1883 Label dummy_label;
1884 Label* code_stub = &dummy_label;
1885 if (!C->output()->in_scratch_emit_size()) {
1886 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1887 C->output()->add_stub(stub);
1888 code_stub = &stub->entry();
1889 }
1890 __ relocate(relocInfo::poll_return_type);
1891 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1892 }
1893 }
1894
1895 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1896 // Variable size. Determine dynamically.
1897 return MachNode::size(ra_);
1898 }
1899
1900 int MachEpilogNode::reloc() const {
1901 // Return number of relocatable values contained in this instruction.
1902 return 1; // 1 for polling page.
1903 }
1904
1905 const Pipeline * MachEpilogNode::pipeline() const {
1906 return MachNode::pipeline_class();
1907 }
1908
1909 //=============================================================================
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && masm) {
1975 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1976 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1977 // stack->stack
1978 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1979 sve_vector_reg_size_in_bytes);
1980 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1981 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1982 sve_vector_reg_size_in_bytes);
1983 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1984 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1985 sve_vector_reg_size_in_bytes);
1986 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1987 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1988 as_FloatRegister(Matcher::_regEncode[src_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]));
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 } else if (masm) {
1994 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1995 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1996 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1997 // stack->stack
1998 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1999 if (ireg == Op_VecD) {
2000 __ unspill(rscratch1, true, src_offset);
2001 __ spill(rscratch1, true, dst_offset);
2002 } else {
2003 __ spill_copy128(src_offset, dst_offset);
2004 }
2005 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2006 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2007 ireg == Op_VecD ? __ T8B : __ T16B,
2008 as_FloatRegister(Matcher::_regEncode[src_lo]));
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2011 ireg == Op_VecD ? __ D : __ Q,
2012 ra_->reg2offset(dst_lo));
2013 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2014 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2015 ireg == Op_VecD ? __ D : __ Q,
2016 ra_->reg2offset(src_lo));
2017 } else {
2018 ShouldNotReachHere();
2019 }
2020 }
2021 } else if (masm) {
2022 switch (src_lo_rc) {
2023 case rc_int:
2024 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2025 if (is64) {
2026 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2027 as_Register(Matcher::_regEncode[src_lo]));
2028 } else {
2029 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 }
2032 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2033 if (is64) {
2034 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2035 as_Register(Matcher::_regEncode[src_lo]));
2036 } else {
2037 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2038 as_Register(Matcher::_regEncode[src_lo]));
2039 }
2040 } else { // gpr --> stack spill
2041 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2042 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2043 }
2044 break;
2045 case rc_float:
2046 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2047 if (is64) {
2048 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2049 as_FloatRegister(Matcher::_regEncode[src_lo]));
2050 } else {
2051 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2052 as_FloatRegister(Matcher::_regEncode[src_lo]));
2053 }
2054 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2055 if (is64) {
2056 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2057 as_FloatRegister(Matcher::_regEncode[src_lo]));
2058 } else {
2059 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 as_FloatRegister(Matcher::_regEncode[src_lo]));
2061 }
2062 } else { // fpr --> stack spill
2063 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2064 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2065 is64 ? __ D : __ S, dst_offset);
2066 }
2067 break;
2068 case rc_stack:
2069 if (dst_lo_rc == rc_int) { // stack --> gpr load
2070 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2071 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2072 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 is64 ? __ D : __ S, src_offset);
2074 } else if (dst_lo_rc == rc_predicate) {
2075 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2076 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2077 } else { // stack --> stack copy
2078 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2079 if (ideal_reg() == Op_RegVectMask) {
2080 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2081 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2082 } else {
2083 __ unspill(rscratch1, is64, src_offset);
2084 __ spill(rscratch1, is64, dst_offset);
2085 }
2086 }
2087 break;
2088 case rc_predicate:
2089 if (dst_lo_rc == rc_predicate) {
2090 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2091 } else if (dst_lo_rc == rc_stack) {
2092 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2093 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2094 } else {
2095 assert(false, "bad src and dst rc_class combination.");
2096 ShouldNotReachHere();
2097 }
2098 break;
2099 default:
2100 assert(false, "bad rc_class for spill");
2101 ShouldNotReachHere();
2102 }
2103 }
2104
2105 if (st) {
2106 st->print("spill ");
2107 if (src_lo_rc == rc_stack) {
2108 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2109 } else {
2110 st->print("%s -> ", Matcher::regName[src_lo]);
2111 }
2112 if (dst_lo_rc == rc_stack) {
2113 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2114 } else {
2115 st->print("%s", Matcher::regName[dst_lo]);
2116 }
2117 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2118 int vsize = 0;
2119 switch (ideal_reg()) {
2120 case Op_VecD:
2121 vsize = 64;
2122 break;
2123 case Op_VecX:
2124 vsize = 128;
2125 break;
2126 case Op_VecA:
2127 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2128 break;
2129 default:
2130 assert(false, "bad register type for spill");
2131 ShouldNotReachHere();
2132 }
2133 st->print("\t# vector spill size = %d", vsize);
2134 } else if (ideal_reg() == Op_RegVectMask) {
2135 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2136 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2137 st->print("\t# predicate spill size = %d", vsize);
2138 } else {
2139 st->print("\t# spill size = %d", is64 ? 64 : 32);
2140 }
2141 }
2142
2143 return 0;
2144
2145 }
2146
2147 #ifndef PRODUCT
2148 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2149 if (!ra_)
2150 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2151 else
2152 implementation(nullptr, ra_, false, st);
2153 }
2154 #endif
2155
2156 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2157 implementation(masm, ra_, false, nullptr);
2158 }
2159
2160 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2161 return MachNode::size(ra_);
2162 }
2163
2164 //=============================================================================
2165
2166 #ifndef PRODUCT
2167 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2168 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2169 int reg = ra_->get_reg_first(this);
2170 st->print("add %s, rsp, #%d]\t# box lock",
2171 Matcher::regName[reg], offset);
2172 }
2173 #endif
2174
2175 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177 int reg = ra_->get_encode(this);
2178
2179 // This add will handle any 24-bit signed offset. 24 bits allows an
2180 // 8 megabyte stack frame.
2181 __ add(as_Register(reg), sp, offset);
2182 }
2183
2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2185 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2186 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2187
2188 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2189 return NativeInstruction::instruction_size;
2190 } else {
2191 return 2 * NativeInstruction::instruction_size;
2192 }
2193 }
2194
2195 //=============================================================================
2196
2197 #ifndef PRODUCT
2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2199 {
2200 st->print_cr("# MachUEPNode");
2201 if (UseCompressedClassPointers) {
2202 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2203 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2204 st->print_cr("\tcmpw rscratch1, r10");
2205 } else {
2206 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2207 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2208 st->print_cr("\tcmp rscratch1, r10");
2209 }
2210 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2211 }
2212 #endif
2213
2214 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2215 {
2216 __ ic_check(InteriorEntryAlignment);
2217 }
2218
2219 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2220 {
2221 return MachNode::size(ra_);
2222 }
2223
2224 // REQUIRED EMIT CODE
2225
2226 //=============================================================================
2227
2228 // Emit exception handler code.
2229 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2230 {
2231 // mov rscratch1 #exception_blob_entry_point
2232 // br rscratch1
2233 // Note that the code buffer's insts_mark is always relative to insts.
2234 // That's why we must use the macroassembler to generate a handler.
2235 address base = __ start_a_stub(size_exception_handler());
2236 if (base == nullptr) {
2237 ciEnv::current()->record_failure("CodeCache is full");
2238 return 0; // CodeBuffer::expand failed
2239 }
2240 int offset = __ offset();
2241 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2242 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2243 __ end_a_stub();
2244 return offset;
2245 }
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258
2259 __ adr(lr, __ pc());
2260 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 __ end_a_stub();
2264 return offset;
2265 }
2266
2267 // REQUIRED MATCHER CODE
2268
2269 //=============================================================================
2270
2271 bool Matcher::match_rule_supported(int opcode) {
2272 if (!has_match_rule(opcode))
2273 return false;
2274
2275 switch (opcode) {
2276 case Op_OnSpinWait:
2277 return VM_Version::supports_on_spin_wait();
2278 case Op_CacheWB:
2279 case Op_CacheWBPreSync:
2280 case Op_CacheWBPostSync:
2281 if (!VM_Version::supports_data_cache_line_flush()) {
2282 return false;
2283 }
2284 break;
2285 case Op_ExpandBits:
2286 case Op_CompressBits:
2287 if (!VM_Version::supports_svebitperm()) {
2288 return false;
2289 }
2290 break;
2291 case Op_FmaF:
2292 case Op_FmaD:
2293 case Op_FmaVF:
2294 case Op_FmaVD:
2295 if (!UseFMA) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaHF:
2300 // UseFMA flag also needs to be checked along with FEAT_FP16
2301 if (!UseFMA || !is_feat_fp16_supported()) {
2302 return false;
2303 }
2304 break;
2305 case Op_AddHF:
2306 case Op_SubHF:
2307 case Op_MulHF:
2308 case Op_DivHF:
2309 case Op_MinHF:
2310 case Op_MaxHF:
2311 case Op_SqrtHF:
2312 // Half-precision floating point scalar operations require FEAT_FP16
2313 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2314 // features are supported.
2315 if (!is_feat_fp16_supported()) {
2316 return false;
2317 }
2318 break;
2319 }
2320
2321 return true; // Per default match rules are supported.
2322 }
2323
2324 const RegMask* Matcher::predicate_reg_mask(void) {
2325 return &_PR_REG_mask;
2326 }
2327
2328 bool Matcher::supports_vector_calling_convention(void) {
2329 return EnableVectorSupport;
2330 }
2331
2332 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2333 assert(EnableVectorSupport, "sanity");
2334 int lo = V0_num;
2335 int hi = V0_H_num;
2336 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2337 hi = V0_K_num;
2338 }
2339 return OptoRegPair(hi, lo);
2340 }
2341
2342 // Is this branch offset short enough that a short branch can be used?
2343 //
2344 // NOTE: If the platform does not provide any short branch variants, then
2345 // this method should return false for offset 0.
2346 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2347 // The passed offset is relative to address of the branch.
2348
2349 return (-32768 <= offset && offset < 32768);
2350 }
2351
2352 // Vector width in bytes.
2353 int Matcher::vector_width_in_bytes(BasicType bt) {
2354 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2355 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2356 // Minimum 2 values in vector
2357 if (size < 2*type2aelembytes(bt)) size = 0;
2358 // But never < 4
2359 if (size < 4) size = 0;
2360 return size;
2361 }
2362
2363 // Limits on vector size (number of elements) loaded into vector.
2364 int Matcher::max_vector_size(const BasicType bt) {
2365 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2366 }
2367
2368 int Matcher::min_vector_size(const BasicType bt) {
2369 int max_size = max_vector_size(bt);
2370 // Limit the min vector size to 8 bytes.
2371 int size = 8 / type2aelembytes(bt);
2372 if (bt == T_BYTE) {
2373 // To support vector api shuffle/rearrange.
2374 size = 4;
2375 } else if (bt == T_BOOLEAN) {
2376 // To support vector api load/store mask.
2377 size = 2;
2378 }
2379 if (size < 2) size = 2;
2380 return MIN2(size, max_size);
2381 }
2382
2383 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2384 return Matcher::max_vector_size(bt);
2385 }
2386
2387 // Actual max scalable vector register length.
2388 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2389 return Matcher::max_vector_size(bt);
2390 }
2391
2392 // Vector ideal reg.
2393 uint Matcher::vector_ideal_reg(int len) {
2394 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2395 return Op_VecA;
2396 }
2397 switch(len) {
2398 // For 16-bit/32-bit mask vector, reuse VecD.
2399 case 2:
2400 case 4:
2401 case 8: return Op_VecD;
2402 case 16: return Op_VecX;
2403 }
2404 ShouldNotReachHere();
2405 return 0;
2406 }
2407
2408 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2409 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2410 switch (ideal_reg) {
2411 case Op_VecA: return new vecAOper();
2412 case Op_VecD: return new vecDOper();
2413 case Op_VecX: return new vecXOper();
2414 }
2415 ShouldNotReachHere();
2416 return nullptr;
2417 }
2418
2419 bool Matcher::is_reg2reg_move(MachNode* m) {
2420 return false;
2421 }
2422
2423 bool Matcher::is_generic_vector(MachOper* opnd) {
2424 return opnd->opcode() == VREG;
2425 }
2426
2427 // Return whether or not this register is ever used as an argument.
2428 // This function is used on startup to build the trampoline stubs in
2429 // generateOptoStub. Registers not mentioned will be killed by the VM
2430 // call in the trampoline, and arguments in those registers not be
2431 // available to the callee.
2432 bool Matcher::can_be_java_arg(int reg)
2433 {
2434 return
2435 reg == R0_num || reg == R0_H_num ||
2436 reg == R1_num || reg == R1_H_num ||
2437 reg == R2_num || reg == R2_H_num ||
2438 reg == R3_num || reg == R3_H_num ||
2439 reg == R4_num || reg == R4_H_num ||
2440 reg == R5_num || reg == R5_H_num ||
2441 reg == R6_num || reg == R6_H_num ||
2442 reg == R7_num || reg == R7_H_num ||
2443 reg == V0_num || reg == V0_H_num ||
2444 reg == V1_num || reg == V1_H_num ||
2445 reg == V2_num || reg == V2_H_num ||
2446 reg == V3_num || reg == V3_H_num ||
2447 reg == V4_num || reg == V4_H_num ||
2448 reg == V5_num || reg == V5_H_num ||
2449 reg == V6_num || reg == V6_H_num ||
2450 reg == V7_num || reg == V7_H_num;
2451 }
2452
2453 bool Matcher::is_spillable_arg(int reg)
2454 {
2455 return can_be_java_arg(reg);
2456 }
2457
2458 uint Matcher::int_pressure_limit()
2459 {
2460 // JDK-8183543: When taking the number of available registers as int
2461 // register pressure threshold, the jtreg test:
2462 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2463 // failed due to C2 compilation failure with
2464 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2465 //
2466 // A derived pointer is live at CallNode and then is flagged by RA
2467 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2468 // derived pointers and lastly fail to spill after reaching maximum
2469 // number of iterations. Lowering the default pressure threshold to
2470 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2471 // a high register pressure area of the code so that split_DEF can
2472 // generate DefinitionSpillCopy for the derived pointer.
2473 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2474 if (!PreserveFramePointer) {
2475 // When PreserveFramePointer is off, frame pointer is allocatable,
2476 // but different from other SOC registers, it is excluded from
2477 // fatproj's mask because its save type is No-Save. Decrease 1 to
2478 // ensure high pressure at fatproj when PreserveFramePointer is off.
2479 // See check_pressure_at_fatproj().
2480 default_int_pressure_threshold--;
2481 }
2482 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2483 }
2484
2485 uint Matcher::float_pressure_limit()
2486 {
2487 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2488 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2489 }
2490
2491 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2492 return false;
2493 }
2494
2495 RegMask Matcher::divI_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 // Register for MODI projection of divmodI.
2501 RegMask Matcher::modI_proj_mask() {
2502 ShouldNotReachHere();
2503 return RegMask();
2504 }
2505
2506 // Register for DIVL projection of divmodL.
2507 RegMask Matcher::divL_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask();
2510 }
2511
2512 // Register for MODL projection of divmodL.
2513 RegMask Matcher::modL_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask();
2516 }
2517
2518 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2519 return FP_REG_mask();
2520 }
2521
2522 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2523 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2524 Node* u = addp->fast_out(i);
2525 if (u->is_LoadStore()) {
2526 // On AArch64, LoadStoreNodes (i.e. compare and swap
2527 // instructions) only take register indirect as an operand, so
2528 // any attempt to use an AddPNode as an input to a LoadStoreNode
2529 // must fail.
2530 return false;
2531 }
2532 if (u->is_Mem()) {
2533 int opsize = u->as_Mem()->memory_size();
2534 assert(opsize > 0, "unexpected memory operand size");
2535 if (u->as_Mem()->memory_size() != (1<<shift)) {
2536 return false;
2537 }
2538 }
2539 }
2540 return true;
2541 }
2542
2543 // Convert BootTest condition to Assembler condition.
2544 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2545 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2546 Assembler::Condition result;
2547 switch(cond) {
2548 case BoolTest::eq:
2549 result = Assembler::EQ; break;
2550 case BoolTest::ne:
2551 result = Assembler::NE; break;
2552 case BoolTest::le:
2553 result = Assembler::LE; break;
2554 case BoolTest::ge:
2555 result = Assembler::GE; break;
2556 case BoolTest::lt:
2557 result = Assembler::LT; break;
2558 case BoolTest::gt:
2559 result = Assembler::GT; break;
2560 case BoolTest::ule:
2561 result = Assembler::LS; break;
2562 case BoolTest::uge:
2563 result = Assembler::HS; break;
2564 case BoolTest::ult:
2565 result = Assembler::LO; break;
2566 case BoolTest::ugt:
2567 result = Assembler::HI; break;
2568 case BoolTest::overflow:
2569 result = Assembler::VS; break;
2570 case BoolTest::no_overflow:
2571 result = Assembler::VC; break;
2572 default:
2573 ShouldNotReachHere();
2574 return Assembler::Condition(-1);
2575 }
2576
2577 // Check conversion
2578 if (cond & BoolTest::unsigned_compare) {
2579 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2580 } else {
2581 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2582 }
2583
2584 return result;
2585 }
2586
2587 // Binary src (Replicate con)
2588 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2589 if (n == nullptr || m == nullptr) {
2590 return false;
2591 }
2592
2593 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2594 return false;
2595 }
2596
2597 Node* imm_node = m->in(1);
2598 if (!imm_node->is_Con()) {
2599 return false;
2600 }
2601
2602 const Type* t = imm_node->bottom_type();
2603 if (!(t->isa_int() || t->isa_long())) {
2604 return false;
2605 }
2606
2607 switch (n->Opcode()) {
2608 case Op_AndV:
2609 case Op_OrV:
2610 case Op_XorV: {
2611 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2612 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2613 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2614 }
2615 case Op_AddVB:
2616 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2617 case Op_AddVS:
2618 case Op_AddVI:
2619 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2620 case Op_AddVL:
2621 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2622 default:
2623 return false;
2624 }
2625 }
2626
2627 // (XorV src (Replicate m1))
2628 // (XorVMask src (MaskAll m1))
2629 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2630 if (n != nullptr && m != nullptr) {
2631 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2632 VectorNode::is_all_ones_vector(m);
2633 }
2634 return false;
2635 }
2636
2637 // Should the matcher clone input 'm' of node 'n'?
2638 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2639 if (is_vshift_con_pattern(n, m) ||
2640 is_vector_bitwise_not_pattern(n, m) ||
2641 is_valid_sve_arith_imm_pattern(n, m) ||
2642 is_encode_and_store_pattern(n, m)) {
2643 mstack.push(m, Visit);
2644 return true;
2645 }
2646 return false;
2647 }
2648
2649 // Should the Matcher clone shifts on addressing modes, expecting them
2650 // to be subsumed into complex addressing expressions or compute them
2651 // into registers?
2652 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2653
2654 // Loads and stores with indirect memory input (e.g., volatile loads and
2655 // stores) do not subsume the input into complex addressing expressions. If
2656 // the addressing expression is input to at least one such load or store, do
2657 // not clone the addressing expression. Query needs_acquiring_load and
2658 // needs_releasing_store as a proxy for indirect memory input, as it is not
2659 // possible to directly query for indirect memory input at this stage.
2660 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2661 Node* n = m->fast_out(i);
2662 if (n->is_Load() && needs_acquiring_load(n)) {
2663 return false;
2664 }
2665 if (n->is_Store() && needs_releasing_store(n)) {
2666 return false;
2667 }
2668 }
2669
2670 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2671 return true;
2672 }
2673
2674 Node *off = m->in(AddPNode::Offset);
2675 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2676 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.set(off->_idx); // Flag as address_visited
2680 mstack.push(off->in(2), Visit);
2681 Node *conv = off->in(1);
2682 if (conv->Opcode() == Op_ConvI2L &&
2683 // Are there other uses besides address expressions?
2684 !is_visited(conv)) {
2685 address_visited.set(conv->_idx); // Flag as address_visited
2686 mstack.push(conv->in(1), Pre_Visit);
2687 } else {
2688 mstack.push(conv, Pre_Visit);
2689 }
2690 address_visited.test_set(m->_idx); // Flag as address_visited
2691 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2692 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2693 return true;
2694 } else if (off->Opcode() == Op_ConvI2L &&
2695 // Are there other uses besides address expressions?
2696 !is_visited(off)) {
2697 address_visited.test_set(m->_idx); // Flag as address_visited
2698 address_visited.set(off->_idx); // Flag as address_visited
2699 mstack.push(off->in(1), Pre_Visit);
2700 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2701 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2702 return true;
2703 }
2704 return false;
2705 }
2706
2707 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2708 { \
2709 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2710 guarantee(DISP == 0, "mode not permitted for volatile"); \
2711 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2712 __ INSN(REG, as_Register(BASE)); \
2713 }
2714
2715
2716 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2717 {
2718 Address::extend scale;
2719
2720 // Hooboy, this is fugly. We need a way to communicate to the
2721 // encoder that the index needs to be sign extended, so we have to
2722 // enumerate all the cases.
2723 switch (opcode) {
2724 case INDINDEXSCALEDI2L:
2725 case INDINDEXSCALEDI2LN:
2726 case INDINDEXI2L:
2727 case INDINDEXI2LN:
2728 scale = Address::sxtw(size);
2729 break;
2730 default:
2731 scale = Address::lsl(size);
2732 }
2733
2734 if (index == -1) {
2735 return Address(base, disp);
2736 } else {
2737 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2738 return Address(base, as_Register(index), scale);
2739 }
2740 }
2741
2742
2743 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2744 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2745 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2746 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2747 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2748
2749 // Used for all non-volatile memory accesses. The use of
2750 // $mem->opcode() to discover whether this pattern uses sign-extended
2751 // offsets is something of a kludge.
2752 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2753 Register reg, int opcode,
2754 Register base, int index, int scale, int disp,
2755 int size_in_memory)
2756 {
2757 Address addr = mem2address(opcode, base, index, scale, disp);
2758 if (addr.getMode() == Address::base_plus_offset) {
2759 /* Fix up any out-of-range offsets. */
2760 assert_different_registers(rscratch1, base);
2761 assert_different_registers(rscratch1, reg);
2762 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2763 }
2764 (masm->*insn)(reg, addr);
2765 }
2766
2767 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2768 FloatRegister reg, int opcode,
2769 Register base, int index, int size, int disp,
2770 int size_in_memory)
2771 {
2772 Address::extend scale;
2773
2774 switch (opcode) {
2775 case INDINDEXSCALEDI2L:
2776 case INDINDEXSCALEDI2LN:
2777 scale = Address::sxtw(size);
2778 break;
2779 default:
2780 scale = Address::lsl(size);
2781 }
2782
2783 if (index == -1) {
2784 // Fix up any out-of-range offsets.
2785 assert_different_registers(rscratch1, base);
2786 Address addr = Address(base, disp);
2787 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2788 (masm->*insn)(reg, addr);
2789 } else {
2790 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2791 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2792 }
2793 }
2794
2795 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2796 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2797 int opcode, Register base, int index, int size, int disp)
2798 {
2799 if (index == -1) {
2800 (masm->*insn)(reg, T, Address(base, disp));
2801 } else {
2802 assert(disp == 0, "unsupported address mode");
2803 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2804 }
2805 }
2806
2807 %}
2808
2809
2810
2811 //----------ENCODING BLOCK-----------------------------------------------------
2812 // This block specifies the encoding classes used by the compiler to
2813 // output byte streams. Encoding classes are parameterized macros
2814 // used by Machine Instruction Nodes in order to generate the bit
2815 // encoding of the instruction. Operands specify their base encoding
2816 // interface with the interface keyword. There are currently
2817 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2818 // COND_INTER. REG_INTER causes an operand to generate a function
2819 // which returns its register number when queried. CONST_INTER causes
2820 // an operand to generate a function which returns the value of the
2821 // constant when queried. MEMORY_INTER causes an operand to generate
2822 // four functions which return the Base Register, the Index Register,
2823 // the Scale Value, and the Offset Value of the operand when queried.
2824 // COND_INTER causes an operand to generate six functions which return
2825 // the encoding code (ie - encoding bits for the instruction)
2826 // associated with each basic boolean condition for a conditional
2827 // instruction.
2828 //
2829 // Instructions specify two basic values for encoding. Again, a
2830 // function is available to check if the constant displacement is an
2831 // oop. They use the ins_encode keyword to specify their encoding
2832 // classes (which must be a sequence of enc_class names, and their
2833 // parameters, specified in the encoding block), and they use the
2834 // opcode keyword to specify, in order, their primary, secondary, and
2835 // tertiary opcode. Only the opcode sections which a particular
2836 // instruction needs for encoding need to be specified.
2837 encode %{
2838 // Build emit functions for each basic byte or larger field in the
2839 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2840 // from C++ code in the enc_class source block. Emit functions will
2841 // live in the main source block for now. In future, we can
2842 // generalize this by adding a syntax that specifies the sizes of
2843 // fields in an order, so that the adlc can build the emit functions
2844 // automagically
2845
2846 // catch all for unimplemented encodings
2847 enc_class enc_unimplemented %{
2848 __ unimplemented("C2 catch all");
2849 %}
2850
2851 // BEGIN Non-volatile memory access
2852
2853 // This encoding class is generated automatically from ad_encode.m4.
2854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2855 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2856 Register dst_reg = as_Register($dst$$reg);
2857 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2859 %}
2860
2861 // This encoding class is generated automatically from ad_encode.m4.
2862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2863 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2864 Register dst_reg = as_Register($dst$$reg);
2865 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2867 %}
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2952 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2960 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2968 Register src_reg = as_Register($src$$reg);
2969 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_strb0(memory1 mem) %{
2976 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2983 Register src_reg = as_Register($src$$reg);
2984 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_strh0(memory2 mem) %{
2991 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strw0(memory4 mem) %{
3006 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 // we sometimes get asked to store the stack pointer into the
3015 // current thread -- we cannot do that directly on AArch64
3016 if (src_reg == r31_sp) {
3017 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3018 __ mov(rscratch2, sp);
3019 src_reg = rscratch2;
3020 }
3021 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_str0(memory8 mem) %{
3028 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3035 FloatRegister src_reg = as_FloatRegister($src$$reg);
3036 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3043 FloatRegister src_reg = as_FloatRegister($src$$reg);
3044 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3051 __ membar(Assembler::StoreStore);
3052 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3054 %}
3055
3056 // END Non-volatile memory access
3057
3058 // Vector loads and stores
3059 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3078 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3079 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3102 FloatRegister src_reg = as_FloatRegister($src$$reg);
3103 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 // volatile loads and stores
3108
3109 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3110 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3111 rscratch1, stlrb);
3112 %}
3113
3114 enc_class aarch64_enc_stlrb0(memory mem) %{
3115 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, stlrb);
3117 %}
3118
3119 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3120 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3121 rscratch1, stlrh);
3122 %}
3123
3124 enc_class aarch64_enc_stlrh0(memory mem) %{
3125 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3126 rscratch1, stlrh);
3127 %}
3128
3129 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3130 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, stlrw);
3132 %}
3133
3134 enc_class aarch64_enc_stlrw0(memory mem) %{
3135 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, stlrw);
3137 %}
3138
3139 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3140 Register dst_reg = as_Register($dst$$reg);
3141 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 __ sxtbw(dst_reg, dst_reg);
3144 %}
3145
3146 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3147 Register dst_reg = as_Register($dst$$reg);
3148 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 __ sxtb(dst_reg, dst_reg);
3151 %}
3152
3153 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3154 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarb);
3156 %}
3157
3158 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3159 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarb);
3161 %}
3162
3163 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3164 Register dst_reg = as_Register($dst$$reg);
3165 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 __ sxthw(dst_reg, dst_reg);
3168 %}
3169
3170 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3171 Register dst_reg = as_Register($dst$$reg);
3172 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 __ sxth(dst_reg, dst_reg);
3175 %}
3176
3177 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3178 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarh);
3180 %}
3181
3182 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3183 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarh);
3185 %}
3186
3187 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3188 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarw);
3190 %}
3191
3192 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarw);
3195 %}
3196
3197 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3198 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldar);
3200 %}
3201
3202 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldarw);
3205 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3209 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldar);
3211 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3212 %}
3213
3214 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3215 Register src_reg = as_Register($src$$reg);
3216 // we sometimes get asked to store the stack pointer into the
3217 // current thread -- we cannot do that directly on AArch64
3218 if (src_reg == r31_sp) {
3219 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3220 __ mov(rscratch2, sp);
3221 src_reg = rscratch2;
3222 }
3223 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, stlr);
3225 %}
3226
3227 enc_class aarch64_enc_stlr0(memory mem) %{
3228 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3233 {
3234 FloatRegister src_reg = as_FloatRegister($src$$reg);
3235 __ fmovs(rscratch2, src_reg);
3236 }
3237 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, stlrw);
3239 %}
3240
3241 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3242 {
3243 FloatRegister src_reg = as_FloatRegister($src$$reg);
3244 __ fmovd(rscratch2, src_reg);
3245 }
3246 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, stlr);
3248 %}
3249
3250 // synchronized read/update encodings
3251
3252 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3253 Register dst_reg = as_Register($dst$$reg);
3254 Register base = as_Register($mem$$base);
3255 int index = $mem$$index;
3256 int scale = $mem$$scale;
3257 int disp = $mem$$disp;
3258 if (index == -1) {
3259 if (disp != 0) {
3260 __ lea(rscratch1, Address(base, disp));
3261 __ ldaxr(dst_reg, rscratch1);
3262 } else {
3263 // TODO
3264 // should we ever get anything other than this case?
3265 __ ldaxr(dst_reg, base);
3266 }
3267 } else {
3268 Register index_reg = as_Register(index);
3269 if (disp == 0) {
3270 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3271 __ ldaxr(dst_reg, rscratch1);
3272 } else {
3273 __ lea(rscratch1, Address(base, disp));
3274 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3275 __ ldaxr(dst_reg, rscratch1);
3276 }
3277 }
3278 %}
3279
3280 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3281 Register src_reg = as_Register($src$$reg);
3282 Register base = as_Register($mem$$base);
3283 int index = $mem$$index;
3284 int scale = $mem$$scale;
3285 int disp = $mem$$disp;
3286 if (index == -1) {
3287 if (disp != 0) {
3288 __ lea(rscratch2, Address(base, disp));
3289 __ stlxr(rscratch1, src_reg, rscratch2);
3290 } else {
3291 // TODO
3292 // should we ever get anything other than this case?
3293 __ stlxr(rscratch1, src_reg, base);
3294 }
3295 } else {
3296 Register index_reg = as_Register(index);
3297 if (disp == 0) {
3298 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3299 __ stlxr(rscratch1, src_reg, rscratch2);
3300 } else {
3301 __ lea(rscratch2, Address(base, disp));
3302 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3303 __ stlxr(rscratch1, src_reg, rscratch2);
3304 }
3305 }
3306 __ cmpw(rscratch1, zr);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ false, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::word, /*acquire*/ false, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::byte, /*acquire*/ false, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337
3338 // The only difference between aarch64_enc_cmpxchg and
3339 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3340 // CompareAndSwap sequence to serve as a barrier on acquiring a
3341 // lock.
3342 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3343 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3344 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3345 Assembler::xword, /*acquire*/ true, /*release*/ true,
3346 /*weak*/ false, noreg);
3347 %}
3348
3349 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3350 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3351 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3352 Assembler::word, /*acquire*/ true, /*release*/ true,
3353 /*weak*/ false, noreg);
3354 %}
3355
3356 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3357 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3358 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3359 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3360 /*weak*/ false, noreg);
3361 %}
3362
3363 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 Register res_reg = as_Register($res$$reg);
3373 __ cset(res_reg, Assembler::EQ);
3374 %}
3375
3376 // prefetch encodings
3377
3378 enc_class aarch64_enc_prefetchw(memory mem) %{
3379 Register base = as_Register($mem$$base);
3380 int index = $mem$$index;
3381 int scale = $mem$$scale;
3382 int disp = $mem$$disp;
3383 if (index == -1) {
3384 // Fix up any out-of-range offsets.
3385 assert_different_registers(rscratch1, base);
3386 Address addr = Address(base, disp);
3387 addr = __ legitimize_address(addr, 8, rscratch1);
3388 __ prfm(addr, PSTL1KEEP);
3389 } else {
3390 Register index_reg = as_Register(index);
3391 if (disp == 0) {
3392 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3393 } else {
3394 __ lea(rscratch1, Address(base, disp));
3395 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3396 }
3397 }
3398 %}
3399
3400 // mov encodings
3401
3402 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3403 uint32_t con = (uint32_t)$src$$constant;
3404 Register dst_reg = as_Register($dst$$reg);
3405 if (con == 0) {
3406 __ movw(dst_reg, zr);
3407 } else {
3408 __ movw(dst_reg, con);
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr || con == (address)1) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 if (rtype == relocInfo::oop_type) {
3430 __ movoop(dst_reg, (jobject)con);
3431 } else if (rtype == relocInfo::metadata_type) {
3432 __ mov_metadata(dst_reg, (Metadata*)con);
3433 } else {
3434 assert(rtype == relocInfo::none, "unexpected reloc type");
3435 if (! __ is_valid_AArch64_address(con) ||
3436 con < (address)(uintptr_t)os::vm_page_size()) {
3437 __ mov(dst_reg, con);
3438 } else {
3439 uint64_t offset;
3440 __ adrp(dst_reg, con, offset);
3441 __ add(dst_reg, dst_reg, offset);
3442 }
3443 }
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 __ mov(dst_reg, (uint64_t)1);
3455 %}
3456
3457 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3458 __ load_byte_map_base($dst$$Register);
3459 %}
3460
3461 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3462 Register dst_reg = as_Register($dst$$reg);
3463 address con = (address)$src$$constant;
3464 if (con == nullptr) {
3465 ShouldNotReachHere();
3466 } else {
3467 relocInfo::relocType rtype = $src->constant_reloc();
3468 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3469 __ set_narrow_oop(dst_reg, (jobject)con);
3470 }
3471 %}
3472
3473 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3474 Register dst_reg = as_Register($dst$$reg);
3475 __ mov(dst_reg, zr);
3476 %}
3477
3478 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 address con = (address)$src$$constant;
3481 if (con == nullptr) {
3482 ShouldNotReachHere();
3483 } else {
3484 relocInfo::relocType rtype = $src->constant_reloc();
3485 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3486 __ set_narrow_klass(dst_reg, (Klass *)con);
3487 }
3488 %}
3489
3490 // arithmetic encodings
3491
3492 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src_reg = as_Register($src1$$reg);
3495 int32_t con = (int32_t)$src2$$constant;
3496 // add has primary == 0, subtract has primary == 1
3497 if ($primary) { con = -con; }
3498 if (con < 0) {
3499 __ subw(dst_reg, src_reg, -con);
3500 } else {
3501 __ addw(dst_reg, src_reg, con);
3502 }
3503 %}
3504
3505 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 Register src_reg = as_Register($src1$$reg);
3508 int32_t con = (int32_t)$src2$$constant;
3509 // add has primary == 0, subtract has primary == 1
3510 if ($primary) { con = -con; }
3511 if (con < 0) {
3512 __ sub(dst_reg, src_reg, -con);
3513 } else {
3514 __ add(dst_reg, src_reg, con);
3515 }
3516 %}
3517
3518 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3519 Register dst_reg = as_Register($dst$$reg);
3520 Register src1_reg = as_Register($src1$$reg);
3521 Register src2_reg = as_Register($src2$$reg);
3522 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3523 %}
3524
3525 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3526 Register dst_reg = as_Register($dst$$reg);
3527 Register src1_reg = as_Register($src1$$reg);
3528 Register src2_reg = as_Register($src2$$reg);
3529 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3530 %}
3531
3532 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3533 Register dst_reg = as_Register($dst$$reg);
3534 Register src1_reg = as_Register($src1$$reg);
3535 Register src2_reg = as_Register($src2$$reg);
3536 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3537 %}
3538
3539 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3540 Register dst_reg = as_Register($dst$$reg);
3541 Register src1_reg = as_Register($src1$$reg);
3542 Register src2_reg = as_Register($src2$$reg);
3543 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3544 %}
3545
3546 // compare instruction encodings
3547
3548 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3549 Register reg1 = as_Register($src1$$reg);
3550 Register reg2 = as_Register($src2$$reg);
3551 __ cmpw(reg1, reg2);
3552 %}
3553
3554 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3555 Register reg = as_Register($src1$$reg);
3556 int32_t val = $src2$$constant;
3557 if (val >= 0) {
3558 __ subsw(zr, reg, val);
3559 } else {
3560 __ addsw(zr, reg, -val);
3561 }
3562 %}
3563
3564 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3565 Register reg1 = as_Register($src1$$reg);
3566 uint32_t val = (uint32_t)$src2$$constant;
3567 __ movw(rscratch1, val);
3568 __ cmpw(reg1, rscratch1);
3569 %}
3570
3571 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3572 Register reg1 = as_Register($src1$$reg);
3573 Register reg2 = as_Register($src2$$reg);
3574 __ cmp(reg1, reg2);
3575 %}
3576
3577 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3578 Register reg = as_Register($src1$$reg);
3579 int64_t val = $src2$$constant;
3580 if (val >= 0) {
3581 __ subs(zr, reg, val);
3582 } else if (val != -val) {
3583 __ adds(zr, reg, -val);
3584 } else {
3585 // aargh, Long.MIN_VALUE is a special case
3586 __ orr(rscratch1, zr, (uint64_t)val);
3587 __ subs(zr, reg, rscratch1);
3588 }
3589 %}
3590
3591 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3592 Register reg1 = as_Register($src1$$reg);
3593 uint64_t val = (uint64_t)$src2$$constant;
3594 __ mov(rscratch1, val);
3595 __ cmp(reg1, rscratch1);
3596 %}
3597
3598 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3599 Register reg1 = as_Register($src1$$reg);
3600 Register reg2 = as_Register($src2$$reg);
3601 __ cmp(reg1, reg2);
3602 %}
3603
3604 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3605 Register reg1 = as_Register($src1$$reg);
3606 Register reg2 = as_Register($src2$$reg);
3607 __ cmpw(reg1, reg2);
3608 %}
3609
3610 enc_class aarch64_enc_testp(iRegP src) %{
3611 Register reg = as_Register($src$$reg);
3612 __ cmp(reg, zr);
3613 %}
3614
3615 enc_class aarch64_enc_testn(iRegN src) %{
3616 Register reg = as_Register($src$$reg);
3617 __ cmpw(reg, zr);
3618 %}
3619
3620 enc_class aarch64_enc_b(label lbl) %{
3621 Label *L = $lbl$$label;
3622 __ b(*L);
3623 %}
3624
3625 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3626 Label *L = $lbl$$label;
3627 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3628 %}
3629
3630 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3631 Label *L = $lbl$$label;
3632 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3633 %}
3634
3635 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3636 %{
3637 Register sub_reg = as_Register($sub$$reg);
3638 Register super_reg = as_Register($super$$reg);
3639 Register temp_reg = as_Register($temp$$reg);
3640 Register result_reg = as_Register($result$$reg);
3641
3642 Label miss;
3643 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3644 nullptr, &miss,
3645 /*set_cond_codes:*/ true);
3646 if ($primary) {
3647 __ mov(result_reg, zr);
3648 }
3649 __ bind(miss);
3650 %}
3651
3652 enc_class aarch64_enc_java_static_call(method meth) %{
3653 address addr = (address)$meth$$method;
3654 address call;
3655 if (!_method) {
3656 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3657 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3658 if (call == nullptr) {
3659 ciEnv::current()->record_failure("CodeCache is full");
3660 return;
3661 }
3662 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3663 // The NOP here is purely to ensure that eliding a call to
3664 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3665 __ nop();
3666 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3667 } else {
3668 int method_index = resolved_method_index(masm);
3669 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3670 : static_call_Relocation::spec(method_index);
3671 call = __ trampoline_call(Address(addr, rspec));
3672 if (call == nullptr) {
3673 ciEnv::current()->record_failure("CodeCache is full");
3674 return;
3675 }
3676 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3677 // Calls of the same statically bound method can share
3678 // a stub to the interpreter.
3679 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3680 } else {
3681 // Emit stub for static call
3682 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3683 if (stub == nullptr) {
3684 ciEnv::current()->record_failure("CodeCache is full");
3685 return;
3686 }
3687 }
3688 }
3689
3690 __ post_call_nop();
3691
3692 // Only non uncommon_trap calls need to reinitialize ptrue.
3693 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3694 __ reinitialize_ptrue();
3695 }
3696 %}
3697
3698 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3699 int method_index = resolved_method_index(masm);
3700 address call = __ ic_call((address)$meth$$method, method_index);
3701 if (call == nullptr) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 __ post_call_nop();
3706 if (Compile::current()->max_vector_size() > 0) {
3707 __ reinitialize_ptrue();
3708 }
3709 %}
3710
3711 enc_class aarch64_enc_call_epilog() %{
3712 if (VerifyStackAtCalls) {
3713 // Check that stack depth is unchanged: find majik cookie on stack
3714 __ call_Unimplemented();
3715 }
3716 %}
3717
3718 enc_class aarch64_enc_java_to_runtime(method meth) %{
3719 // some calls to generated routines (arraycopy code) are scheduled
3720 // by C2 as runtime calls. if so we can call them using a br (they
3721 // will be in a reachable segment) otherwise we have to use a blr
3722 // which loads the absolute address into a register.
3723 address entry = (address)$meth$$method;
3724 CodeBlob *cb = CodeCache::find_blob(entry);
3725 if (cb) {
3726 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3727 if (call == nullptr) {
3728 ciEnv::current()->record_failure("CodeCache is full");
3729 return;
3730 }
3731 __ post_call_nop();
3732 } else {
3733 Label retaddr;
3734 // Make the anchor frame walkable
3735 __ adr(rscratch2, retaddr);
3736 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3737 __ lea(rscratch1, RuntimeAddress(entry));
3738 __ blr(rscratch1);
3739 __ bind(retaddr);
3740 __ post_call_nop();
3741 }
3742 if (Compile::current()->max_vector_size() > 0) {
3743 __ reinitialize_ptrue();
3744 }
3745 %}
3746
3747 enc_class aarch64_enc_rethrow() %{
3748 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3749 %}
3750
3751 enc_class aarch64_enc_ret() %{
3752 #ifdef ASSERT
3753 if (Compile::current()->max_vector_size() > 0) {
3754 __ verify_ptrue();
3755 }
3756 #endif
3757 __ ret(lr);
3758 %}
3759
3760 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3761 Register target_reg = as_Register($jump_target$$reg);
3762 __ br(target_reg);
3763 %}
3764
3765 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3766 Register target_reg = as_Register($jump_target$$reg);
3767 // exception oop should be in r0
3768 // ret addr has been popped into lr
3769 // callee expects it in r3
3770 __ mov(r3, lr);
3771 __ br(target_reg);
3772 %}
3773
3774 %}
3775
3776 //----------FRAME--------------------------------------------------------------
3777 // Definition of frame structure and management information.
3778 //
3779 // S T A C K L A Y O U T Allocators stack-slot number
3780 // | (to get allocators register number
3781 // G Owned by | | v add OptoReg::stack0())
3782 // r CALLER | |
3783 // o | +--------+ pad to even-align allocators stack-slot
3784 // w V | pad0 | numbers; owned by CALLER
3785 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3786 // h ^ | in | 5
3787 // | | args | 4 Holes in incoming args owned by SELF
3788 // | | | | 3
3789 // | | +--------+
3790 // V | | old out| Empty on Intel, window on Sparc
3791 // | old |preserve| Must be even aligned.
3792 // | SP-+--------+----> Matcher::_old_SP, even aligned
3793 // | | in | 3 area for Intel ret address
3794 // Owned by |preserve| Empty on Sparc.
3795 // SELF +--------+
3796 // | | pad2 | 2 pad to align old SP
3797 // | +--------+ 1
3798 // | | locks | 0
3799 // | +--------+----> OptoReg::stack0(), even aligned
3800 // | | pad1 | 11 pad to align new SP
3801 // | +--------+
3802 // | | | 10
3803 // | | spills | 9 spills
3804 // V | | 8 (pad0 slot for callee)
3805 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3806 // ^ | out | 7
3807 // | | args | 6 Holes in outgoing args owned by CALLEE
3808 // Owned by +--------+
3809 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3810 // | new |preserve| Must be even-aligned.
3811 // | SP-+--------+----> Matcher::_new_SP, even aligned
3812 // | | |
3813 //
3814 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3815 // known from SELF's arguments and the Java calling convention.
3816 // Region 6-7 is determined per call site.
3817 // Note 2: If the calling convention leaves holes in the incoming argument
3818 // area, those holes are owned by SELF. Holes in the outgoing area
3819 // are owned by the CALLEE. Holes should not be necessary in the
3820 // incoming area, as the Java calling convention is completely under
3821 // the control of the AD file. Doubles can be sorted and packed to
3822 // avoid holes. Holes in the outgoing arguments may be necessary for
3823 // varargs C calling conventions.
3824 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3825 // even aligned with pad0 as needed.
3826 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3827 // (the latter is true on Intel but is it false on AArch64?)
3828 // region 6-11 is even aligned; it may be padded out more so that
3829 // the region from SP to FP meets the minimum stack alignment.
3830 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3831 // alignment. Region 11, pad1, may be dynamically extended so that
3832 // SP meets the minimum alignment.
3833
3834 frame %{
3835 // These three registers define part of the calling convention
3836 // between compiled code and the interpreter.
3837
3838 // Inline Cache Register or Method for I2C.
3839 inline_cache_reg(R12);
3840
3841 // Number of stack slots consumed by locking an object
3842 sync_stack_slots(2);
3843
3844 // Compiled code's Frame Pointer
3845 frame_pointer(R31);
3846
3847 // Interpreter stores its frame pointer in a register which is
3848 // stored to the stack by I2CAdaptors.
3849 // I2CAdaptors convert from interpreted java to compiled java.
3850 interpreter_frame_pointer(R29);
3851
3852 // Stack alignment requirement
3853 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3854
3855 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3856 // for calls to C. Supports the var-args backing area for register parms.
3857 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3858
3859 // The after-PROLOG location of the return address. Location of
3860 // return address specifies a type (REG or STACK) and a number
3861 // representing the register number (i.e. - use a register name) or
3862 // stack slot.
3863 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3864 // Otherwise, it is above the locks and verification slot and alignment word
3865 // TODO this may well be correct but need to check why that - 2 is there
3866 // ppc port uses 0 but we definitely need to allow for fixed_slots
3867 // which folds in the space used for monitors
3868 return_addr(STACK - 2 +
3869 align_up((Compile::current()->in_preserve_stack_slots() +
3870 Compile::current()->fixed_slots()),
3871 stack_alignment_in_slots()));
3872
3873 // Location of compiled Java return values. Same as C for now.
3874 return_value
3875 %{
3876 // TODO do we allow ideal_reg == Op_RegN???
3877 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3878 "only return normal values");
3879
3880 static const int lo[Op_RegL + 1] = { // enum name
3881 0, // Op_Node
3882 0, // Op_Set
3883 R0_num, // Op_RegN
3884 R0_num, // Op_RegI
3885 R0_num, // Op_RegP
3886 V0_num, // Op_RegF
3887 V0_num, // Op_RegD
3888 R0_num // Op_RegL
3889 };
3890
3891 static const int hi[Op_RegL + 1] = { // enum name
3892 0, // Op_Node
3893 0, // Op_Set
3894 OptoReg::Bad, // Op_RegN
3895 OptoReg::Bad, // Op_RegI
3896 R0_H_num, // Op_RegP
3897 OptoReg::Bad, // Op_RegF
3898 V0_H_num, // Op_RegD
3899 R0_H_num // Op_RegL
3900 };
3901
3902 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3903 %}
3904 %}
3905
3906 //----------ATTRIBUTES---------------------------------------------------------
3907 //----------Operand Attributes-------------------------------------------------
3908 op_attrib op_cost(1); // Required cost attribute
3909
3910 //----------Instruction Attributes---------------------------------------------
3911 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3912 ins_attrib ins_size(32); // Required size attribute (in bits)
3913 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3914 // a non-matching short branch variant
3915 // of some long branch?
3916 ins_attrib ins_alignment(4); // Required alignment attribute (must
3917 // be a power of 2) specifies the
3918 // alignment that some part of the
3919 // instruction (not necessarily the
3920 // start) requires. If > 1, a
3921 // compute_padding() function must be
3922 // provided for the instruction
3923
3924 //----------OPERANDS-----------------------------------------------------------
3925 // Operand definitions must precede instruction definitions for correct parsing
3926 // in the ADLC because operands constitute user defined types which are used in
3927 // instruction definitions.
3928
3929 //----------Simple Operands----------------------------------------------------
3930
3931 // Integer operands 32 bit
3932 // 32 bit immediate
3933 operand immI()
3934 %{
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 // 32 bit zero
3943 operand immI0()
3944 %{
3945 predicate(n->get_int() == 0);
3946 match(ConI);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 // 32 bit unit increment
3954 operand immI_1()
3955 %{
3956 predicate(n->get_int() == 1);
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 // 32 bit unit decrement
3965 operand immI_M1()
3966 %{
3967 predicate(n->get_int() == -1);
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 // Shift values for add/sub extension shift
3976 operand immIExt()
3977 %{
3978 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3979 match(ConI);
3980
3981 op_cost(0);
3982 format %{ %}
3983 interface(CONST_INTER);
3984 %}
3985
3986 operand immI_gt_1()
3987 %{
3988 predicate(n->get_int() > 1);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 operand immI_le_4()
3997 %{
3998 predicate(n->get_int() <= 4);
3999 match(ConI);
4000
4001 op_cost(0);
4002 format %{ %}
4003 interface(CONST_INTER);
4004 %}
4005
4006 operand immI_16()
4007 %{
4008 predicate(n->get_int() == 16);
4009 match(ConI);
4010
4011 op_cost(0);
4012 format %{ %}
4013 interface(CONST_INTER);
4014 %}
4015
4016 operand immI_24()
4017 %{
4018 predicate(n->get_int() == 24);
4019 match(ConI);
4020
4021 op_cost(0);
4022 format %{ %}
4023 interface(CONST_INTER);
4024 %}
4025
4026 operand immI_32()
4027 %{
4028 predicate(n->get_int() == 32);
4029 match(ConI);
4030
4031 op_cost(0);
4032 format %{ %}
4033 interface(CONST_INTER);
4034 %}
4035
4036 operand immI_48()
4037 %{
4038 predicate(n->get_int() == 48);
4039 match(ConI);
4040
4041 op_cost(0);
4042 format %{ %}
4043 interface(CONST_INTER);
4044 %}
4045
4046 operand immI_56()
4047 %{
4048 predicate(n->get_int() == 56);
4049 match(ConI);
4050
4051 op_cost(0);
4052 format %{ %}
4053 interface(CONST_INTER);
4054 %}
4055
4056 operand immI_255()
4057 %{
4058 predicate(n->get_int() == 255);
4059 match(ConI);
4060
4061 op_cost(0);
4062 format %{ %}
4063 interface(CONST_INTER);
4064 %}
4065
4066 operand immI_65535()
4067 %{
4068 predicate(n->get_int() == 65535);
4069 match(ConI);
4070
4071 op_cost(0);
4072 format %{ %}
4073 interface(CONST_INTER);
4074 %}
4075
4076 operand immI_positive()
4077 %{
4078 predicate(n->get_int() > 0);
4079 match(ConI);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 // BoolTest condition for signed compare
4087 operand immI_cmp_cond()
4088 %{
4089 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4090 match(ConI);
4091
4092 op_cost(0);
4093 format %{ %}
4094 interface(CONST_INTER);
4095 %}
4096
4097 // BoolTest condition for unsigned compare
4098 operand immI_cmpU_cond()
4099 %{
4100 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immL_255()
4109 %{
4110 predicate(n->get_long() == 255L);
4111 match(ConL);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 operand immL_65535()
4119 %{
4120 predicate(n->get_long() == 65535L);
4121 match(ConL);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immL_4294967295()
4129 %{
4130 predicate(n->get_long() == 4294967295L);
4131 match(ConL);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 operand immL_bitmask()
4139 %{
4140 predicate((n->get_long() != 0)
4141 && ((n->get_long() & 0xc000000000000000l) == 0)
4142 && is_power_of_2(n->get_long() + 1));
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immI_bitmask()
4151 %{
4152 predicate((n->get_int() != 0)
4153 && ((n->get_int() & 0xc0000000) == 0)
4154 && is_power_of_2(n->get_int() + 1));
4155 match(ConI);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 operand immL_positive_bitmaskI()
4163 %{
4164 predicate((n->get_long() != 0)
4165 && ((julong)n->get_long() < 0x80000000ULL)
4166 && is_power_of_2(n->get_long() + 1));
4167 match(ConL);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 // Scale values for scaled offset addressing modes (up to long but not quad)
4175 operand immIScale()
4176 %{
4177 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4178 match(ConI);
4179
4180 op_cost(0);
4181 format %{ %}
4182 interface(CONST_INTER);
4183 %}
4184
4185 // 5 bit signed integer
4186 operand immI5()
4187 %{
4188 predicate(Assembler::is_simm(n->get_int(), 5));
4189 match(ConI);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 // 7 bit unsigned integer
4197 operand immIU7()
4198 %{
4199 predicate(Assembler::is_uimm(n->get_int(), 7));
4200 match(ConI);
4201
4202 op_cost(0);
4203 format %{ %}
4204 interface(CONST_INTER);
4205 %}
4206
4207 // Offset for scaled or unscaled immediate loads and stores
4208 operand immIOffset()
4209 %{
4210 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4211 match(ConI);
4212
4213 op_cost(0);
4214 format %{ %}
4215 interface(CONST_INTER);
4216 %}
4217
4218 operand immIOffset1()
4219 %{
4220 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 operand immIOffset2()
4229 %{
4230 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 operand immIOffset4()
4239 %{
4240 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 operand immIOffset8()
4249 %{
4250 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 operand immIOffset16()
4259 %{
4260 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4261 match(ConI);
4262
4263 op_cost(0);
4264 format %{ %}
4265 interface(CONST_INTER);
4266 %}
4267
4268 operand immLOffset()
4269 %{
4270 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4271 match(ConL);
4272
4273 op_cost(0);
4274 format %{ %}
4275 interface(CONST_INTER);
4276 %}
4277
4278 operand immLoffset1()
4279 %{
4280 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4281 match(ConL);
4282
4283 op_cost(0);
4284 format %{ %}
4285 interface(CONST_INTER);
4286 %}
4287
4288 operand immLoffset2()
4289 %{
4290 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4291 match(ConL);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immLoffset4()
4299 %{
4300 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 operand immLoffset8()
4309 %{
4310 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 operand immLoffset16()
4319 %{
4320 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 // 5 bit signed long integer
4329 operand immL5()
4330 %{
4331 predicate(Assembler::is_simm(n->get_long(), 5));
4332 match(ConL);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 // 7 bit unsigned long integer
4340 operand immLU7()
4341 %{
4342 predicate(Assembler::is_uimm(n->get_long(), 7));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 // 8 bit signed value.
4351 operand immI8()
4352 %{
4353 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4354 match(ConI);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 8 bit signed value (simm8), or #simm8 LSL 8.
4362 operand immI8_shift8()
4363 %{
4364 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4365 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4366 match(ConI);
4367
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // 8 bit signed value (simm8), or #simm8 LSL 8.
4374 operand immL8_shift8()
4375 %{
4376 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4377 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4378 match(ConL);
4379
4380 op_cost(0);
4381 format %{ %}
4382 interface(CONST_INTER);
4383 %}
4384
4385 // 8 bit integer valid for vector add sub immediate
4386 operand immBAddSubV()
4387 %{
4388 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4389 match(ConI);
4390
4391 op_cost(0);
4392 format %{ %}
4393 interface(CONST_INTER);
4394 %}
4395
4396 // 32 bit integer valid for add sub immediate
4397 operand immIAddSub()
4398 %{
4399 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4400 match(ConI);
4401 op_cost(0);
4402 format %{ %}
4403 interface(CONST_INTER);
4404 %}
4405
4406 // 32 bit integer valid for vector add sub immediate
4407 operand immIAddSubV()
4408 %{
4409 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4410 match(ConI);
4411
4412 op_cost(0);
4413 format %{ %}
4414 interface(CONST_INTER);
4415 %}
4416
4417 // 32 bit unsigned integer valid for logical immediate
4418
4419 operand immBLog()
4420 %{
4421 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4422 match(ConI);
4423
4424 op_cost(0);
4425 format %{ %}
4426 interface(CONST_INTER);
4427 %}
4428
4429 operand immSLog()
4430 %{
4431 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 operand immILog()
4440 %{
4441 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4442 match(ConI);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // Integer operands 64 bit
4450 // 64 bit immediate
4451 operand immL()
4452 %{
4453 match(ConL);
4454
4455 op_cost(0);
4456 format %{ %}
4457 interface(CONST_INTER);
4458 %}
4459
4460 // 64 bit zero
4461 operand immL0()
4462 %{
4463 predicate(n->get_long() == 0);
4464 match(ConL);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 // 64 bit unit decrement
4472 operand immL_M1()
4473 %{
4474 predicate(n->get_long() == -1);
4475 match(ConL);
4476
4477 op_cost(0);
4478 format %{ %}
4479 interface(CONST_INTER);
4480 %}
4481
4482 // 64 bit integer valid for add sub immediate
4483 operand immLAddSub()
4484 %{
4485 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4486 match(ConL);
4487 op_cost(0);
4488 format %{ %}
4489 interface(CONST_INTER);
4490 %}
4491
4492 // 64 bit integer valid for addv subv immediate
4493 operand immLAddSubV()
4494 %{
4495 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4496 match(ConL);
4497
4498 op_cost(0);
4499 format %{ %}
4500 interface(CONST_INTER);
4501 %}
4502
4503 // 64 bit integer valid for logical immediate
4504 operand immLLog()
4505 %{
4506 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4507 match(ConL);
4508 op_cost(0);
4509 format %{ %}
4510 interface(CONST_INTER);
4511 %}
4512
4513 // Long Immediate: low 32-bit mask
4514 operand immL_32bits()
4515 %{
4516 predicate(n->get_long() == 0xFFFFFFFFL);
4517 match(ConL);
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // Pointer operands
4524 // Pointer Immediate
4525 operand immP()
4526 %{
4527 match(ConP);
4528
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // nullptr Pointer Immediate
4535 operand immP0()
4536 %{
4537 predicate(n->get_ptr() == 0);
4538 match(ConP);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // Pointer Immediate One
4546 // this is used in object initialization (initial object header)
4547 operand immP_1()
4548 %{
4549 predicate(n->get_ptr() == 1);
4550 match(ConP);
4551
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // Card Table Byte Map Base
4558 operand immByteMapBase()
4559 %{
4560 // Get base of card map
4561 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4562 SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
4563 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4564 match(ConP);
4565
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // Float and Double operands
4572 // Double Immediate
4573 operand immD()
4574 %{
4575 match(ConD);
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Double Immediate: +0.0d
4582 operand immD0()
4583 %{
4584 predicate(jlong_cast(n->getd()) == 0);
4585 match(ConD);
4586
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 // constant 'double +0.0'.
4593 operand immDPacked()
4594 %{
4595 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4596 match(ConD);
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 // Float Immediate
4603 operand immF()
4604 %{
4605 match(ConF);
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Float Immediate: +0.0f.
4612 operand immF0()
4613 %{
4614 predicate(jint_cast(n->getf()) == 0);
4615 match(ConF);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 // Half Float (FP16) Immediate
4623 operand immH()
4624 %{
4625 match(ConH);
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 //
4632 operand immFPacked()
4633 %{
4634 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4635 match(ConF);
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Narrow pointer operands
4642 // Narrow Pointer Immediate
4643 operand immN()
4644 %{
4645 match(ConN);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 // Narrow nullptr Pointer Immediate
4653 operand immN0()
4654 %{
4655 predicate(n->get_narrowcon() == 0);
4656 match(ConN);
4657
4658 op_cost(0);
4659 format %{ %}
4660 interface(CONST_INTER);
4661 %}
4662
4663 operand immNKlass()
4664 %{
4665 match(ConNKlass);
4666
4667 op_cost(0);
4668 format %{ %}
4669 interface(CONST_INTER);
4670 %}
4671
4672 // Integer 32 bit Register Operands
4673 // Integer 32 bitRegister (excludes SP)
4674 operand iRegI()
4675 %{
4676 constraint(ALLOC_IN_RC(any_reg32));
4677 match(RegI);
4678 match(iRegINoSp);
4679 op_cost(0);
4680 format %{ %}
4681 interface(REG_INTER);
4682 %}
4683
4684 // Integer 32 bit Register not Special
4685 operand iRegINoSp()
4686 %{
4687 constraint(ALLOC_IN_RC(no_special_reg32));
4688 match(RegI);
4689 op_cost(0);
4690 format %{ %}
4691 interface(REG_INTER);
4692 %}
4693
4694 // Integer 64 bit Register Operands
4695 // Integer 64 bit Register (includes SP)
4696 operand iRegL()
4697 %{
4698 constraint(ALLOC_IN_RC(any_reg));
4699 match(RegL);
4700 match(iRegLNoSp);
4701 op_cost(0);
4702 format %{ %}
4703 interface(REG_INTER);
4704 %}
4705
4706 // Integer 64 bit Register not Special
4707 operand iRegLNoSp()
4708 %{
4709 constraint(ALLOC_IN_RC(no_special_reg));
4710 match(RegL);
4711 match(iRegL_R0);
4712 format %{ %}
4713 interface(REG_INTER);
4714 %}
4715
4716 // Pointer Register Operands
4717 // Pointer Register
4718 operand iRegP()
4719 %{
4720 constraint(ALLOC_IN_RC(ptr_reg));
4721 match(RegP);
4722 match(iRegPNoSp);
4723 match(iRegP_R0);
4724 //match(iRegP_R2);
4725 //match(iRegP_R4);
4726 match(iRegP_R5);
4727 match(thread_RegP);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Pointer 64 bit Register not Special
4734 operand iRegPNoSp()
4735 %{
4736 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4737 match(RegP);
4738 // match(iRegP);
4739 // match(iRegP_R0);
4740 // match(iRegP_R2);
4741 // match(iRegP_R4);
4742 // match(iRegP_R5);
4743 // match(thread_RegP);
4744 op_cost(0);
4745 format %{ %}
4746 interface(REG_INTER);
4747 %}
4748
4749 // This operand is not allowed to use rfp even if
4750 // rfp is not used to hold the frame pointer.
4751 operand iRegPNoSpNoRfp()
4752 %{
4753 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4754 match(RegP);
4755 match(iRegPNoSp);
4756 op_cost(0);
4757 format %{ %}
4758 interface(REG_INTER);
4759 %}
4760
4761 // Pointer 64 bit Register R0 only
4762 operand iRegP_R0()
4763 %{
4764 constraint(ALLOC_IN_RC(r0_reg));
4765 match(RegP);
4766 // match(iRegP);
4767 match(iRegPNoSp);
4768 op_cost(0);
4769 format %{ %}
4770 interface(REG_INTER);
4771 %}
4772
4773 // Pointer 64 bit Register R1 only
4774 operand iRegP_R1()
4775 %{
4776 constraint(ALLOC_IN_RC(r1_reg));
4777 match(RegP);
4778 // match(iRegP);
4779 match(iRegPNoSp);
4780 op_cost(0);
4781 format %{ %}
4782 interface(REG_INTER);
4783 %}
4784
4785 // Pointer 64 bit Register R2 only
4786 operand iRegP_R2()
4787 %{
4788 constraint(ALLOC_IN_RC(r2_reg));
4789 match(RegP);
4790 // match(iRegP);
4791 match(iRegPNoSp);
4792 op_cost(0);
4793 format %{ %}
4794 interface(REG_INTER);
4795 %}
4796
4797 // Pointer 64 bit Register R3 only
4798 operand iRegP_R3()
4799 %{
4800 constraint(ALLOC_IN_RC(r3_reg));
4801 match(RegP);
4802 // match(iRegP);
4803 match(iRegPNoSp);
4804 op_cost(0);
4805 format %{ %}
4806 interface(REG_INTER);
4807 %}
4808
4809 // Pointer 64 bit Register R4 only
4810 operand iRegP_R4()
4811 %{
4812 constraint(ALLOC_IN_RC(r4_reg));
4813 match(RegP);
4814 // match(iRegP);
4815 match(iRegPNoSp);
4816 op_cost(0);
4817 format %{ %}
4818 interface(REG_INTER);
4819 %}
4820
4821 // Pointer 64 bit Register R5 only
4822 operand iRegP_R5()
4823 %{
4824 constraint(ALLOC_IN_RC(r5_reg));
4825 match(RegP);
4826 // match(iRegP);
4827 match(iRegPNoSp);
4828 op_cost(0);
4829 format %{ %}
4830 interface(REG_INTER);
4831 %}
4832
4833 // Pointer 64 bit Register R10 only
4834 operand iRegP_R10()
4835 %{
4836 constraint(ALLOC_IN_RC(r10_reg));
4837 match(RegP);
4838 // match(iRegP);
4839 match(iRegPNoSp);
4840 op_cost(0);
4841 format %{ %}
4842 interface(REG_INTER);
4843 %}
4844
4845 // Long 64 bit Register R0 only
4846 operand iRegL_R0()
4847 %{
4848 constraint(ALLOC_IN_RC(r0_reg));
4849 match(RegL);
4850 match(iRegLNoSp);
4851 op_cost(0);
4852 format %{ %}
4853 interface(REG_INTER);
4854 %}
4855
4856 // Long 64 bit Register R11 only
4857 operand iRegL_R11()
4858 %{
4859 constraint(ALLOC_IN_RC(r11_reg));
4860 match(RegL);
4861 match(iRegLNoSp);
4862 op_cost(0);
4863 format %{ %}
4864 interface(REG_INTER);
4865 %}
4866
4867 // Register R0 only
4868 operand iRegI_R0()
4869 %{
4870 constraint(ALLOC_IN_RC(int_r0_reg));
4871 match(RegI);
4872 match(iRegINoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Register R2 only
4879 operand iRegI_R2()
4880 %{
4881 constraint(ALLOC_IN_RC(int_r2_reg));
4882 match(RegI);
4883 match(iRegINoSp);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Register R3 only
4890 operand iRegI_R3()
4891 %{
4892 constraint(ALLOC_IN_RC(int_r3_reg));
4893 match(RegI);
4894 match(iRegINoSp);
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900
4901 // Register R4 only
4902 operand iRegI_R4()
4903 %{
4904 constraint(ALLOC_IN_RC(int_r4_reg));
4905 match(RegI);
4906 match(iRegINoSp);
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912
4913 // Pointer Register Operands
4914 // Narrow Pointer Register
4915 operand iRegN()
4916 %{
4917 constraint(ALLOC_IN_RC(any_reg32));
4918 match(RegN);
4919 match(iRegNNoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925 // Integer 64 bit Register not Special
4926 operand iRegNNoSp()
4927 %{
4928 constraint(ALLOC_IN_RC(no_special_reg32));
4929 match(RegN);
4930 op_cost(0);
4931 format %{ %}
4932 interface(REG_INTER);
4933 %}
4934
4935 // Float Register
4936 // Float register operands
4937 operand vRegF()
4938 %{
4939 constraint(ALLOC_IN_RC(float_reg));
4940 match(RegF);
4941
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947 // Double Register
4948 // Double register operands
4949 operand vRegD()
4950 %{
4951 constraint(ALLOC_IN_RC(double_reg));
4952 match(RegD);
4953
4954 op_cost(0);
4955 format %{ %}
4956 interface(REG_INTER);
4957 %}
4958
4959 // Generic vector class. This will be used for
4960 // all vector operands, including NEON and SVE.
4961 operand vReg()
4962 %{
4963 constraint(ALLOC_IN_RC(dynamic));
4964 match(VecA);
4965 match(VecD);
4966 match(VecX);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vecA()
4974 %{
4975 constraint(ALLOC_IN_RC(vectora_reg));
4976 match(VecA);
4977
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 operand vecD()
4984 %{
4985 constraint(ALLOC_IN_RC(vectord_reg));
4986 match(VecD);
4987
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vecX()
4994 %{
4995 constraint(ALLOC_IN_RC(vectorx_reg));
4996 match(VecX);
4997
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vRegD_V0()
5004 %{
5005 constraint(ALLOC_IN_RC(v0_reg));
5006 match(RegD);
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vRegD_V1()
5013 %{
5014 constraint(ALLOC_IN_RC(v1_reg));
5015 match(RegD);
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 operand vRegD_V2()
5022 %{
5023 constraint(ALLOC_IN_RC(v2_reg));
5024 match(RegD);
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vRegD_V3()
5031 %{
5032 constraint(ALLOC_IN_RC(v3_reg));
5033 match(RegD);
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand vRegD_V4()
5040 %{
5041 constraint(ALLOC_IN_RC(v4_reg));
5042 match(RegD);
5043 op_cost(0);
5044 format %{ %}
5045 interface(REG_INTER);
5046 %}
5047
5048 operand vRegD_V5()
5049 %{
5050 constraint(ALLOC_IN_RC(v5_reg));
5051 match(RegD);
5052 op_cost(0);
5053 format %{ %}
5054 interface(REG_INTER);
5055 %}
5056
5057 operand vRegD_V6()
5058 %{
5059 constraint(ALLOC_IN_RC(v6_reg));
5060 match(RegD);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 operand vRegD_V7()
5067 %{
5068 constraint(ALLOC_IN_RC(v7_reg));
5069 match(RegD);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 operand vRegD_V12()
5076 %{
5077 constraint(ALLOC_IN_RC(v12_reg));
5078 match(RegD);
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 operand vRegD_V13()
5085 %{
5086 constraint(ALLOC_IN_RC(v13_reg));
5087 match(RegD);
5088 op_cost(0);
5089 format %{ %}
5090 interface(REG_INTER);
5091 %}
5092
5093 operand pReg()
5094 %{
5095 constraint(ALLOC_IN_RC(pr_reg));
5096 match(RegVectMask);
5097 match(pRegGov);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 operand pRegGov()
5104 %{
5105 constraint(ALLOC_IN_RC(gov_pr));
5106 match(RegVectMask);
5107 match(pReg);
5108 op_cost(0);
5109 format %{ %}
5110 interface(REG_INTER);
5111 %}
5112
5113 operand pRegGov_P0()
5114 %{
5115 constraint(ALLOC_IN_RC(p0_reg));
5116 match(RegVectMask);
5117 op_cost(0);
5118 format %{ %}
5119 interface(REG_INTER);
5120 %}
5121
5122 operand pRegGov_P1()
5123 %{
5124 constraint(ALLOC_IN_RC(p1_reg));
5125 match(RegVectMask);
5126 op_cost(0);
5127 format %{ %}
5128 interface(REG_INTER);
5129 %}
5130
5131 // Flags register, used as output of signed compare instructions
5132
5133 // note that on AArch64 we also use this register as the output for
5134 // for floating point compare instructions (CmpF CmpD). this ensures
5135 // that ordered inequality tests use GT, GE, LT or LE none of which
5136 // pass through cases where the result is unordered i.e. one or both
5137 // inputs to the compare is a NaN. this means that the ideal code can
5138 // replace e.g. a GT with an LE and not end up capturing the NaN case
5139 // (where the comparison should always fail). EQ and NE tests are
5140 // always generated in ideal code so that unordered folds into the NE
5141 // case, matching the behaviour of AArch64 NE.
5142 //
5143 // This differs from x86 where the outputs of FP compares use a
5144 // special FP flags registers and where compares based on this
5145 // register are distinguished into ordered inequalities (cmpOpUCF) and
5146 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5147 // to explicitly handle the unordered case in branches. x86 also has
5148 // to include extra CMoveX rules to accept a cmpOpUCF input.
5149
5150 operand rFlagsReg()
5151 %{
5152 constraint(ALLOC_IN_RC(int_flags));
5153 match(RegFlags);
5154
5155 op_cost(0);
5156 format %{ "RFLAGS" %}
5157 interface(REG_INTER);
5158 %}
5159
5160 // Flags register, used as output of unsigned compare instructions
5161 operand rFlagsRegU()
5162 %{
5163 constraint(ALLOC_IN_RC(int_flags));
5164 match(RegFlags);
5165
5166 op_cost(0);
5167 format %{ "RFLAGSU" %}
5168 interface(REG_INTER);
5169 %}
5170
5171 // Special Registers
5172
5173 // Method Register
5174 operand inline_cache_RegP(iRegP reg)
5175 %{
5176 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5177 match(reg);
5178 match(iRegPNoSp);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 // Thread Register
5185 operand thread_RegP(iRegP reg)
5186 %{
5187 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5188 match(reg);
5189 op_cost(0);
5190 format %{ %}
5191 interface(REG_INTER);
5192 %}
5193
5194 //----------Memory Operands----------------------------------------------------
5195
5196 operand indirect(iRegP reg)
5197 %{
5198 constraint(ALLOC_IN_RC(ptr_reg));
5199 match(reg);
5200 op_cost(0);
5201 format %{ "[$reg]" %}
5202 interface(MEMORY_INTER) %{
5203 base($reg);
5204 index(0xffffffff);
5205 scale(0x0);
5206 disp(0x0);
5207 %}
5208 %}
5209
5210 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5211 %{
5212 constraint(ALLOC_IN_RC(ptr_reg));
5213 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5214 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5215 op_cost(0);
5216 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5217 interface(MEMORY_INTER) %{
5218 base($reg);
5219 index($ireg);
5220 scale($scale);
5221 disp(0x0);
5222 %}
5223 %}
5224
5225 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5226 %{
5227 constraint(ALLOC_IN_RC(ptr_reg));
5228 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5229 match(AddP reg (LShiftL lreg scale));
5230 op_cost(0);
5231 format %{ "$reg, $lreg lsl($scale)" %}
5232 interface(MEMORY_INTER) %{
5233 base($reg);
5234 index($lreg);
5235 scale($scale);
5236 disp(0x0);
5237 %}
5238 %}
5239
5240 operand indIndexI2L(iRegP reg, iRegI ireg)
5241 %{
5242 constraint(ALLOC_IN_RC(ptr_reg));
5243 match(AddP reg (ConvI2L ireg));
5244 op_cost(0);
5245 format %{ "$reg, $ireg, 0, I2L" %}
5246 interface(MEMORY_INTER) %{
5247 base($reg);
5248 index($ireg);
5249 scale(0x0);
5250 disp(0x0);
5251 %}
5252 %}
5253
5254 operand indIndex(iRegP reg, iRegL lreg)
5255 %{
5256 constraint(ALLOC_IN_RC(ptr_reg));
5257 match(AddP reg lreg);
5258 op_cost(0);
5259 format %{ "$reg, $lreg" %}
5260 interface(MEMORY_INTER) %{
5261 base($reg);
5262 index($lreg);
5263 scale(0x0);
5264 disp(0x0);
5265 %}
5266 %}
5267
5268 operand indOffI1(iRegP reg, immIOffset1 off)
5269 %{
5270 constraint(ALLOC_IN_RC(ptr_reg));
5271 match(AddP reg off);
5272 op_cost(0);
5273 format %{ "[$reg, $off]" %}
5274 interface(MEMORY_INTER) %{
5275 base($reg);
5276 index(0xffffffff);
5277 scale(0x0);
5278 disp($off);
5279 %}
5280 %}
5281
5282 operand indOffI2(iRegP reg, immIOffset2 off)
5283 %{
5284 constraint(ALLOC_IN_RC(ptr_reg));
5285 match(AddP reg off);
5286 op_cost(0);
5287 format %{ "[$reg, $off]" %}
5288 interface(MEMORY_INTER) %{
5289 base($reg);
5290 index(0xffffffff);
5291 scale(0x0);
5292 disp($off);
5293 %}
5294 %}
5295
5296 operand indOffI4(iRegP reg, immIOffset4 off)
5297 %{
5298 constraint(ALLOC_IN_RC(ptr_reg));
5299 match(AddP reg off);
5300 op_cost(0);
5301 format %{ "[$reg, $off]" %}
5302 interface(MEMORY_INTER) %{
5303 base($reg);
5304 index(0xffffffff);
5305 scale(0x0);
5306 disp($off);
5307 %}
5308 %}
5309
5310 operand indOffI8(iRegP reg, immIOffset8 off)
5311 %{
5312 constraint(ALLOC_IN_RC(ptr_reg));
5313 match(AddP reg off);
5314 op_cost(0);
5315 format %{ "[$reg, $off]" %}
5316 interface(MEMORY_INTER) %{
5317 base($reg);
5318 index(0xffffffff);
5319 scale(0x0);
5320 disp($off);
5321 %}
5322 %}
5323
5324 operand indOffI16(iRegP reg, immIOffset16 off)
5325 %{
5326 constraint(ALLOC_IN_RC(ptr_reg));
5327 match(AddP reg off);
5328 op_cost(0);
5329 format %{ "[$reg, $off]" %}
5330 interface(MEMORY_INTER) %{
5331 base($reg);
5332 index(0xffffffff);
5333 scale(0x0);
5334 disp($off);
5335 %}
5336 %}
5337
5338 operand indOffL1(iRegP reg, immLoffset1 off)
5339 %{
5340 constraint(ALLOC_IN_RC(ptr_reg));
5341 match(AddP reg off);
5342 op_cost(0);
5343 format %{ "[$reg, $off]" %}
5344 interface(MEMORY_INTER) %{
5345 base($reg);
5346 index(0xffffffff);
5347 scale(0x0);
5348 disp($off);
5349 %}
5350 %}
5351
5352 operand indOffL2(iRegP reg, immLoffset2 off)
5353 %{
5354 constraint(ALLOC_IN_RC(ptr_reg));
5355 match(AddP reg off);
5356 op_cost(0);
5357 format %{ "[$reg, $off]" %}
5358 interface(MEMORY_INTER) %{
5359 base($reg);
5360 index(0xffffffff);
5361 scale(0x0);
5362 disp($off);
5363 %}
5364 %}
5365
5366 operand indOffL4(iRegP reg, immLoffset4 off)
5367 %{
5368 constraint(ALLOC_IN_RC(ptr_reg));
5369 match(AddP reg off);
5370 op_cost(0);
5371 format %{ "[$reg, $off]" %}
5372 interface(MEMORY_INTER) %{
5373 base($reg);
5374 index(0xffffffff);
5375 scale(0x0);
5376 disp($off);
5377 %}
5378 %}
5379
5380 operand indOffL8(iRegP reg, immLoffset8 off)
5381 %{
5382 constraint(ALLOC_IN_RC(ptr_reg));
5383 match(AddP reg off);
5384 op_cost(0);
5385 format %{ "[$reg, $off]" %}
5386 interface(MEMORY_INTER) %{
5387 base($reg);
5388 index(0xffffffff);
5389 scale(0x0);
5390 disp($off);
5391 %}
5392 %}
5393
5394 operand indOffL16(iRegP reg, immLoffset16 off)
5395 %{
5396 constraint(ALLOC_IN_RC(ptr_reg));
5397 match(AddP reg off);
5398 op_cost(0);
5399 format %{ "[$reg, $off]" %}
5400 interface(MEMORY_INTER) %{
5401 base($reg);
5402 index(0xffffffff);
5403 scale(0x0);
5404 disp($off);
5405 %}
5406 %}
5407
5408 operand indirectX2P(iRegL reg)
5409 %{
5410 constraint(ALLOC_IN_RC(ptr_reg));
5411 match(CastX2P reg);
5412 op_cost(0);
5413 format %{ "[$reg]\t# long -> ptr" %}
5414 interface(MEMORY_INTER) %{
5415 base($reg);
5416 index(0xffffffff);
5417 scale(0x0);
5418 disp(0x0);
5419 %}
5420 %}
5421
5422 operand indOffX2P(iRegL reg, immLOffset off)
5423 %{
5424 constraint(ALLOC_IN_RC(ptr_reg));
5425 match(AddP (CastX2P reg) off);
5426 op_cost(0);
5427 format %{ "[$reg, $off]\t# long -> ptr" %}
5428 interface(MEMORY_INTER) %{
5429 base($reg);
5430 index(0xffffffff);
5431 scale(0x0);
5432 disp($off);
5433 %}
5434 %}
5435
5436 operand indirectN(iRegN reg)
5437 %{
5438 predicate(CompressedOops::shift() == 0);
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(DecodeN reg);
5441 op_cost(0);
5442 format %{ "[$reg]\t# narrow" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp(0x0);
5448 %}
5449 %}
5450
5451 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5452 %{
5453 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5454 constraint(ALLOC_IN_RC(ptr_reg));
5455 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5456 op_cost(0);
5457 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5458 interface(MEMORY_INTER) %{
5459 base($reg);
5460 index($ireg);
5461 scale($scale);
5462 disp(0x0);
5463 %}
5464 %}
5465
5466 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5467 %{
5468 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5469 constraint(ALLOC_IN_RC(ptr_reg));
5470 match(AddP (DecodeN reg) (LShiftL lreg scale));
5471 op_cost(0);
5472 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5473 interface(MEMORY_INTER) %{
5474 base($reg);
5475 index($lreg);
5476 scale($scale);
5477 disp(0x0);
5478 %}
5479 %}
5480
5481 operand indIndexI2LN(iRegN reg, iRegI ireg)
5482 %{
5483 predicate(CompressedOops::shift() == 0);
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(AddP (DecodeN reg) (ConvI2L ireg));
5486 op_cost(0);
5487 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index($ireg);
5491 scale(0x0);
5492 disp(0x0);
5493 %}
5494 %}
5495
5496 operand indIndexN(iRegN reg, iRegL lreg)
5497 %{
5498 predicate(CompressedOops::shift() == 0);
5499 constraint(ALLOC_IN_RC(ptr_reg));
5500 match(AddP (DecodeN reg) lreg);
5501 op_cost(0);
5502 format %{ "$reg, $lreg\t# narrow" %}
5503 interface(MEMORY_INTER) %{
5504 base($reg);
5505 index($lreg);
5506 scale(0x0);
5507 disp(0x0);
5508 %}
5509 %}
5510
5511 operand indOffIN(iRegN reg, immIOffset off)
5512 %{
5513 predicate(CompressedOops::shift() == 0);
5514 constraint(ALLOC_IN_RC(ptr_reg));
5515 match(AddP (DecodeN reg) off);
5516 op_cost(0);
5517 format %{ "[$reg, $off]\t# narrow" %}
5518 interface(MEMORY_INTER) %{
5519 base($reg);
5520 index(0xffffffff);
5521 scale(0x0);
5522 disp($off);
5523 %}
5524 %}
5525
5526 operand indOffLN(iRegN reg, immLOffset off)
5527 %{
5528 predicate(CompressedOops::shift() == 0);
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(AddP (DecodeN reg) off);
5531 op_cost(0);
5532 format %{ "[$reg, $off]\t# narrow" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index(0xffffffff);
5536 scale(0x0);
5537 disp($off);
5538 %}
5539 %}
5540
5541
5542 //----------Special Memory Operands--------------------------------------------
5543 // Stack Slot Operand - This operand is used for loading and storing temporary
5544 // values on the stack where a match requires a value to
5545 // flow through memory.
5546 operand stackSlotP(sRegP reg)
5547 %{
5548 constraint(ALLOC_IN_RC(stack_slots));
5549 op_cost(100);
5550 // No match rule because this operand is only generated in matching
5551 // match(RegP);
5552 format %{ "[$reg]" %}
5553 interface(MEMORY_INTER) %{
5554 base(0x1e); // RSP
5555 index(0x0); // No Index
5556 scale(0x0); // No Scale
5557 disp($reg); // Stack Offset
5558 %}
5559 %}
5560
5561 operand stackSlotI(sRegI reg)
5562 %{
5563 constraint(ALLOC_IN_RC(stack_slots));
5564 // No match rule because this operand is only generated in matching
5565 // match(RegI);
5566 format %{ "[$reg]" %}
5567 interface(MEMORY_INTER) %{
5568 base(0x1e); // RSP
5569 index(0x0); // No Index
5570 scale(0x0); // No Scale
5571 disp($reg); // Stack Offset
5572 %}
5573 %}
5574
5575 operand stackSlotF(sRegF reg)
5576 %{
5577 constraint(ALLOC_IN_RC(stack_slots));
5578 // No match rule because this operand is only generated in matching
5579 // match(RegF);
5580 format %{ "[$reg]" %}
5581 interface(MEMORY_INTER) %{
5582 base(0x1e); // RSP
5583 index(0x0); // No Index
5584 scale(0x0); // No Scale
5585 disp($reg); // Stack Offset
5586 %}
5587 %}
5588
5589 operand stackSlotD(sRegD reg)
5590 %{
5591 constraint(ALLOC_IN_RC(stack_slots));
5592 // No match rule because this operand is only generated in matching
5593 // match(RegD);
5594 format %{ "[$reg]" %}
5595 interface(MEMORY_INTER) %{
5596 base(0x1e); // RSP
5597 index(0x0); // No Index
5598 scale(0x0); // No Scale
5599 disp($reg); // Stack Offset
5600 %}
5601 %}
5602
5603 operand stackSlotL(sRegL reg)
5604 %{
5605 constraint(ALLOC_IN_RC(stack_slots));
5606 // No match rule because this operand is only generated in matching
5607 // match(RegL);
5608 format %{ "[$reg]" %}
5609 interface(MEMORY_INTER) %{
5610 base(0x1e); // RSP
5611 index(0x0); // No Index
5612 scale(0x0); // No Scale
5613 disp($reg); // Stack Offset
5614 %}
5615 %}
5616
5617 // Operands for expressing Control Flow
5618 // NOTE: Label is a predefined operand which should not be redefined in
5619 // the AD file. It is generically handled within the ADLC.
5620
5621 //----------Conditional Branch Operands----------------------------------------
5622 // Comparison Op - This is the operation of the comparison, and is limited to
5623 // the following set of codes:
5624 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5625 //
5626 // Other attributes of the comparison, such as unsignedness, are specified
5627 // by the comparison instruction that sets a condition code flags register.
5628 // That result is represented by a flags operand whose subtype is appropriate
5629 // to the unsignedness (etc.) of the comparison.
5630 //
5631 // Later, the instruction which matches both the Comparison Op (a Bool) and
5632 // the flags (produced by the Cmp) specifies the coding of the comparison op
5633 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5634
5635 // used for signed integral comparisons and fp comparisons
5636
5637 operand cmpOp()
5638 %{
5639 match(Bool);
5640
5641 format %{ "" %}
5642 interface(COND_INTER) %{
5643 equal(0x0, "eq");
5644 not_equal(0x1, "ne");
5645 less(0xb, "lt");
5646 greater_equal(0xa, "ge");
5647 less_equal(0xd, "le");
5648 greater(0xc, "gt");
5649 overflow(0x6, "vs");
5650 no_overflow(0x7, "vc");
5651 %}
5652 %}
5653
5654 // used for unsigned integral comparisons
5655
5656 operand cmpOpU()
5657 %{
5658 match(Bool);
5659
5660 format %{ "" %}
5661 interface(COND_INTER) %{
5662 equal(0x0, "eq");
5663 not_equal(0x1, "ne");
5664 less(0x3, "lo");
5665 greater_equal(0x2, "hs");
5666 less_equal(0x9, "ls");
5667 greater(0x8, "hi");
5668 overflow(0x6, "vs");
5669 no_overflow(0x7, "vc");
5670 %}
5671 %}
5672
5673 // used for certain integral comparisons which can be
5674 // converted to cbxx or tbxx instructions
5675
5676 operand cmpOpEqNe()
5677 %{
5678 match(Bool);
5679 op_cost(0);
5680 predicate(n->as_Bool()->_test._test == BoolTest::ne
5681 || n->as_Bool()->_test._test == BoolTest::eq);
5682
5683 format %{ "" %}
5684 interface(COND_INTER) %{
5685 equal(0x0, "eq");
5686 not_equal(0x1, "ne");
5687 less(0xb, "lt");
5688 greater_equal(0xa, "ge");
5689 less_equal(0xd, "le");
5690 greater(0xc, "gt");
5691 overflow(0x6, "vs");
5692 no_overflow(0x7, "vc");
5693 %}
5694 %}
5695
5696 // used for certain integral comparisons which can be
5697 // converted to cbxx or tbxx instructions
5698
5699 operand cmpOpLtGe()
5700 %{
5701 match(Bool);
5702 op_cost(0);
5703
5704 predicate(n->as_Bool()->_test._test == BoolTest::lt
5705 || n->as_Bool()->_test._test == BoolTest::ge);
5706
5707 format %{ "" %}
5708 interface(COND_INTER) %{
5709 equal(0x0, "eq");
5710 not_equal(0x1, "ne");
5711 less(0xb, "lt");
5712 greater_equal(0xa, "ge");
5713 less_equal(0xd, "le");
5714 greater(0xc, "gt");
5715 overflow(0x6, "vs");
5716 no_overflow(0x7, "vc");
5717 %}
5718 %}
5719
5720 // used for certain unsigned integral comparisons which can be
5721 // converted to cbxx or tbxx instructions
5722
5723 operand cmpOpUEqNeLeGt()
5724 %{
5725 match(Bool);
5726 op_cost(0);
5727
5728 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5729 n->as_Bool()->_test._test == BoolTest::ne ||
5730 n->as_Bool()->_test._test == BoolTest::le ||
5731 n->as_Bool()->_test._test == BoolTest::gt);
5732
5733 format %{ "" %}
5734 interface(COND_INTER) %{
5735 equal(0x0, "eq");
5736 not_equal(0x1, "ne");
5737 less(0x3, "lo");
5738 greater_equal(0x2, "hs");
5739 less_equal(0x9, "ls");
5740 greater(0x8, "hi");
5741 overflow(0x6, "vs");
5742 no_overflow(0x7, "vc");
5743 %}
5744 %}
5745
5746 // Special operand allowing long args to int ops to be truncated for free
5747
5748 operand iRegL2I(iRegL reg) %{
5749
5750 op_cost(0);
5751
5752 match(ConvL2I reg);
5753
5754 format %{ "l2i($reg)" %}
5755
5756 interface(REG_INTER)
5757 %}
5758
5759 operand iRegL2P(iRegL reg) %{
5760
5761 op_cost(0);
5762
5763 match(CastX2P reg);
5764
5765 format %{ "l2p($reg)" %}
5766
5767 interface(REG_INTER)
5768 %}
5769
5770 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5771 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5772 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5773 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5774
5775 //----------OPERAND CLASSES----------------------------------------------------
5776 // Operand Classes are groups of operands that are used as to simplify
5777 // instruction definitions by not requiring the AD writer to specify
5778 // separate instructions for every form of operand when the
5779 // instruction accepts multiple operand types with the same basic
5780 // encoding and format. The classic case of this is memory operands.
5781
5782 // memory is used to define read/write location for load/store
5783 // instruction defs. we can turn a memory op into an Address
5784
5785 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5786 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5787
5788 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5789 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5790
5791 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5792 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5793
5794 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5795 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5796
5797 // All of the memory operands. For the pipeline description.
5798 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5799 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5800 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5801
5802
5803 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5804 // operations. it allows the src to be either an iRegI or a (ConvL2I
5805 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5806 // can be elided because the 32-bit instruction will just employ the
5807 // lower 32 bits anyway.
5808 //
5809 // n.b. this does not elide all L2I conversions. if the truncated
5810 // value is consumed by more than one operation then the ConvL2I
5811 // cannot be bundled into the consuming nodes so an l2i gets planted
5812 // (actually a movw $dst $src) and the downstream instructions consume
5813 // the result of the l2i as an iRegI input. That's a shame since the
5814 // movw is actually redundant but its not too costly.
5815
5816 opclass iRegIorL2I(iRegI, iRegL2I);
5817 opclass iRegPorL2P(iRegP, iRegL2P);
5818
5819 //----------PIPELINE-----------------------------------------------------------
5820 // Rules which define the behavior of the target architectures pipeline.
5821
5822 // For specific pipelines, eg A53, define the stages of that pipeline
5823 //pipe_desc(ISS, EX1, EX2, WR);
5824 #define ISS S0
5825 #define EX1 S1
5826 #define EX2 S2
5827 #define WR S3
5828
5829 // Integer ALU reg operation
5830 pipeline %{
5831
5832 attributes %{
5833 // ARM instructions are of fixed length
5834 fixed_size_instructions; // Fixed size instructions TODO does
5835 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5836 // ARM instructions come in 32-bit word units
5837 instruction_unit_size = 4; // An instruction is 4 bytes long
5838 instruction_fetch_unit_size = 64; // The processor fetches one line
5839 instruction_fetch_units = 1; // of 64 bytes
5840
5841 // List of nop instructions
5842 nops( MachNop );
5843 %}
5844
5845 // We don't use an actual pipeline model so don't care about resources
5846 // or description. we do use pipeline classes to introduce fixed
5847 // latencies
5848
5849 //----------RESOURCES----------------------------------------------------------
5850 // Resources are the functional units available to the machine
5851
5852 resources( INS0, INS1, INS01 = INS0 | INS1,
5853 ALU0, ALU1, ALU = ALU0 | ALU1,
5854 MAC,
5855 DIV,
5856 BRANCH,
5857 LDST,
5858 NEON_FP);
5859
5860 //----------PIPELINE DESCRIPTION-----------------------------------------------
5861 // Pipeline Description specifies the stages in the machine's pipeline
5862
5863 // Define the pipeline as a generic 6 stage pipeline
5864 pipe_desc(S0, S1, S2, S3, S4, S5);
5865
5866 //----------PIPELINE CLASSES---------------------------------------------------
5867 // Pipeline Classes describe the stages in which input and output are
5868 // referenced by the hardware pipeline.
5869
5870 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5871 %{
5872 single_instruction;
5873 src1 : S1(read);
5874 src2 : S2(read);
5875 dst : S5(write);
5876 INS01 : ISS;
5877 NEON_FP : S5;
5878 %}
5879
5880 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5881 %{
5882 single_instruction;
5883 src1 : S1(read);
5884 src2 : S2(read);
5885 dst : S5(write);
5886 INS01 : ISS;
5887 NEON_FP : S5;
5888 %}
5889
5890 pipe_class fp_uop_s(vRegF dst, vRegF src)
5891 %{
5892 single_instruction;
5893 src : S1(read);
5894 dst : S5(write);
5895 INS01 : ISS;
5896 NEON_FP : S5;
5897 %}
5898
5899 pipe_class fp_uop_d(vRegD dst, vRegD src)
5900 %{
5901 single_instruction;
5902 src : S1(read);
5903 dst : S5(write);
5904 INS01 : ISS;
5905 NEON_FP : S5;
5906 %}
5907
5908 pipe_class fp_d2f(vRegF dst, vRegD src)
5909 %{
5910 single_instruction;
5911 src : S1(read);
5912 dst : S5(write);
5913 INS01 : ISS;
5914 NEON_FP : S5;
5915 %}
5916
5917 pipe_class fp_f2d(vRegD dst, vRegF src)
5918 %{
5919 single_instruction;
5920 src : S1(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5927 %{
5928 single_instruction;
5929 src : S1(read);
5930 dst : S5(write);
5931 INS01 : ISS;
5932 NEON_FP : S5;
5933 %}
5934
5935 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5936 %{
5937 single_instruction;
5938 src : S1(read);
5939 dst : S5(write);
5940 INS01 : ISS;
5941 NEON_FP : S5;
5942 %}
5943
5944 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5945 %{
5946 single_instruction;
5947 src : S1(read);
5948 dst : S5(write);
5949 INS01 : ISS;
5950 NEON_FP : S5;
5951 %}
5952
5953 pipe_class fp_l2f(vRegF dst, iRegL src)
5954 %{
5955 single_instruction;
5956 src : S1(read);
5957 dst : S5(write);
5958 INS01 : ISS;
5959 NEON_FP : S5;
5960 %}
5961
5962 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5963 %{
5964 single_instruction;
5965 src : S1(read);
5966 dst : S5(write);
5967 INS01 : ISS;
5968 NEON_FP : S5;
5969 %}
5970
5971 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5972 %{
5973 single_instruction;
5974 src : S1(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5999 %{
6000 single_instruction;
6001 src1 : S1(read);
6002 src2 : S2(read);
6003 dst : S5(write);
6004 INS0 : ISS;
6005 NEON_FP : S5;
6006 %}
6007
6008 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6009 %{
6010 single_instruction;
6011 src1 : S1(read);
6012 src2 : S2(read);
6013 dst : S5(write);
6014 INS0 : ISS;
6015 NEON_FP : S5;
6016 %}
6017
6018 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6019 %{
6020 single_instruction;
6021 cr : S1(read);
6022 src1 : S1(read);
6023 src2 : S1(read);
6024 dst : S3(write);
6025 INS01 : ISS;
6026 NEON_FP : S3;
6027 %}
6028
6029 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6030 %{
6031 single_instruction;
6032 cr : S1(read);
6033 src1 : S1(read);
6034 src2 : S1(read);
6035 dst : S3(write);
6036 INS01 : ISS;
6037 NEON_FP : S3;
6038 %}
6039
6040 pipe_class fp_imm_s(vRegF dst)
6041 %{
6042 single_instruction;
6043 dst : S3(write);
6044 INS01 : ISS;
6045 NEON_FP : S3;
6046 %}
6047
6048 pipe_class fp_imm_d(vRegD dst)
6049 %{
6050 single_instruction;
6051 dst : S3(write);
6052 INS01 : ISS;
6053 NEON_FP : S3;
6054 %}
6055
6056 pipe_class fp_load_constant_s(vRegF dst)
6057 %{
6058 single_instruction;
6059 dst : S4(write);
6060 INS01 : ISS;
6061 NEON_FP : S4;
6062 %}
6063
6064 pipe_class fp_load_constant_d(vRegD dst)
6065 %{
6066 single_instruction;
6067 dst : S4(write);
6068 INS01 : ISS;
6069 NEON_FP : S4;
6070 %}
6071
6072 //------- Integer ALU operations --------------------------
6073
6074 // Integer ALU reg-reg operation
6075 // Operands needed in EX1, result generated in EX2
6076 // Eg. ADD x0, x1, x2
6077 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6078 %{
6079 single_instruction;
6080 dst : EX2(write);
6081 src1 : EX1(read);
6082 src2 : EX1(read);
6083 INS01 : ISS; // Dual issue as instruction 0 or 1
6084 ALU : EX2;
6085 %}
6086
6087 // Integer ALU reg-reg operation with constant shift
6088 // Shifted register must be available in LATE_ISS instead of EX1
6089 // Eg. ADD x0, x1, x2, LSL #2
6090 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6091 %{
6092 single_instruction;
6093 dst : EX2(write);
6094 src1 : EX1(read);
6095 src2 : ISS(read);
6096 INS01 : ISS;
6097 ALU : EX2;
6098 %}
6099
6100 // Integer ALU reg operation with constant shift
6101 // Eg. LSL x0, x1, #shift
6102 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6103 %{
6104 single_instruction;
6105 dst : EX2(write);
6106 src1 : ISS(read);
6107 INS01 : ISS;
6108 ALU : EX2;
6109 %}
6110
6111 // Integer ALU reg-reg operation with variable shift
6112 // Both operands must be available in LATE_ISS instead of EX1
6113 // Result is available in EX1 instead of EX2
6114 // Eg. LSLV x0, x1, x2
6115 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6116 %{
6117 single_instruction;
6118 dst : EX1(write);
6119 src1 : ISS(read);
6120 src2 : ISS(read);
6121 INS01 : ISS;
6122 ALU : EX1;
6123 %}
6124
6125 // Integer ALU reg-reg operation with extract
6126 // As for _vshift above, but result generated in EX2
6127 // Eg. EXTR x0, x1, x2, #N
6128 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6129 %{
6130 single_instruction;
6131 dst : EX2(write);
6132 src1 : ISS(read);
6133 src2 : ISS(read);
6134 INS1 : ISS; // Can only dual issue as Instruction 1
6135 ALU : EX1;
6136 %}
6137
6138 // Integer ALU reg operation
6139 // Eg. NEG x0, x1
6140 pipe_class ialu_reg(iRegI dst, iRegI src)
6141 %{
6142 single_instruction;
6143 dst : EX2(write);
6144 src : EX1(read);
6145 INS01 : ISS;
6146 ALU : EX2;
6147 %}
6148
6149 // Integer ALU reg mmediate operation
6150 // Eg. ADD x0, x1, #N
6151 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6152 %{
6153 single_instruction;
6154 dst : EX2(write);
6155 src1 : EX1(read);
6156 INS01 : ISS;
6157 ALU : EX2;
6158 %}
6159
6160 // Integer ALU immediate operation (no source operands)
6161 // Eg. MOV x0, #N
6162 pipe_class ialu_imm(iRegI dst)
6163 %{
6164 single_instruction;
6165 dst : EX1(write);
6166 INS01 : ISS;
6167 ALU : EX1;
6168 %}
6169
6170 //------- Compare operation -------------------------------
6171
6172 // Compare reg-reg
6173 // Eg. CMP x0, x1
6174 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6175 %{
6176 single_instruction;
6177 // fixed_latency(16);
6178 cr : EX2(write);
6179 op1 : EX1(read);
6180 op2 : EX1(read);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Compare reg-reg
6186 // Eg. CMP x0, #N
6187 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6188 %{
6189 single_instruction;
6190 // fixed_latency(16);
6191 cr : EX2(write);
6192 op1 : EX1(read);
6193 INS01 : ISS;
6194 ALU : EX2;
6195 %}
6196
6197 //------- Conditional instructions ------------------------
6198
6199 // Conditional no operands
6200 // Eg. CSINC x0, zr, zr, <cond>
6201 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6202 %{
6203 single_instruction;
6204 cr : EX1(read);
6205 dst : EX2(write);
6206 INS01 : ISS;
6207 ALU : EX2;
6208 %}
6209
6210 // Conditional 2 operand
6211 // EG. CSEL X0, X1, X2, <cond>
6212 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6213 %{
6214 single_instruction;
6215 cr : EX1(read);
6216 src1 : EX1(read);
6217 src2 : EX1(read);
6218 dst : EX2(write);
6219 INS01 : ISS;
6220 ALU : EX2;
6221 %}
6222
6223 // Conditional 2 operand
6224 // EG. CSEL X0, X1, X2, <cond>
6225 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6226 %{
6227 single_instruction;
6228 cr : EX1(read);
6229 src : EX1(read);
6230 dst : EX2(write);
6231 INS01 : ISS;
6232 ALU : EX2;
6233 %}
6234
6235 //------- Multiply pipeline operations --------------------
6236
6237 // Multiply reg-reg
6238 // Eg. MUL w0, w1, w2
6239 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6240 %{
6241 single_instruction;
6242 dst : WR(write);
6243 src1 : ISS(read);
6244 src2 : ISS(read);
6245 INS01 : ISS;
6246 MAC : WR;
6247 %}
6248
6249 // Multiply accumulate
6250 // Eg. MADD w0, w1, w2, w3
6251 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6252 %{
6253 single_instruction;
6254 dst : WR(write);
6255 src1 : ISS(read);
6256 src2 : ISS(read);
6257 src3 : ISS(read);
6258 INS01 : ISS;
6259 MAC : WR;
6260 %}
6261
6262 // Eg. MUL w0, w1, w2
6263 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6264 %{
6265 single_instruction;
6266 fixed_latency(3); // Maximum latency for 64 bit mul
6267 dst : WR(write);
6268 src1 : ISS(read);
6269 src2 : ISS(read);
6270 INS01 : ISS;
6271 MAC : WR;
6272 %}
6273
6274 // Multiply accumulate
6275 // Eg. MADD w0, w1, w2, w3
6276 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6277 %{
6278 single_instruction;
6279 fixed_latency(3); // Maximum latency for 64 bit mul
6280 dst : WR(write);
6281 src1 : ISS(read);
6282 src2 : ISS(read);
6283 src3 : ISS(read);
6284 INS01 : ISS;
6285 MAC : WR;
6286 %}
6287
6288 //------- Divide pipeline operations --------------------
6289
6290 // Eg. SDIV w0, w1, w2
6291 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6292 %{
6293 single_instruction;
6294 fixed_latency(8); // Maximum latency for 32 bit divide
6295 dst : WR(write);
6296 src1 : ISS(read);
6297 src2 : ISS(read);
6298 INS0 : ISS; // Can only dual issue as instruction 0
6299 DIV : WR;
6300 %}
6301
6302 // Eg. SDIV x0, x1, x2
6303 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6304 %{
6305 single_instruction;
6306 fixed_latency(16); // Maximum latency for 64 bit divide
6307 dst : WR(write);
6308 src1 : ISS(read);
6309 src2 : ISS(read);
6310 INS0 : ISS; // Can only dual issue as instruction 0
6311 DIV : WR;
6312 %}
6313
6314 //------- Load pipeline operations ------------------------
6315
6316 // Load - prefetch
6317 // Eg. PFRM <mem>
6318 pipe_class iload_prefetch(memory mem)
6319 %{
6320 single_instruction;
6321 mem : ISS(read);
6322 INS01 : ISS;
6323 LDST : WR;
6324 %}
6325
6326 // Load - reg, mem
6327 // Eg. LDR x0, <mem>
6328 pipe_class iload_reg_mem(iRegI dst, memory mem)
6329 %{
6330 single_instruction;
6331 dst : WR(write);
6332 mem : ISS(read);
6333 INS01 : ISS;
6334 LDST : WR;
6335 %}
6336
6337 // Load - reg, reg
6338 // Eg. LDR x0, [sp, x1]
6339 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6340 %{
6341 single_instruction;
6342 dst : WR(write);
6343 src : ISS(read);
6344 INS01 : ISS;
6345 LDST : WR;
6346 %}
6347
6348 //------- Store pipeline operations -----------------------
6349
6350 // Store - zr, mem
6351 // Eg. STR zr, <mem>
6352 pipe_class istore_mem(memory mem)
6353 %{
6354 single_instruction;
6355 mem : ISS(read);
6356 INS01 : ISS;
6357 LDST : WR;
6358 %}
6359
6360 // Store - reg, mem
6361 // Eg. STR x0, <mem>
6362 pipe_class istore_reg_mem(iRegI src, memory mem)
6363 %{
6364 single_instruction;
6365 mem : ISS(read);
6366 src : EX2(read);
6367 INS01 : ISS;
6368 LDST : WR;
6369 %}
6370
6371 // Store - reg, reg
6372 // Eg. STR x0, [sp, x1]
6373 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6374 %{
6375 single_instruction;
6376 dst : ISS(read);
6377 src : EX2(read);
6378 INS01 : ISS;
6379 LDST : WR;
6380 %}
6381
6382 //------- Store pipeline operations -----------------------
6383
6384 // Branch
6385 pipe_class pipe_branch()
6386 %{
6387 single_instruction;
6388 INS01 : ISS;
6389 BRANCH : EX1;
6390 %}
6391
6392 // Conditional branch
6393 pipe_class pipe_branch_cond(rFlagsReg cr)
6394 %{
6395 single_instruction;
6396 cr : EX1(read);
6397 INS01 : ISS;
6398 BRANCH : EX1;
6399 %}
6400
6401 // Compare & Branch
6402 // EG. CBZ/CBNZ
6403 pipe_class pipe_cmp_branch(iRegI op1)
6404 %{
6405 single_instruction;
6406 op1 : EX1(read);
6407 INS01 : ISS;
6408 BRANCH : EX1;
6409 %}
6410
6411 //------- Synchronisation operations ----------------------
6412
6413 // Any operation requiring serialization.
6414 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6415 pipe_class pipe_serial()
6416 %{
6417 single_instruction;
6418 force_serialization;
6419 fixed_latency(16);
6420 INS01 : ISS(2); // Cannot dual issue with any other instruction
6421 LDST : WR;
6422 %}
6423
6424 // Generic big/slow expanded idiom - also serialized
6425 pipe_class pipe_slow()
6426 %{
6427 instruction_count(10);
6428 multiple_bundles;
6429 force_serialization;
6430 fixed_latency(16);
6431 INS01 : ISS(2); // Cannot dual issue with any other instruction
6432 LDST : WR;
6433 %}
6434
6435 // Empty pipeline class
6436 pipe_class pipe_class_empty()
6437 %{
6438 single_instruction;
6439 fixed_latency(0);
6440 %}
6441
6442 // Default pipeline class.
6443 pipe_class pipe_class_default()
6444 %{
6445 single_instruction;
6446 fixed_latency(2);
6447 %}
6448
6449 // Pipeline class for compares.
6450 pipe_class pipe_class_compare()
6451 %{
6452 single_instruction;
6453 fixed_latency(16);
6454 %}
6455
6456 // Pipeline class for memory operations.
6457 pipe_class pipe_class_memory()
6458 %{
6459 single_instruction;
6460 fixed_latency(16);
6461 %}
6462
6463 // Pipeline class for call.
6464 pipe_class pipe_class_call()
6465 %{
6466 single_instruction;
6467 fixed_latency(100);
6468 %}
6469
6470 // Define the class for the Nop node.
6471 define %{
6472 MachNop = pipe_class_empty;
6473 %}
6474
6475 %}
6476 //----------INSTRUCTIONS-------------------------------------------------------
6477 //
6478 // match -- States which machine-independent subtree may be replaced
6479 // by this instruction.
6480 // ins_cost -- The estimated cost of this instruction is used by instruction
6481 // selection to identify a minimum cost tree of machine
6482 // instructions that matches a tree of machine-independent
6483 // instructions.
6484 // format -- A string providing the disassembly for this instruction.
6485 // The value of an instruction's operand may be inserted
6486 // by referring to it with a '$' prefix.
6487 // opcode -- Three instruction opcodes may be provided. These are referred
6488 // to within an encode class as $primary, $secondary, and $tertiary
6489 // rrspectively. The primary opcode is commonly used to
6490 // indicate the type of machine instruction, while secondary
6491 // and tertiary are often used for prefix options or addressing
6492 // modes.
6493 // ins_encode -- A list of encode classes with parameters. The encode class
6494 // name must have been defined in an 'enc_class' specification
6495 // in the encode section of the architecture description.
6496
6497 // ============================================================================
6498 // Memory (Load/Store) Instructions
6499
6500 // Load Instructions
6501
6502 // Load Byte (8 bit signed)
6503 instruct loadB(iRegINoSp dst, memory1 mem)
6504 %{
6505 match(Set dst (LoadB mem));
6506 predicate(!needs_acquiring_load(n));
6507
6508 ins_cost(4 * INSN_COST);
6509 format %{ "ldrsbw $dst, $mem\t# byte" %}
6510
6511 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6512
6513 ins_pipe(iload_reg_mem);
6514 %}
6515
6516 // Load Byte (8 bit signed) into long
6517 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6518 %{
6519 match(Set dst (ConvI2L (LoadB mem)));
6520 predicate(!needs_acquiring_load(n->in(1)));
6521
6522 ins_cost(4 * INSN_COST);
6523 format %{ "ldrsb $dst, $mem\t# byte" %}
6524
6525 ins_encode(aarch64_enc_ldrsb(dst, mem));
6526
6527 ins_pipe(iload_reg_mem);
6528 %}
6529
6530 // Load Byte (8 bit unsigned)
6531 instruct loadUB(iRegINoSp dst, memory1 mem)
6532 %{
6533 match(Set dst (LoadUB mem));
6534 predicate(!needs_acquiring_load(n));
6535
6536 ins_cost(4 * INSN_COST);
6537 format %{ "ldrbw $dst, $mem\t# byte" %}
6538
6539 ins_encode(aarch64_enc_ldrb(dst, mem));
6540
6541 ins_pipe(iload_reg_mem);
6542 %}
6543
6544 // Load Byte (8 bit unsigned) into long
6545 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6546 %{
6547 match(Set dst (ConvI2L (LoadUB mem)));
6548 predicate(!needs_acquiring_load(n->in(1)));
6549
6550 ins_cost(4 * INSN_COST);
6551 format %{ "ldrb $dst, $mem\t# byte" %}
6552
6553 ins_encode(aarch64_enc_ldrb(dst, mem));
6554
6555 ins_pipe(iload_reg_mem);
6556 %}
6557
6558 // Load Short (16 bit signed)
6559 instruct loadS(iRegINoSp dst, memory2 mem)
6560 %{
6561 match(Set dst (LoadS mem));
6562 predicate(!needs_acquiring_load(n));
6563
6564 ins_cost(4 * INSN_COST);
6565 format %{ "ldrshw $dst, $mem\t# short" %}
6566
6567 ins_encode(aarch64_enc_ldrshw(dst, mem));
6568
6569 ins_pipe(iload_reg_mem);
6570 %}
6571
6572 // Load Short (16 bit signed) into long
6573 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6574 %{
6575 match(Set dst (ConvI2L (LoadS mem)));
6576 predicate(!needs_acquiring_load(n->in(1)));
6577
6578 ins_cost(4 * INSN_COST);
6579 format %{ "ldrsh $dst, $mem\t# short" %}
6580
6581 ins_encode(aarch64_enc_ldrsh(dst, mem));
6582
6583 ins_pipe(iload_reg_mem);
6584 %}
6585
6586 // Load Char (16 bit unsigned)
6587 instruct loadUS(iRegINoSp dst, memory2 mem)
6588 %{
6589 match(Set dst (LoadUS mem));
6590 predicate(!needs_acquiring_load(n));
6591
6592 ins_cost(4 * INSN_COST);
6593 format %{ "ldrh $dst, $mem\t# short" %}
6594
6595 ins_encode(aarch64_enc_ldrh(dst, mem));
6596
6597 ins_pipe(iload_reg_mem);
6598 %}
6599
6600 // Load Short/Char (16 bit unsigned) into long
6601 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6602 %{
6603 match(Set dst (ConvI2L (LoadUS mem)));
6604 predicate(!needs_acquiring_load(n->in(1)));
6605
6606 ins_cost(4 * INSN_COST);
6607 format %{ "ldrh $dst, $mem\t# short" %}
6608
6609 ins_encode(aarch64_enc_ldrh(dst, mem));
6610
6611 ins_pipe(iload_reg_mem);
6612 %}
6613
6614 // Load Integer (32 bit signed)
6615 instruct loadI(iRegINoSp dst, memory4 mem)
6616 %{
6617 match(Set dst (LoadI mem));
6618 predicate(!needs_acquiring_load(n));
6619
6620 ins_cost(4 * INSN_COST);
6621 format %{ "ldrw $dst, $mem\t# int" %}
6622
6623 ins_encode(aarch64_enc_ldrw(dst, mem));
6624
6625 ins_pipe(iload_reg_mem);
6626 %}
6627
6628 // Load Integer (32 bit signed) into long
6629 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6630 %{
6631 match(Set dst (ConvI2L (LoadI mem)));
6632 predicate(!needs_acquiring_load(n->in(1)));
6633
6634 ins_cost(4 * INSN_COST);
6635 format %{ "ldrsw $dst, $mem\t# int" %}
6636
6637 ins_encode(aarch64_enc_ldrsw(dst, mem));
6638
6639 ins_pipe(iload_reg_mem);
6640 %}
6641
6642 // Load Integer (32 bit unsigned) into long
6643 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6644 %{
6645 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6646 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6647
6648 ins_cost(4 * INSN_COST);
6649 format %{ "ldrw $dst, $mem\t# int" %}
6650
6651 ins_encode(aarch64_enc_ldrw(dst, mem));
6652
6653 ins_pipe(iload_reg_mem);
6654 %}
6655
6656 // Load Long (64 bit signed)
6657 instruct loadL(iRegLNoSp dst, memory8 mem)
6658 %{
6659 match(Set dst (LoadL mem));
6660 predicate(!needs_acquiring_load(n));
6661
6662 ins_cost(4 * INSN_COST);
6663 format %{ "ldr $dst, $mem\t# int" %}
6664
6665 ins_encode(aarch64_enc_ldr(dst, mem));
6666
6667 ins_pipe(iload_reg_mem);
6668 %}
6669
6670 // Load Range
6671 instruct loadRange(iRegINoSp dst, memory4 mem)
6672 %{
6673 match(Set dst (LoadRange mem));
6674
6675 ins_cost(4 * INSN_COST);
6676 format %{ "ldrw $dst, $mem\t# range" %}
6677
6678 ins_encode(aarch64_enc_ldrw(dst, mem));
6679
6680 ins_pipe(iload_reg_mem);
6681 %}
6682
6683 // Load Pointer
6684 instruct loadP(iRegPNoSp dst, memory8 mem)
6685 %{
6686 match(Set dst (LoadP mem));
6687 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6688
6689 ins_cost(4 * INSN_COST);
6690 format %{ "ldr $dst, $mem\t# ptr" %}
6691
6692 ins_encode(aarch64_enc_ldr(dst, mem));
6693
6694 ins_pipe(iload_reg_mem);
6695 %}
6696
6697 // Load Compressed Pointer
6698 instruct loadN(iRegNNoSp dst, memory4 mem)
6699 %{
6700 match(Set dst (LoadN mem));
6701 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6702
6703 ins_cost(4 * INSN_COST);
6704 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6705
6706 ins_encode(aarch64_enc_ldrw(dst, mem));
6707
6708 ins_pipe(iload_reg_mem);
6709 %}
6710
6711 // Load Klass Pointer
6712 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6713 %{
6714 match(Set dst (LoadKlass mem));
6715 predicate(!needs_acquiring_load(n));
6716
6717 ins_cost(4 * INSN_COST);
6718 format %{ "ldr $dst, $mem\t# class" %}
6719
6720 ins_encode(aarch64_enc_ldr(dst, mem));
6721
6722 ins_pipe(iload_reg_mem);
6723 %}
6724
6725 // Load Narrow Klass Pointer
6726 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6727 %{
6728 match(Set dst (LoadNKlass mem));
6729 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6730
6731 ins_cost(4 * INSN_COST);
6732 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6733
6734 ins_encode(aarch64_enc_ldrw(dst, mem));
6735
6736 ins_pipe(iload_reg_mem);
6737 %}
6738
6739 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6740 %{
6741 match(Set dst (LoadNKlass mem));
6742 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6743
6744 ins_cost(4 * INSN_COST);
6745 format %{
6746 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6747 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6748 %}
6749 ins_encode %{
6750 // inlined aarch64_enc_ldrw
6751 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6752 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6753 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6754 %}
6755 ins_pipe(iload_reg_mem);
6756 %}
6757
6758 // Load Float
6759 instruct loadF(vRegF dst, memory4 mem)
6760 %{
6761 match(Set dst (LoadF mem));
6762 predicate(!needs_acquiring_load(n));
6763
6764 ins_cost(4 * INSN_COST);
6765 format %{ "ldrs $dst, $mem\t# float" %}
6766
6767 ins_encode( aarch64_enc_ldrs(dst, mem) );
6768
6769 ins_pipe(pipe_class_memory);
6770 %}
6771
6772 // Load Double
6773 instruct loadD(vRegD dst, memory8 mem)
6774 %{
6775 match(Set dst (LoadD mem));
6776 predicate(!needs_acquiring_load(n));
6777
6778 ins_cost(4 * INSN_COST);
6779 format %{ "ldrd $dst, $mem\t# double" %}
6780
6781 ins_encode( aarch64_enc_ldrd(dst, mem) );
6782
6783 ins_pipe(pipe_class_memory);
6784 %}
6785
6786
6787 // Load Int Constant
6788 instruct loadConI(iRegINoSp dst, immI src)
6789 %{
6790 match(Set dst src);
6791
6792 ins_cost(INSN_COST);
6793 format %{ "mov $dst, $src\t# int" %}
6794
6795 ins_encode( aarch64_enc_movw_imm(dst, src) );
6796
6797 ins_pipe(ialu_imm);
6798 %}
6799
6800 // Load Long Constant
6801 instruct loadConL(iRegLNoSp dst, immL src)
6802 %{
6803 match(Set dst src);
6804
6805 ins_cost(INSN_COST);
6806 format %{ "mov $dst, $src\t# long" %}
6807
6808 ins_encode( aarch64_enc_mov_imm(dst, src) );
6809
6810 ins_pipe(ialu_imm);
6811 %}
6812
6813 // Load Pointer Constant
6814
6815 instruct loadConP(iRegPNoSp dst, immP con)
6816 %{
6817 match(Set dst con);
6818
6819 ins_cost(INSN_COST * 4);
6820 format %{
6821 "mov $dst, $con\t# ptr\n\t"
6822 %}
6823
6824 ins_encode(aarch64_enc_mov_p(dst, con));
6825
6826 ins_pipe(ialu_imm);
6827 %}
6828
6829 // Load Null Pointer Constant
6830
6831 instruct loadConP0(iRegPNoSp dst, immP0 con)
6832 %{
6833 match(Set dst con);
6834
6835 ins_cost(INSN_COST);
6836 format %{ "mov $dst, $con\t# nullptr ptr" %}
6837
6838 ins_encode(aarch64_enc_mov_p0(dst, con));
6839
6840 ins_pipe(ialu_imm);
6841 %}
6842
6843 // Load Pointer Constant One
6844
6845 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6846 %{
6847 match(Set dst con);
6848
6849 ins_cost(INSN_COST);
6850 format %{ "mov $dst, $con\t# nullptr ptr" %}
6851
6852 ins_encode(aarch64_enc_mov_p1(dst, con));
6853
6854 ins_pipe(ialu_imm);
6855 %}
6856
6857 // Load Byte Map Base Constant
6858
6859 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6860 %{
6861 match(Set dst con);
6862
6863 ins_cost(INSN_COST);
6864 format %{ "adr $dst, $con\t# Byte Map Base" %}
6865
6866 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6867
6868 ins_pipe(ialu_imm);
6869 %}
6870
6871 // Load Narrow Pointer Constant
6872
6873 instruct loadConN(iRegNNoSp dst, immN con)
6874 %{
6875 match(Set dst con);
6876
6877 ins_cost(INSN_COST * 4);
6878 format %{ "mov $dst, $con\t# compressed ptr" %}
6879
6880 ins_encode(aarch64_enc_mov_n(dst, con));
6881
6882 ins_pipe(ialu_imm);
6883 %}
6884
6885 // Load Narrow Null Pointer Constant
6886
6887 instruct loadConN0(iRegNNoSp dst, immN0 con)
6888 %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST);
6892 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6893
6894 ins_encode(aarch64_enc_mov_n0(dst, con));
6895
6896 ins_pipe(ialu_imm);
6897 %}
6898
6899 // Load Narrow Klass Constant
6900
6901 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6902 %{
6903 match(Set dst con);
6904
6905 ins_cost(INSN_COST);
6906 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6907
6908 ins_encode(aarch64_enc_mov_nk(dst, con));
6909
6910 ins_pipe(ialu_imm);
6911 %}
6912
6913 // Load Packed Float Constant
6914
6915 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6916 match(Set dst con);
6917 ins_cost(INSN_COST * 4);
6918 format %{ "fmovs $dst, $con"%}
6919 ins_encode %{
6920 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6921 %}
6922
6923 ins_pipe(fp_imm_s);
6924 %}
6925
6926 // Load Float Constant
6927
6928 instruct loadConF(vRegF dst, immF con) %{
6929 match(Set dst con);
6930
6931 ins_cost(INSN_COST * 4);
6932
6933 format %{
6934 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6935 %}
6936
6937 ins_encode %{
6938 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6939 %}
6940
6941 ins_pipe(fp_load_constant_s);
6942 %}
6943
6944 // Load Packed Double Constant
6945
6946 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6947 match(Set dst con);
6948 ins_cost(INSN_COST);
6949 format %{ "fmovd $dst, $con"%}
6950 ins_encode %{
6951 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6952 %}
6953
6954 ins_pipe(fp_imm_d);
6955 %}
6956
6957 // Load Double Constant
6958
6959 instruct loadConD(vRegD dst, immD con) %{
6960 match(Set dst con);
6961
6962 ins_cost(INSN_COST * 5);
6963 format %{
6964 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6965 %}
6966
6967 ins_encode %{
6968 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6969 %}
6970
6971 ins_pipe(fp_load_constant_d);
6972 %}
6973
6974 // Load Half Float Constant
6975 // The "ldr" instruction loads a 32-bit word from the constant pool into a
6976 // 32-bit register but only the bottom half will be populated and the top
6977 // 16 bits are zero.
6978 instruct loadConH(vRegF dst, immH con) %{
6979 match(Set dst con);
6980 format %{
6981 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
6982 %}
6983 ins_encode %{
6984 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6985 %}
6986 ins_pipe(fp_load_constant_s);
6987 %}
6988
6989 // Store Instructions
6990
6991 // Store Byte
6992 instruct storeB(iRegIorL2I src, memory1 mem)
6993 %{
6994 match(Set mem (StoreB mem src));
6995 predicate(!needs_releasing_store(n));
6996
6997 ins_cost(INSN_COST);
6998 format %{ "strb $src, $mem\t# byte" %}
6999
7000 ins_encode(aarch64_enc_strb(src, mem));
7001
7002 ins_pipe(istore_reg_mem);
7003 %}
7004
7005
7006 instruct storeimmB0(immI0 zero, memory1 mem)
7007 %{
7008 match(Set mem (StoreB mem zero));
7009 predicate(!needs_releasing_store(n));
7010
7011 ins_cost(INSN_COST);
7012 format %{ "strb rscractch2, $mem\t# byte" %}
7013
7014 ins_encode(aarch64_enc_strb0(mem));
7015
7016 ins_pipe(istore_mem);
7017 %}
7018
7019 // Store Char/Short
7020 instruct storeC(iRegIorL2I src, memory2 mem)
7021 %{
7022 match(Set mem (StoreC mem src));
7023 predicate(!needs_releasing_store(n));
7024
7025 ins_cost(INSN_COST);
7026 format %{ "strh $src, $mem\t# short" %}
7027
7028 ins_encode(aarch64_enc_strh(src, mem));
7029
7030 ins_pipe(istore_reg_mem);
7031 %}
7032
7033 instruct storeimmC0(immI0 zero, memory2 mem)
7034 %{
7035 match(Set mem (StoreC mem zero));
7036 predicate(!needs_releasing_store(n));
7037
7038 ins_cost(INSN_COST);
7039 format %{ "strh zr, $mem\t# short" %}
7040
7041 ins_encode(aarch64_enc_strh0(mem));
7042
7043 ins_pipe(istore_mem);
7044 %}
7045
7046 // Store Integer
7047
7048 instruct storeI(iRegIorL2I src, memory4 mem)
7049 %{
7050 match(Set mem(StoreI mem src));
7051 predicate(!needs_releasing_store(n));
7052
7053 ins_cost(INSN_COST);
7054 format %{ "strw $src, $mem\t# int" %}
7055
7056 ins_encode(aarch64_enc_strw(src, mem));
7057
7058 ins_pipe(istore_reg_mem);
7059 %}
7060
7061 instruct storeimmI0(immI0 zero, memory4 mem)
7062 %{
7063 match(Set mem(StoreI mem zero));
7064 predicate(!needs_releasing_store(n));
7065
7066 ins_cost(INSN_COST);
7067 format %{ "strw zr, $mem\t# int" %}
7068
7069 ins_encode(aarch64_enc_strw0(mem));
7070
7071 ins_pipe(istore_mem);
7072 %}
7073
7074 // Store Long (64 bit signed)
7075 instruct storeL(iRegL src, memory8 mem)
7076 %{
7077 match(Set mem (StoreL mem src));
7078 predicate(!needs_releasing_store(n));
7079
7080 ins_cost(INSN_COST);
7081 format %{ "str $src, $mem\t# int" %}
7082
7083 ins_encode(aarch64_enc_str(src, mem));
7084
7085 ins_pipe(istore_reg_mem);
7086 %}
7087
7088 // Store Long (64 bit signed)
7089 instruct storeimmL0(immL0 zero, memory8 mem)
7090 %{
7091 match(Set mem (StoreL mem zero));
7092 predicate(!needs_releasing_store(n));
7093
7094 ins_cost(INSN_COST);
7095 format %{ "str zr, $mem\t# int" %}
7096
7097 ins_encode(aarch64_enc_str0(mem));
7098
7099 ins_pipe(istore_mem);
7100 %}
7101
7102 // Store Pointer
7103 instruct storeP(iRegP src, memory8 mem)
7104 %{
7105 match(Set mem (StoreP mem src));
7106 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7107
7108 ins_cost(INSN_COST);
7109 format %{ "str $src, $mem\t# ptr" %}
7110
7111 ins_encode(aarch64_enc_str(src, mem));
7112
7113 ins_pipe(istore_reg_mem);
7114 %}
7115
7116 // Store Pointer
7117 instruct storeimmP0(immP0 zero, memory8 mem)
7118 %{
7119 match(Set mem (StoreP mem zero));
7120 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7121
7122 ins_cost(INSN_COST);
7123 format %{ "str zr, $mem\t# ptr" %}
7124
7125 ins_encode(aarch64_enc_str0(mem));
7126
7127 ins_pipe(istore_mem);
7128 %}
7129
7130 // Store Compressed Pointer
7131 instruct storeN(iRegN src, memory4 mem)
7132 %{
7133 match(Set mem (StoreN mem src));
7134 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7135
7136 ins_cost(INSN_COST);
7137 format %{ "strw $src, $mem\t# compressed ptr" %}
7138
7139 ins_encode(aarch64_enc_strw(src, mem));
7140
7141 ins_pipe(istore_reg_mem);
7142 %}
7143
7144 instruct storeImmN0(immN0 zero, memory4 mem)
7145 %{
7146 match(Set mem (StoreN mem zero));
7147 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7148
7149 ins_cost(INSN_COST);
7150 format %{ "strw zr, $mem\t# compressed ptr" %}
7151
7152 ins_encode(aarch64_enc_strw0(mem));
7153
7154 ins_pipe(istore_mem);
7155 %}
7156
7157 // Store Float
7158 instruct storeF(vRegF src, memory4 mem)
7159 %{
7160 match(Set mem (StoreF mem src));
7161 predicate(!needs_releasing_store(n));
7162
7163 ins_cost(INSN_COST);
7164 format %{ "strs $src, $mem\t# float" %}
7165
7166 ins_encode( aarch64_enc_strs(src, mem) );
7167
7168 ins_pipe(pipe_class_memory);
7169 %}
7170
7171 // TODO
7172 // implement storeImmF0 and storeFImmPacked
7173
7174 // Store Double
7175 instruct storeD(vRegD src, memory8 mem)
7176 %{
7177 match(Set mem (StoreD mem src));
7178 predicate(!needs_releasing_store(n));
7179
7180 ins_cost(INSN_COST);
7181 format %{ "strd $src, $mem\t# double" %}
7182
7183 ins_encode( aarch64_enc_strd(src, mem) );
7184
7185 ins_pipe(pipe_class_memory);
7186 %}
7187
7188 // Store Compressed Klass Pointer
7189 instruct storeNKlass(iRegN src, memory4 mem)
7190 %{
7191 predicate(!needs_releasing_store(n));
7192 match(Set mem (StoreNKlass mem src));
7193
7194 ins_cost(INSN_COST);
7195 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7196
7197 ins_encode(aarch64_enc_strw(src, mem));
7198
7199 ins_pipe(istore_reg_mem);
7200 %}
7201
7202 // TODO
7203 // implement storeImmD0 and storeDImmPacked
7204
7205 // prefetch instructions
7206 // Must be safe to execute with invalid address (cannot fault).
7207
7208 instruct prefetchalloc( memory8 mem ) %{
7209 match(PrefetchAllocation mem);
7210
7211 ins_cost(INSN_COST);
7212 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7213
7214 ins_encode( aarch64_enc_prefetchw(mem) );
7215
7216 ins_pipe(iload_prefetch);
7217 %}
7218
7219 // ---------------- volatile loads and stores ----------------
7220
7221 // Load Byte (8 bit signed)
7222 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7223 %{
7224 match(Set dst (LoadB mem));
7225
7226 ins_cost(VOLATILE_REF_COST);
7227 format %{ "ldarsb $dst, $mem\t# byte" %}
7228
7229 ins_encode(aarch64_enc_ldarsb(dst, mem));
7230
7231 ins_pipe(pipe_serial);
7232 %}
7233
7234 // Load Byte (8 bit signed) into long
7235 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7236 %{
7237 match(Set dst (ConvI2L (LoadB mem)));
7238
7239 ins_cost(VOLATILE_REF_COST);
7240 format %{ "ldarsb $dst, $mem\t# byte" %}
7241
7242 ins_encode(aarch64_enc_ldarsb(dst, mem));
7243
7244 ins_pipe(pipe_serial);
7245 %}
7246
7247 // Load Byte (8 bit unsigned)
7248 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7249 %{
7250 match(Set dst (LoadUB mem));
7251
7252 ins_cost(VOLATILE_REF_COST);
7253 format %{ "ldarb $dst, $mem\t# byte" %}
7254
7255 ins_encode(aarch64_enc_ldarb(dst, mem));
7256
7257 ins_pipe(pipe_serial);
7258 %}
7259
7260 // Load Byte (8 bit unsigned) into long
7261 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7262 %{
7263 match(Set dst (ConvI2L (LoadUB mem)));
7264
7265 ins_cost(VOLATILE_REF_COST);
7266 format %{ "ldarb $dst, $mem\t# byte" %}
7267
7268 ins_encode(aarch64_enc_ldarb(dst, mem));
7269
7270 ins_pipe(pipe_serial);
7271 %}
7272
7273 // Load Short (16 bit signed)
7274 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7275 %{
7276 match(Set dst (LoadS mem));
7277
7278 ins_cost(VOLATILE_REF_COST);
7279 format %{ "ldarshw $dst, $mem\t# short" %}
7280
7281 ins_encode(aarch64_enc_ldarshw(dst, mem));
7282
7283 ins_pipe(pipe_serial);
7284 %}
7285
7286 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (LoadUS mem));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarhw $dst, $mem\t# short" %}
7292
7293 ins_encode(aarch64_enc_ldarhw(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Short/Char (16 bit unsigned) into long
7299 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7300 %{
7301 match(Set dst (ConvI2L (LoadUS mem)));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarh $dst, $mem\t# short" %}
7305
7306 ins_encode(aarch64_enc_ldarh(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 // Load Short/Char (16 bit signed) into long
7312 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7313 %{
7314 match(Set dst (ConvI2L (LoadS mem)));
7315
7316 ins_cost(VOLATILE_REF_COST);
7317 format %{ "ldarh $dst, $mem\t# short" %}
7318
7319 ins_encode(aarch64_enc_ldarsh(dst, mem));
7320
7321 ins_pipe(pipe_serial);
7322 %}
7323
7324 // Load Integer (32 bit signed)
7325 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7326 %{
7327 match(Set dst (LoadI mem));
7328
7329 ins_cost(VOLATILE_REF_COST);
7330 format %{ "ldarw $dst, $mem\t# int" %}
7331
7332 ins_encode(aarch64_enc_ldarw(dst, mem));
7333
7334 ins_pipe(pipe_serial);
7335 %}
7336
7337 // Load Integer (32 bit unsigned) into long
7338 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7339 %{
7340 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7341
7342 ins_cost(VOLATILE_REF_COST);
7343 format %{ "ldarw $dst, $mem\t# int" %}
7344
7345 ins_encode(aarch64_enc_ldarw(dst, mem));
7346
7347 ins_pipe(pipe_serial);
7348 %}
7349
7350 // Load Long (64 bit signed)
7351 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7352 %{
7353 match(Set dst (LoadL mem));
7354
7355 ins_cost(VOLATILE_REF_COST);
7356 format %{ "ldar $dst, $mem\t# int" %}
7357
7358 ins_encode(aarch64_enc_ldar(dst, mem));
7359
7360 ins_pipe(pipe_serial);
7361 %}
7362
7363 // Load Pointer
7364 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7365 %{
7366 match(Set dst (LoadP mem));
7367 predicate(n->as_Load()->barrier_data() == 0);
7368
7369 ins_cost(VOLATILE_REF_COST);
7370 format %{ "ldar $dst, $mem\t# ptr" %}
7371
7372 ins_encode(aarch64_enc_ldar(dst, mem));
7373
7374 ins_pipe(pipe_serial);
7375 %}
7376
7377 // Load Compressed Pointer
7378 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7379 %{
7380 match(Set dst (LoadN mem));
7381 predicate(n->as_Load()->barrier_data() == 0);
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7385
7386 ins_encode(aarch64_enc_ldarw(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Float
7392 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set dst (LoadF mem));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "ldars $dst, $mem\t# float" %}
7398
7399 ins_encode( aarch64_enc_fldars(dst, mem) );
7400
7401 ins_pipe(pipe_serial);
7402 %}
7403
7404 // Load Double
7405 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7406 %{
7407 match(Set dst (LoadD mem));
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "ldard $dst, $mem\t# double" %}
7411
7412 ins_encode( aarch64_enc_fldard(dst, mem) );
7413
7414 ins_pipe(pipe_serial);
7415 %}
7416
7417 // Store Byte
7418 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7419 %{
7420 match(Set mem (StoreB mem src));
7421
7422 ins_cost(VOLATILE_REF_COST);
7423 format %{ "stlrb $src, $mem\t# byte" %}
7424
7425 ins_encode(aarch64_enc_stlrb(src, mem));
7426
7427 ins_pipe(pipe_class_memory);
7428 %}
7429
7430 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set mem (StoreB mem zero));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "stlrb zr, $mem\t# byte" %}
7436
7437 ins_encode(aarch64_enc_stlrb0(mem));
7438
7439 ins_pipe(pipe_class_memory);
7440 %}
7441
7442 // Store Char/Short
7443 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set mem (StoreC mem src));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "stlrh $src, $mem\t# short" %}
7449
7450 ins_encode(aarch64_enc_stlrh(src, mem));
7451
7452 ins_pipe(pipe_class_memory);
7453 %}
7454
7455 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set mem (StoreC mem zero));
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "stlrh zr, $mem\t# short" %}
7461
7462 ins_encode(aarch64_enc_stlrh0(mem));
7463
7464 ins_pipe(pipe_class_memory);
7465 %}
7466
7467 // Store Integer
7468
7469 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7470 %{
7471 match(Set mem(StoreI mem src));
7472
7473 ins_cost(VOLATILE_REF_COST);
7474 format %{ "stlrw $src, $mem\t# int" %}
7475
7476 ins_encode(aarch64_enc_stlrw(src, mem));
7477
7478 ins_pipe(pipe_class_memory);
7479 %}
7480
7481 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7482 %{
7483 match(Set mem(StoreI mem zero));
7484
7485 ins_cost(VOLATILE_REF_COST);
7486 format %{ "stlrw zr, $mem\t# int" %}
7487
7488 ins_encode(aarch64_enc_stlrw0(mem));
7489
7490 ins_pipe(pipe_class_memory);
7491 %}
7492
7493 // Store Long (64 bit signed)
7494 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7495 %{
7496 match(Set mem (StoreL mem src));
7497
7498 ins_cost(VOLATILE_REF_COST);
7499 format %{ "stlr $src, $mem\t# int" %}
7500
7501 ins_encode(aarch64_enc_stlr(src, mem));
7502
7503 ins_pipe(pipe_class_memory);
7504 %}
7505
7506 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem (StoreL mem zero));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlr zr, $mem\t# int" %}
7512
7513 ins_encode(aarch64_enc_stlr0(mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 // Store Pointer
7519 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem (StoreP mem src));
7522 predicate(n->as_Store()->barrier_data() == 0);
7523
7524 ins_cost(VOLATILE_REF_COST);
7525 format %{ "stlr $src, $mem\t# ptr" %}
7526
7527 ins_encode(aarch64_enc_stlr(src, mem));
7528
7529 ins_pipe(pipe_class_memory);
7530 %}
7531
7532 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7533 %{
7534 match(Set mem (StoreP mem zero));
7535 predicate(n->as_Store()->barrier_data() == 0);
7536
7537 ins_cost(VOLATILE_REF_COST);
7538 format %{ "stlr zr, $mem\t# ptr" %}
7539
7540 ins_encode(aarch64_enc_stlr0(mem));
7541
7542 ins_pipe(pipe_class_memory);
7543 %}
7544
7545 // Store Compressed Pointer
7546 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7547 %{
7548 match(Set mem (StoreN mem src));
7549 predicate(n->as_Store()->barrier_data() == 0);
7550
7551 ins_cost(VOLATILE_REF_COST);
7552 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7553
7554 ins_encode(aarch64_enc_stlrw(src, mem));
7555
7556 ins_pipe(pipe_class_memory);
7557 %}
7558
7559 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7560 %{
7561 match(Set mem (StoreN mem zero));
7562 predicate(n->as_Store()->barrier_data() == 0);
7563
7564 ins_cost(VOLATILE_REF_COST);
7565 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7566
7567 ins_encode(aarch64_enc_stlrw0(mem));
7568
7569 ins_pipe(pipe_class_memory);
7570 %}
7571
7572 // Store Float
7573 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7574 %{
7575 match(Set mem (StoreF mem src));
7576
7577 ins_cost(VOLATILE_REF_COST);
7578 format %{ "stlrs $src, $mem\t# float" %}
7579
7580 ins_encode( aarch64_enc_fstlrs(src, mem) );
7581
7582 ins_pipe(pipe_class_memory);
7583 %}
7584
7585 // TODO
7586 // implement storeImmF0 and storeFImmPacked
7587
7588 // Store Double
7589 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7590 %{
7591 match(Set mem (StoreD mem src));
7592
7593 ins_cost(VOLATILE_REF_COST);
7594 format %{ "stlrd $src, $mem\t# double" %}
7595
7596 ins_encode( aarch64_enc_fstlrd(src, mem) );
7597
7598 ins_pipe(pipe_class_memory);
7599 %}
7600
7601 // ---------------- end of volatile loads and stores ----------------
7602
7603 instruct cacheWB(indirect addr)
7604 %{
7605 predicate(VM_Version::supports_data_cache_line_flush());
7606 match(CacheWB addr);
7607
7608 ins_cost(100);
7609 format %{"cache wb $addr" %}
7610 ins_encode %{
7611 assert($addr->index_position() < 0, "should be");
7612 assert($addr$$disp == 0, "should be");
7613 __ cache_wb(Address($addr$$base$$Register, 0));
7614 %}
7615 ins_pipe(pipe_slow); // XXX
7616 %}
7617
7618 instruct cacheWBPreSync()
7619 %{
7620 predicate(VM_Version::supports_data_cache_line_flush());
7621 match(CacheWBPreSync);
7622
7623 ins_cost(100);
7624 format %{"cache wb presync" %}
7625 ins_encode %{
7626 __ cache_wbsync(true);
7627 %}
7628 ins_pipe(pipe_slow); // XXX
7629 %}
7630
7631 instruct cacheWBPostSync()
7632 %{
7633 predicate(VM_Version::supports_data_cache_line_flush());
7634 match(CacheWBPostSync);
7635
7636 ins_cost(100);
7637 format %{"cache wb postsync" %}
7638 ins_encode %{
7639 __ cache_wbsync(false);
7640 %}
7641 ins_pipe(pipe_slow); // XXX
7642 %}
7643
7644 // ============================================================================
7645 // BSWAP Instructions
7646
7647 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7648 match(Set dst (ReverseBytesI src));
7649
7650 ins_cost(INSN_COST);
7651 format %{ "revw $dst, $src" %}
7652
7653 ins_encode %{
7654 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7655 %}
7656
7657 ins_pipe(ialu_reg);
7658 %}
7659
7660 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7661 match(Set dst (ReverseBytesL src));
7662
7663 ins_cost(INSN_COST);
7664 format %{ "rev $dst, $src" %}
7665
7666 ins_encode %{
7667 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7668 %}
7669
7670 ins_pipe(ialu_reg);
7671 %}
7672
7673 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7674 match(Set dst (ReverseBytesUS src));
7675
7676 ins_cost(INSN_COST);
7677 format %{ "rev16w $dst, $src" %}
7678
7679 ins_encode %{
7680 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7681 %}
7682
7683 ins_pipe(ialu_reg);
7684 %}
7685
7686 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7687 match(Set dst (ReverseBytesS src));
7688
7689 ins_cost(INSN_COST);
7690 format %{ "rev16w $dst, $src\n\t"
7691 "sbfmw $dst, $dst, #0, #15" %}
7692
7693 ins_encode %{
7694 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7695 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7696 %}
7697
7698 ins_pipe(ialu_reg);
7699 %}
7700
7701 // ============================================================================
7702 // Zero Count Instructions
7703
7704 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7705 match(Set dst (CountLeadingZerosI src));
7706
7707 ins_cost(INSN_COST);
7708 format %{ "clzw $dst, $src" %}
7709 ins_encode %{
7710 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7711 %}
7712
7713 ins_pipe(ialu_reg);
7714 %}
7715
7716 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7717 match(Set dst (CountLeadingZerosL src));
7718
7719 ins_cost(INSN_COST);
7720 format %{ "clz $dst, $src" %}
7721 ins_encode %{
7722 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7723 %}
7724
7725 ins_pipe(ialu_reg);
7726 %}
7727
7728 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7729 match(Set dst (CountTrailingZerosI src));
7730
7731 ins_cost(INSN_COST * 2);
7732 format %{ "rbitw $dst, $src\n\t"
7733 "clzw $dst, $dst" %}
7734 ins_encode %{
7735 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7736 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7737 %}
7738
7739 ins_pipe(ialu_reg);
7740 %}
7741
7742 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7743 match(Set dst (CountTrailingZerosL src));
7744
7745 ins_cost(INSN_COST * 2);
7746 format %{ "rbit $dst, $src\n\t"
7747 "clz $dst, $dst" %}
7748 ins_encode %{
7749 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7750 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7751 %}
7752
7753 ins_pipe(ialu_reg);
7754 %}
7755
7756 //---------- Population Count Instructions -------------------------------------
7757 //
7758
7759 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7760 match(Set dst (PopCountI src));
7761 effect(TEMP tmp);
7762 ins_cost(INSN_COST * 13);
7763
7764 format %{ "movw $src, $src\n\t"
7765 "mov $tmp, $src\t# vector (1D)\n\t"
7766 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7767 "addv $tmp, $tmp\t# vector (8B)\n\t"
7768 "mov $dst, $tmp\t# vector (1D)" %}
7769 ins_encode %{
7770 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7771 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7772 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7773 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7774 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7775 %}
7776
7777 ins_pipe(pipe_class_default);
7778 %}
7779
7780 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7781 match(Set dst (PopCountI (LoadI mem)));
7782 effect(TEMP tmp);
7783 ins_cost(INSN_COST * 13);
7784
7785 format %{ "ldrs $tmp, $mem\n\t"
7786 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7787 "addv $tmp, $tmp\t# vector (8B)\n\t"
7788 "mov $dst, $tmp\t# vector (1D)" %}
7789 ins_encode %{
7790 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7791 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7792 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7793 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7795 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7796 %}
7797
7798 ins_pipe(pipe_class_default);
7799 %}
7800
7801 // Note: Long.bitCount(long) returns an int.
7802 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7803 match(Set dst (PopCountL src));
7804 effect(TEMP tmp);
7805 ins_cost(INSN_COST * 13);
7806
7807 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7808 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7809 "addv $tmp, $tmp\t# vector (8B)\n\t"
7810 "mov $dst, $tmp\t# vector (1D)" %}
7811 ins_encode %{
7812 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7813 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7814 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7816 %}
7817
7818 ins_pipe(pipe_class_default);
7819 %}
7820
7821 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7822 match(Set dst (PopCountL (LoadL mem)));
7823 effect(TEMP tmp);
7824 ins_cost(INSN_COST * 13);
7825
7826 format %{ "ldrd $tmp, $mem\n\t"
7827 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7828 "addv $tmp, $tmp\t# vector (8B)\n\t"
7829 "mov $dst, $tmp\t# vector (1D)" %}
7830 ins_encode %{
7831 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7832 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7833 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7834 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7835 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7836 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7837 %}
7838
7839 ins_pipe(pipe_class_default);
7840 %}
7841
7842 // ============================================================================
7843 // VerifyVectorAlignment Instruction
7844
7845 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7846 match(Set addr (VerifyVectorAlignment addr mask));
7847 effect(KILL cr);
7848 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7849 ins_encode %{
7850 Label Lskip;
7851 // check if masked bits of addr are zero
7852 __ tst($addr$$Register, $mask$$constant);
7853 __ br(Assembler::EQ, Lskip);
7854 __ stop("verify_vector_alignment found a misaligned vector memory access");
7855 __ bind(Lskip);
7856 %}
7857 ins_pipe(pipe_slow);
7858 %}
7859
7860 // ============================================================================
7861 // MemBar Instruction
7862
7863 instruct load_fence() %{
7864 match(LoadFence);
7865 ins_cost(VOLATILE_REF_COST);
7866
7867 format %{ "load_fence" %}
7868
7869 ins_encode %{
7870 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7871 %}
7872 ins_pipe(pipe_serial);
7873 %}
7874
7875 instruct unnecessary_membar_acquire() %{
7876 predicate(unnecessary_acquire(n));
7877 match(MemBarAcquire);
7878 ins_cost(0);
7879
7880 format %{ "membar_acquire (elided)" %}
7881
7882 ins_encode %{
7883 __ block_comment("membar_acquire (elided)");
7884 %}
7885
7886 ins_pipe(pipe_class_empty);
7887 %}
7888
7889 instruct membar_acquire() %{
7890 match(MemBarAcquire);
7891 ins_cost(VOLATILE_REF_COST);
7892
7893 format %{ "membar_acquire\n\t"
7894 "dmb ishld" %}
7895
7896 ins_encode %{
7897 __ block_comment("membar_acquire");
7898 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7899 %}
7900
7901 ins_pipe(pipe_serial);
7902 %}
7903
7904
7905 instruct membar_acquire_lock() %{
7906 match(MemBarAcquireLock);
7907 ins_cost(VOLATILE_REF_COST);
7908
7909 format %{ "membar_acquire_lock (elided)" %}
7910
7911 ins_encode %{
7912 __ block_comment("membar_acquire_lock (elided)");
7913 %}
7914
7915 ins_pipe(pipe_serial);
7916 %}
7917
7918 instruct store_fence() %{
7919 match(StoreFence);
7920 ins_cost(VOLATILE_REF_COST);
7921
7922 format %{ "store_fence" %}
7923
7924 ins_encode %{
7925 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7926 %}
7927 ins_pipe(pipe_serial);
7928 %}
7929
7930 instruct unnecessary_membar_release() %{
7931 predicate(unnecessary_release(n));
7932 match(MemBarRelease);
7933 ins_cost(0);
7934
7935 format %{ "membar_release (elided)" %}
7936
7937 ins_encode %{
7938 __ block_comment("membar_release (elided)");
7939 %}
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 instruct membar_release() %{
7944 match(MemBarRelease);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "membar_release\n\t"
7948 "dmb ishst\n\tdmb ishld" %}
7949
7950 ins_encode %{
7951 __ block_comment("membar_release");
7952 // These will be merged if AlwaysMergeDMB is enabled.
7953 __ membar(Assembler::StoreStore);
7954 __ membar(Assembler::LoadStore);
7955 %}
7956 ins_pipe(pipe_serial);
7957 %}
7958
7959 instruct membar_storestore() %{
7960 match(MemBarStoreStore);
7961 match(StoreStoreFence);
7962 ins_cost(VOLATILE_REF_COST);
7963
7964 format %{ "MEMBAR-store-store" %}
7965
7966 ins_encode %{
7967 __ membar(Assembler::StoreStore);
7968 %}
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 instruct membar_release_lock() %{
7973 match(MemBarReleaseLock);
7974 ins_cost(VOLATILE_REF_COST);
7975
7976 format %{ "membar_release_lock (elided)" %}
7977
7978 ins_encode %{
7979 __ block_comment("membar_release_lock (elided)");
7980 %}
7981
7982 ins_pipe(pipe_serial);
7983 %}
7984
7985 instruct unnecessary_membar_volatile() %{
7986 predicate(unnecessary_volatile(n));
7987 match(MemBarVolatile);
7988 ins_cost(0);
7989
7990 format %{ "membar_volatile (elided)" %}
7991
7992 ins_encode %{
7993 __ block_comment("membar_volatile (elided)");
7994 %}
7995
7996 ins_pipe(pipe_serial);
7997 %}
7998
7999 instruct membar_volatile() %{
8000 match(MemBarVolatile);
8001 ins_cost(VOLATILE_REF_COST*100);
8002
8003 format %{ "membar_volatile\n\t"
8004 "dmb ish"%}
8005
8006 ins_encode %{
8007 __ block_comment("membar_volatile");
8008 __ membar(Assembler::StoreLoad);
8009 %}
8010
8011 ins_pipe(pipe_serial);
8012 %}
8013
8014 // ============================================================================
8015 // Cast/Convert Instructions
8016
8017 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8018 match(Set dst (CastX2P src));
8019
8020 ins_cost(INSN_COST);
8021 format %{ "mov $dst, $src\t# long -> ptr" %}
8022
8023 ins_encode %{
8024 if ($dst$$reg != $src$$reg) {
8025 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8026 }
8027 %}
8028
8029 ins_pipe(ialu_reg);
8030 %}
8031
8032 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8033 match(Set dst (CastP2X src));
8034
8035 ins_cost(INSN_COST);
8036 format %{ "mov $dst, $src\t# ptr -> long" %}
8037
8038 ins_encode %{
8039 if ($dst$$reg != $src$$reg) {
8040 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8041 }
8042 %}
8043
8044 ins_pipe(ialu_reg);
8045 %}
8046
8047 // Convert oop into int for vectors alignment masking
8048 instruct convP2I(iRegINoSp dst, iRegP src) %{
8049 match(Set dst (ConvL2I (CastP2X src)));
8050
8051 ins_cost(INSN_COST);
8052 format %{ "movw $dst, $src\t# ptr -> int" %}
8053 ins_encode %{
8054 __ movw($dst$$Register, $src$$Register);
8055 %}
8056
8057 ins_pipe(ialu_reg);
8058 %}
8059
8060 // Convert compressed oop into int for vectors alignment masking
8061 // in case of 32bit oops (heap < 4Gb).
8062 instruct convN2I(iRegINoSp dst, iRegN src)
8063 %{
8064 predicate(CompressedOops::shift() == 0);
8065 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8066
8067 ins_cost(INSN_COST);
8068 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8069 ins_encode %{
8070 __ movw($dst$$Register, $src$$Register);
8071 %}
8072
8073 ins_pipe(ialu_reg);
8074 %}
8075
8076
8077 // Convert oop pointer into compressed form
8078 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8079 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8080 match(Set dst (EncodeP src));
8081 effect(KILL cr);
8082 ins_cost(INSN_COST * 3);
8083 format %{ "encode_heap_oop $dst, $src" %}
8084 ins_encode %{
8085 Register s = $src$$Register;
8086 Register d = $dst$$Register;
8087 __ encode_heap_oop(d, s);
8088 %}
8089 ins_pipe(ialu_reg);
8090 %}
8091
8092 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8093 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8094 match(Set dst (EncodeP src));
8095 ins_cost(INSN_COST * 3);
8096 format %{ "encode_heap_oop_not_null $dst, $src" %}
8097 ins_encode %{
8098 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8099 %}
8100 ins_pipe(ialu_reg);
8101 %}
8102
8103 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8104 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8105 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8106 match(Set dst (DecodeN src));
8107 ins_cost(INSN_COST * 3);
8108 format %{ "decode_heap_oop $dst, $src" %}
8109 ins_encode %{
8110 Register s = $src$$Register;
8111 Register d = $dst$$Register;
8112 __ decode_heap_oop(d, s);
8113 %}
8114 ins_pipe(ialu_reg);
8115 %}
8116
8117 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8118 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8119 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8120 match(Set dst (DecodeN src));
8121 ins_cost(INSN_COST * 3);
8122 format %{ "decode_heap_oop_not_null $dst, $src" %}
8123 ins_encode %{
8124 Register s = $src$$Register;
8125 Register d = $dst$$Register;
8126 __ decode_heap_oop_not_null(d, s);
8127 %}
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131 // n.b. AArch64 implementations of encode_klass_not_null and
8132 // decode_klass_not_null do not modify the flags register so, unlike
8133 // Intel, we don't kill CR as a side effect here
8134
8135 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8136 match(Set dst (EncodePKlass src));
8137
8138 ins_cost(INSN_COST * 3);
8139 format %{ "encode_klass_not_null $dst,$src" %}
8140
8141 ins_encode %{
8142 Register src_reg = as_Register($src$$reg);
8143 Register dst_reg = as_Register($dst$$reg);
8144 __ encode_klass_not_null(dst_reg, src_reg);
8145 %}
8146
8147 ins_pipe(ialu_reg);
8148 %}
8149
8150 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8151 match(Set dst (DecodeNKlass src));
8152
8153 ins_cost(INSN_COST * 3);
8154 format %{ "decode_klass_not_null $dst,$src" %}
8155
8156 ins_encode %{
8157 Register src_reg = as_Register($src$$reg);
8158 Register dst_reg = as_Register($dst$$reg);
8159 if (dst_reg != src_reg) {
8160 __ decode_klass_not_null(dst_reg, src_reg);
8161 } else {
8162 __ decode_klass_not_null(dst_reg);
8163 }
8164 %}
8165
8166 ins_pipe(ialu_reg);
8167 %}
8168
8169 instruct checkCastPP(iRegPNoSp dst)
8170 %{
8171 match(Set dst (CheckCastPP dst));
8172
8173 size(0);
8174 format %{ "# checkcastPP of $dst" %}
8175 ins_encode(/* empty encoding */);
8176 ins_pipe(pipe_class_empty);
8177 %}
8178
8179 instruct castPP(iRegPNoSp dst)
8180 %{
8181 match(Set dst (CastPP dst));
8182
8183 size(0);
8184 format %{ "# castPP of $dst" %}
8185 ins_encode(/* empty encoding */);
8186 ins_pipe(pipe_class_empty);
8187 %}
8188
8189 instruct castII(iRegI dst)
8190 %{
8191 predicate(VerifyConstraintCasts == 0);
8192 match(Set dst (CastII dst));
8193
8194 size(0);
8195 format %{ "# castII of $dst" %}
8196 ins_encode(/* empty encoding */);
8197 ins_cost(0);
8198 ins_pipe(pipe_class_empty);
8199 %}
8200
8201 instruct castII_checked(iRegI dst, rFlagsReg cr)
8202 %{
8203 predicate(VerifyConstraintCasts > 0);
8204 match(Set dst (CastII dst));
8205 effect(KILL cr);
8206
8207 format %{ "# castII_checked of $dst" %}
8208 ins_encode %{
8209 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8210 %}
8211 ins_pipe(pipe_slow);
8212 %}
8213
8214 instruct castLL(iRegL dst)
8215 %{
8216 predicate(VerifyConstraintCasts == 0);
8217 match(Set dst (CastLL dst));
8218
8219 size(0);
8220 format %{ "# castLL of $dst" %}
8221 ins_encode(/* empty encoding */);
8222 ins_cost(0);
8223 ins_pipe(pipe_class_empty);
8224 %}
8225
8226 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8227 %{
8228 predicate(VerifyConstraintCasts > 0);
8229 match(Set dst (CastLL dst));
8230 effect(KILL cr);
8231
8232 format %{ "# castLL_checked of $dst" %}
8233 ins_encode %{
8234 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8235 %}
8236 ins_pipe(pipe_slow);
8237 %}
8238
8239 instruct castHH(vRegF dst)
8240 %{
8241 match(Set dst (CastHH dst));
8242 size(0);
8243 format %{ "# castHH of $dst" %}
8244 ins_encode(/* empty encoding */);
8245 ins_cost(0);
8246 ins_pipe(pipe_class_empty);
8247 %}
8248
8249 instruct castFF(vRegF dst)
8250 %{
8251 match(Set dst (CastFF dst));
8252
8253 size(0);
8254 format %{ "# castFF of $dst" %}
8255 ins_encode(/* empty encoding */);
8256 ins_cost(0);
8257 ins_pipe(pipe_class_empty);
8258 %}
8259
8260 instruct castDD(vRegD dst)
8261 %{
8262 match(Set dst (CastDD dst));
8263
8264 size(0);
8265 format %{ "# castDD of $dst" %}
8266 ins_encode(/* empty encoding */);
8267 ins_cost(0);
8268 ins_pipe(pipe_class_empty);
8269 %}
8270
8271 instruct castVV(vReg dst)
8272 %{
8273 match(Set dst (CastVV dst));
8274
8275 size(0);
8276 format %{ "# castVV of $dst" %}
8277 ins_encode(/* empty encoding */);
8278 ins_cost(0);
8279 ins_pipe(pipe_class_empty);
8280 %}
8281
8282 instruct castVVMask(pRegGov dst)
8283 %{
8284 match(Set dst (CastVV dst));
8285
8286 size(0);
8287 format %{ "# castVV of $dst" %}
8288 ins_encode(/* empty encoding */);
8289 ins_cost(0);
8290 ins_pipe(pipe_class_empty);
8291 %}
8292
8293 // ============================================================================
8294 // Atomic operation instructions
8295 //
8296
8297 // standard CompareAndSwapX when we are using barriers
8298 // these have higher priority than the rules selected by a predicate
8299
8300 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8301 // can't match them
8302
8303 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8304
8305 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8306 ins_cost(2 * VOLATILE_REF_COST);
8307
8308 effect(KILL cr);
8309
8310 format %{
8311 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8312 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8313 %}
8314
8315 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8316 aarch64_enc_cset_eq(res));
8317
8318 ins_pipe(pipe_slow);
8319 %}
8320
8321 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8322
8323 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8324 ins_cost(2 * VOLATILE_REF_COST);
8325
8326 effect(KILL cr);
8327
8328 format %{
8329 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8330 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8331 %}
8332
8333 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8334 aarch64_enc_cset_eq(res));
8335
8336 ins_pipe(pipe_slow);
8337 %}
8338
8339 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8340
8341 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8342 ins_cost(2 * VOLATILE_REF_COST);
8343
8344 effect(KILL cr);
8345
8346 format %{
8347 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8348 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8349 %}
8350
8351 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8352 aarch64_enc_cset_eq(res));
8353
8354 ins_pipe(pipe_slow);
8355 %}
8356
8357 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8358
8359 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8360 ins_cost(2 * VOLATILE_REF_COST);
8361
8362 effect(KILL cr);
8363
8364 format %{
8365 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8366 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8367 %}
8368
8369 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8370 aarch64_enc_cset_eq(res));
8371
8372 ins_pipe(pipe_slow);
8373 %}
8374
8375 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8376
8377 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8378 predicate(n->as_LoadStore()->barrier_data() == 0);
8379 ins_cost(2 * VOLATILE_REF_COST);
8380
8381 effect(KILL cr);
8382
8383 format %{
8384 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8385 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8386 %}
8387
8388 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8389 aarch64_enc_cset_eq(res));
8390
8391 ins_pipe(pipe_slow);
8392 %}
8393
8394 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8395
8396 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8397 predicate(n->as_LoadStore()->barrier_data() == 0);
8398 ins_cost(2 * VOLATILE_REF_COST);
8399
8400 effect(KILL cr);
8401
8402 format %{
8403 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8404 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8405 %}
8406
8407 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8408 aarch64_enc_cset_eq(res));
8409
8410 ins_pipe(pipe_slow);
8411 %}
8412
8413 // alternative CompareAndSwapX when we are eliding barriers
8414
8415 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8416
8417 predicate(needs_acquiring_load_exclusive(n));
8418 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8419 ins_cost(VOLATILE_REF_COST);
8420
8421 effect(KILL cr);
8422
8423 format %{
8424 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8425 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8426 %}
8427
8428 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8429 aarch64_enc_cset_eq(res));
8430
8431 ins_pipe(pipe_slow);
8432 %}
8433
8434 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8435
8436 predicate(needs_acquiring_load_exclusive(n));
8437 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8438 ins_cost(VOLATILE_REF_COST);
8439
8440 effect(KILL cr);
8441
8442 format %{
8443 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8444 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8445 %}
8446
8447 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8448 aarch64_enc_cset_eq(res));
8449
8450 ins_pipe(pipe_slow);
8451 %}
8452
8453 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8454
8455 predicate(needs_acquiring_load_exclusive(n));
8456 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8457 ins_cost(VOLATILE_REF_COST);
8458
8459 effect(KILL cr);
8460
8461 format %{
8462 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8463 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8464 %}
8465
8466 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8467 aarch64_enc_cset_eq(res));
8468
8469 ins_pipe(pipe_slow);
8470 %}
8471
8472 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8473
8474 predicate(needs_acquiring_load_exclusive(n));
8475 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8476 ins_cost(VOLATILE_REF_COST);
8477
8478 effect(KILL cr);
8479
8480 format %{
8481 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8482 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8483 %}
8484
8485 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8486 aarch64_enc_cset_eq(res));
8487
8488 ins_pipe(pipe_slow);
8489 %}
8490
8491 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8492
8493 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8494 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8495 ins_cost(VOLATILE_REF_COST);
8496
8497 effect(KILL cr);
8498
8499 format %{
8500 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8501 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8502 %}
8503
8504 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8505 aarch64_enc_cset_eq(res));
8506
8507 ins_pipe(pipe_slow);
8508 %}
8509
8510 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8511
8512 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8513 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8514 ins_cost(VOLATILE_REF_COST);
8515
8516 effect(KILL cr);
8517
8518 format %{
8519 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8520 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8521 %}
8522
8523 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8524 aarch64_enc_cset_eq(res));
8525
8526 ins_pipe(pipe_slow);
8527 %}
8528
8529
8530 // ---------------------------------------------------------------------
8531
8532 // BEGIN This section of the file is automatically generated. Do not edit --------------
8533
8534 // Sundry CAS operations. Note that release is always true,
8535 // regardless of the memory ordering of the CAS. This is because we
8536 // need the volatile case to be sequentially consistent but there is
8537 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8538 // can't check the type of memory ordering here, so we always emit a
8539 // STLXR.
8540
8541 // This section is generated from cas.m4
8542
8543
8544 // This pattern is generated automatically from cas.m4.
8545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8546 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8547 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8548 ins_cost(2 * VOLATILE_REF_COST);
8549 effect(TEMP_DEF res, KILL cr);
8550 format %{
8551 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8552 %}
8553 ins_encode %{
8554 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8555 Assembler::byte, /*acquire*/ false, /*release*/ true,
8556 /*weak*/ false, $res$$Register);
8557 __ sxtbw($res$$Register, $res$$Register);
8558 %}
8559 ins_pipe(pipe_slow);
8560 %}
8561
8562 // This pattern is generated automatically from cas.m4.
8563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8564 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8565 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8566 ins_cost(2 * VOLATILE_REF_COST);
8567 effect(TEMP_DEF res, KILL cr);
8568 format %{
8569 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8570 %}
8571 ins_encode %{
8572 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8573 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8574 /*weak*/ false, $res$$Register);
8575 __ sxthw($res$$Register, $res$$Register);
8576 %}
8577 ins_pipe(pipe_slow);
8578 %}
8579
8580 // This pattern is generated automatically from cas.m4.
8581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8582 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8583 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8584 ins_cost(2 * VOLATILE_REF_COST);
8585 effect(TEMP_DEF res, KILL cr);
8586 format %{
8587 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8588 %}
8589 ins_encode %{
8590 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8591 Assembler::word, /*acquire*/ false, /*release*/ true,
8592 /*weak*/ false, $res$$Register);
8593 %}
8594 ins_pipe(pipe_slow);
8595 %}
8596
8597 // This pattern is generated automatically from cas.m4.
8598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8599 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8600 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8601 ins_cost(2 * VOLATILE_REF_COST);
8602 effect(TEMP_DEF res, KILL cr);
8603 format %{
8604 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8605 %}
8606 ins_encode %{
8607 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8608 Assembler::xword, /*acquire*/ false, /*release*/ true,
8609 /*weak*/ false, $res$$Register);
8610 %}
8611 ins_pipe(pipe_slow);
8612 %}
8613
8614 // This pattern is generated automatically from cas.m4.
8615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8616 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8617 predicate(n->as_LoadStore()->barrier_data() == 0);
8618 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8619 ins_cost(2 * VOLATILE_REF_COST);
8620 effect(TEMP_DEF res, KILL cr);
8621 format %{
8622 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8623 %}
8624 ins_encode %{
8625 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8626 Assembler::word, /*acquire*/ false, /*release*/ true,
8627 /*weak*/ false, $res$$Register);
8628 %}
8629 ins_pipe(pipe_slow);
8630 %}
8631
8632 // This pattern is generated automatically from cas.m4.
8633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8634 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8635 predicate(n->as_LoadStore()->barrier_data() == 0);
8636 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8637 ins_cost(2 * VOLATILE_REF_COST);
8638 effect(TEMP_DEF res, KILL cr);
8639 format %{
8640 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8641 %}
8642 ins_encode %{
8643 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8644 Assembler::xword, /*acquire*/ false, /*release*/ true,
8645 /*weak*/ false, $res$$Register);
8646 %}
8647 ins_pipe(pipe_slow);
8648 %}
8649
8650 // This pattern is generated automatically from cas.m4.
8651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8652 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8653 predicate(needs_acquiring_load_exclusive(n));
8654 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8655 ins_cost(VOLATILE_REF_COST);
8656 effect(TEMP_DEF res, KILL cr);
8657 format %{
8658 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8659 %}
8660 ins_encode %{
8661 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8662 Assembler::byte, /*acquire*/ true, /*release*/ true,
8663 /*weak*/ false, $res$$Register);
8664 __ sxtbw($res$$Register, $res$$Register);
8665 %}
8666 ins_pipe(pipe_slow);
8667 %}
8668
8669 // This pattern is generated automatically from cas.m4.
8670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8671 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8672 predicate(needs_acquiring_load_exclusive(n));
8673 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8674 ins_cost(VOLATILE_REF_COST);
8675 effect(TEMP_DEF res, KILL cr);
8676 format %{
8677 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8678 %}
8679 ins_encode %{
8680 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8681 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8682 /*weak*/ false, $res$$Register);
8683 __ sxthw($res$$Register, $res$$Register);
8684 %}
8685 ins_pipe(pipe_slow);
8686 %}
8687
8688 // This pattern is generated automatically from cas.m4.
8689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8690 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8691 predicate(needs_acquiring_load_exclusive(n));
8692 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8693 ins_cost(VOLATILE_REF_COST);
8694 effect(TEMP_DEF res, KILL cr);
8695 format %{
8696 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8697 %}
8698 ins_encode %{
8699 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8700 Assembler::word, /*acquire*/ true, /*release*/ true,
8701 /*weak*/ false, $res$$Register);
8702 %}
8703 ins_pipe(pipe_slow);
8704 %}
8705
8706 // This pattern is generated automatically from cas.m4.
8707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8708 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8709 predicate(needs_acquiring_load_exclusive(n));
8710 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8711 ins_cost(VOLATILE_REF_COST);
8712 effect(TEMP_DEF res, KILL cr);
8713 format %{
8714 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8715 %}
8716 ins_encode %{
8717 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8718 Assembler::xword, /*acquire*/ true, /*release*/ true,
8719 /*weak*/ false, $res$$Register);
8720 %}
8721 ins_pipe(pipe_slow);
8722 %}
8723
8724 // This pattern is generated automatically from cas.m4.
8725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8726 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8727 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8728 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8729 ins_cost(VOLATILE_REF_COST);
8730 effect(TEMP_DEF res, KILL cr);
8731 format %{
8732 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8733 %}
8734 ins_encode %{
8735 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8736 Assembler::word, /*acquire*/ true, /*release*/ true,
8737 /*weak*/ false, $res$$Register);
8738 %}
8739 ins_pipe(pipe_slow);
8740 %}
8741
8742 // This pattern is generated automatically from cas.m4.
8743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8744 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8745 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8746 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8747 ins_cost(VOLATILE_REF_COST);
8748 effect(TEMP_DEF res, KILL cr);
8749 format %{
8750 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8751 %}
8752 ins_encode %{
8753 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8754 Assembler::xword, /*acquire*/ true, /*release*/ true,
8755 /*weak*/ false, $res$$Register);
8756 %}
8757 ins_pipe(pipe_slow);
8758 %}
8759
8760 // This pattern is generated automatically from cas.m4.
8761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8762 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8763 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8764 ins_cost(2 * VOLATILE_REF_COST);
8765 effect(KILL cr);
8766 format %{
8767 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8768 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8769 %}
8770 ins_encode %{
8771 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8772 Assembler::byte, /*acquire*/ false, /*release*/ true,
8773 /*weak*/ true, noreg);
8774 __ csetw($res$$Register, Assembler::EQ);
8775 %}
8776 ins_pipe(pipe_slow);
8777 %}
8778
8779 // This pattern is generated automatically from cas.m4.
8780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8781 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8782 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8783 ins_cost(2 * VOLATILE_REF_COST);
8784 effect(KILL cr);
8785 format %{
8786 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8787 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8792 /*weak*/ true, noreg);
8793 __ csetw($res$$Register, Assembler::EQ);
8794 %}
8795 ins_pipe(pipe_slow);
8796 %}
8797
8798 // This pattern is generated automatically from cas.m4.
8799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8800 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8801 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8802 ins_cost(2 * VOLATILE_REF_COST);
8803 effect(KILL cr);
8804 format %{
8805 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8806 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8807 %}
8808 ins_encode %{
8809 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8810 Assembler::word, /*acquire*/ false, /*release*/ true,
8811 /*weak*/ true, noreg);
8812 __ csetw($res$$Register, Assembler::EQ);
8813 %}
8814 ins_pipe(pipe_slow);
8815 %}
8816
8817 // This pattern is generated automatically from cas.m4.
8818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8819 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8820 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8821 ins_cost(2 * VOLATILE_REF_COST);
8822 effect(KILL cr);
8823 format %{
8824 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8825 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8826 %}
8827 ins_encode %{
8828 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8829 Assembler::xword, /*acquire*/ false, /*release*/ true,
8830 /*weak*/ true, noreg);
8831 __ csetw($res$$Register, Assembler::EQ);
8832 %}
8833 ins_pipe(pipe_slow);
8834 %}
8835
8836 // This pattern is generated automatically from cas.m4.
8837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8838 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8839 predicate(n->as_LoadStore()->barrier_data() == 0);
8840 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8841 ins_cost(2 * VOLATILE_REF_COST);
8842 effect(KILL cr);
8843 format %{
8844 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8845 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8846 %}
8847 ins_encode %{
8848 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8849 Assembler::word, /*acquire*/ false, /*release*/ true,
8850 /*weak*/ true, noreg);
8851 __ csetw($res$$Register, Assembler::EQ);
8852 %}
8853 ins_pipe(pipe_slow);
8854 %}
8855
8856 // This pattern is generated automatically from cas.m4.
8857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8858 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8859 predicate(n->as_LoadStore()->barrier_data() == 0);
8860 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8861 ins_cost(2 * VOLATILE_REF_COST);
8862 effect(KILL cr);
8863 format %{
8864 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8865 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8866 %}
8867 ins_encode %{
8868 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8869 Assembler::xword, /*acquire*/ false, /*release*/ true,
8870 /*weak*/ true, noreg);
8871 __ csetw($res$$Register, Assembler::EQ);
8872 %}
8873 ins_pipe(pipe_slow);
8874 %}
8875
8876 // This pattern is generated automatically from cas.m4.
8877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8878 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8879 predicate(needs_acquiring_load_exclusive(n));
8880 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8881 ins_cost(VOLATILE_REF_COST);
8882 effect(KILL cr);
8883 format %{
8884 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8885 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8886 %}
8887 ins_encode %{
8888 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8889 Assembler::byte, /*acquire*/ true, /*release*/ true,
8890 /*weak*/ true, noreg);
8891 __ csetw($res$$Register, Assembler::EQ);
8892 %}
8893 ins_pipe(pipe_slow);
8894 %}
8895
8896 // This pattern is generated automatically from cas.m4.
8897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8898 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8899 predicate(needs_acquiring_load_exclusive(n));
8900 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8901 ins_cost(VOLATILE_REF_COST);
8902 effect(KILL cr);
8903 format %{
8904 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8905 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8906 %}
8907 ins_encode %{
8908 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8909 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8910 /*weak*/ true, noreg);
8911 __ csetw($res$$Register, Assembler::EQ);
8912 %}
8913 ins_pipe(pipe_slow);
8914 %}
8915
8916 // This pattern is generated automatically from cas.m4.
8917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8918 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8919 predicate(needs_acquiring_load_exclusive(n));
8920 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8921 ins_cost(VOLATILE_REF_COST);
8922 effect(KILL cr);
8923 format %{
8924 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8925 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8926 %}
8927 ins_encode %{
8928 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8929 Assembler::word, /*acquire*/ true, /*release*/ true,
8930 /*weak*/ true, noreg);
8931 __ csetw($res$$Register, Assembler::EQ);
8932 %}
8933 ins_pipe(pipe_slow);
8934 %}
8935
8936 // This pattern is generated automatically from cas.m4.
8937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8938 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8939 predicate(needs_acquiring_load_exclusive(n));
8940 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8941 ins_cost(VOLATILE_REF_COST);
8942 effect(KILL cr);
8943 format %{
8944 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8945 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8946 %}
8947 ins_encode %{
8948 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8949 Assembler::xword, /*acquire*/ true, /*release*/ true,
8950 /*weak*/ true, noreg);
8951 __ csetw($res$$Register, Assembler::EQ);
8952 %}
8953 ins_pipe(pipe_slow);
8954 %}
8955
8956 // This pattern is generated automatically from cas.m4.
8957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8958 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8959 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8960 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8961 ins_cost(VOLATILE_REF_COST);
8962 effect(KILL cr);
8963 format %{
8964 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8965 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8966 %}
8967 ins_encode %{
8968 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8969 Assembler::word, /*acquire*/ true, /*release*/ true,
8970 /*weak*/ true, noreg);
8971 __ csetw($res$$Register, Assembler::EQ);
8972 %}
8973 ins_pipe(pipe_slow);
8974 %}
8975
8976 // This pattern is generated automatically from cas.m4.
8977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8978 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8979 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8980 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8981 ins_cost(VOLATILE_REF_COST);
8982 effect(KILL cr);
8983 format %{
8984 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8985 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8986 %}
8987 ins_encode %{
8988 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8989 Assembler::xword, /*acquire*/ true, /*release*/ true,
8990 /*weak*/ true, noreg);
8991 __ csetw($res$$Register, Assembler::EQ);
8992 %}
8993 ins_pipe(pipe_slow);
8994 %}
8995
8996 // END This section of the file is automatically generated. Do not edit --------------
8997 // ---------------------------------------------------------------------
8998
8999 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9000 match(Set prev (GetAndSetI mem newv));
9001 ins_cost(2 * VOLATILE_REF_COST);
9002 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9003 ins_encode %{
9004 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9005 %}
9006 ins_pipe(pipe_serial);
9007 %}
9008
9009 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9010 match(Set prev (GetAndSetL mem newv));
9011 ins_cost(2 * VOLATILE_REF_COST);
9012 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9013 ins_encode %{
9014 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9015 %}
9016 ins_pipe(pipe_serial);
9017 %}
9018
9019 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9020 predicate(n->as_LoadStore()->barrier_data() == 0);
9021 match(Set prev (GetAndSetN mem newv));
9022 ins_cost(2 * VOLATILE_REF_COST);
9023 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9024 ins_encode %{
9025 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9026 %}
9027 ins_pipe(pipe_serial);
9028 %}
9029
9030 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9031 predicate(n->as_LoadStore()->barrier_data() == 0);
9032 match(Set prev (GetAndSetP mem newv));
9033 ins_cost(2 * VOLATILE_REF_COST);
9034 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9035 ins_encode %{
9036 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9037 %}
9038 ins_pipe(pipe_serial);
9039 %}
9040
9041 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9042 predicate(needs_acquiring_load_exclusive(n));
9043 match(Set prev (GetAndSetI mem newv));
9044 ins_cost(VOLATILE_REF_COST);
9045 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9046 ins_encode %{
9047 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9048 %}
9049 ins_pipe(pipe_serial);
9050 %}
9051
9052 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9053 predicate(needs_acquiring_load_exclusive(n));
9054 match(Set prev (GetAndSetL mem newv));
9055 ins_cost(VOLATILE_REF_COST);
9056 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9057 ins_encode %{
9058 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9059 %}
9060 ins_pipe(pipe_serial);
9061 %}
9062
9063 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9064 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9065 match(Set prev (GetAndSetN mem newv));
9066 ins_cost(VOLATILE_REF_COST);
9067 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9068 ins_encode %{
9069 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9070 %}
9071 ins_pipe(pipe_serial);
9072 %}
9073
9074 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9075 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9076 match(Set prev (GetAndSetP mem newv));
9077 ins_cost(VOLATILE_REF_COST);
9078 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9079 ins_encode %{
9080 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9081 %}
9082 ins_pipe(pipe_serial);
9083 %}
9084
9085
9086 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9087 match(Set newval (GetAndAddL mem incr));
9088 ins_cost(2 * VOLATILE_REF_COST + 1);
9089 format %{ "get_and_addL $newval, [$mem], $incr" %}
9090 ins_encode %{
9091 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9092 %}
9093 ins_pipe(pipe_serial);
9094 %}
9095
9096 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9097 predicate(n->as_LoadStore()->result_not_used());
9098 match(Set dummy (GetAndAddL mem incr));
9099 ins_cost(2 * VOLATILE_REF_COST);
9100 format %{ "get_and_addL [$mem], $incr" %}
9101 ins_encode %{
9102 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9103 %}
9104 ins_pipe(pipe_serial);
9105 %}
9106
9107 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9108 match(Set newval (GetAndAddL mem incr));
9109 ins_cost(2 * VOLATILE_REF_COST + 1);
9110 format %{ "get_and_addL $newval, [$mem], $incr" %}
9111 ins_encode %{
9112 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9113 %}
9114 ins_pipe(pipe_serial);
9115 %}
9116
9117 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9118 predicate(n->as_LoadStore()->result_not_used());
9119 match(Set dummy (GetAndAddL mem incr));
9120 ins_cost(2 * VOLATILE_REF_COST);
9121 format %{ "get_and_addL [$mem], $incr" %}
9122 ins_encode %{
9123 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9124 %}
9125 ins_pipe(pipe_serial);
9126 %}
9127
9128 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9129 match(Set newval (GetAndAddI mem incr));
9130 ins_cost(2 * VOLATILE_REF_COST + 1);
9131 format %{ "get_and_addI $newval, [$mem], $incr" %}
9132 ins_encode %{
9133 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9134 %}
9135 ins_pipe(pipe_serial);
9136 %}
9137
9138 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9139 predicate(n->as_LoadStore()->result_not_used());
9140 match(Set dummy (GetAndAddI mem incr));
9141 ins_cost(2 * VOLATILE_REF_COST);
9142 format %{ "get_and_addI [$mem], $incr" %}
9143 ins_encode %{
9144 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9145 %}
9146 ins_pipe(pipe_serial);
9147 %}
9148
9149 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9150 match(Set newval (GetAndAddI mem incr));
9151 ins_cost(2 * VOLATILE_REF_COST + 1);
9152 format %{ "get_and_addI $newval, [$mem], $incr" %}
9153 ins_encode %{
9154 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9155 %}
9156 ins_pipe(pipe_serial);
9157 %}
9158
9159 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9160 predicate(n->as_LoadStore()->result_not_used());
9161 match(Set dummy (GetAndAddI mem incr));
9162 ins_cost(2 * VOLATILE_REF_COST);
9163 format %{ "get_and_addI [$mem], $incr" %}
9164 ins_encode %{
9165 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9166 %}
9167 ins_pipe(pipe_serial);
9168 %}
9169
9170 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9171 predicate(needs_acquiring_load_exclusive(n));
9172 match(Set newval (GetAndAddL mem incr));
9173 ins_cost(VOLATILE_REF_COST + 1);
9174 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9175 ins_encode %{
9176 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9177 %}
9178 ins_pipe(pipe_serial);
9179 %}
9180
9181 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9182 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9183 match(Set dummy (GetAndAddL mem incr));
9184 ins_cost(VOLATILE_REF_COST);
9185 format %{ "get_and_addL_acq [$mem], $incr" %}
9186 ins_encode %{
9187 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9188 %}
9189 ins_pipe(pipe_serial);
9190 %}
9191
9192 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9193 predicate(needs_acquiring_load_exclusive(n));
9194 match(Set newval (GetAndAddL mem incr));
9195 ins_cost(VOLATILE_REF_COST + 1);
9196 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9197 ins_encode %{
9198 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9199 %}
9200 ins_pipe(pipe_serial);
9201 %}
9202
9203 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9204 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9205 match(Set dummy (GetAndAddL mem incr));
9206 ins_cost(VOLATILE_REF_COST);
9207 format %{ "get_and_addL_acq [$mem], $incr" %}
9208 ins_encode %{
9209 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9210 %}
9211 ins_pipe(pipe_serial);
9212 %}
9213
9214 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9215 predicate(needs_acquiring_load_exclusive(n));
9216 match(Set newval (GetAndAddI mem incr));
9217 ins_cost(VOLATILE_REF_COST + 1);
9218 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9219 ins_encode %{
9220 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9221 %}
9222 ins_pipe(pipe_serial);
9223 %}
9224
9225 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9226 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9227 match(Set dummy (GetAndAddI mem incr));
9228 ins_cost(VOLATILE_REF_COST);
9229 format %{ "get_and_addI_acq [$mem], $incr" %}
9230 ins_encode %{
9231 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9232 %}
9233 ins_pipe(pipe_serial);
9234 %}
9235
9236 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9237 predicate(needs_acquiring_load_exclusive(n));
9238 match(Set newval (GetAndAddI mem incr));
9239 ins_cost(VOLATILE_REF_COST + 1);
9240 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9241 ins_encode %{
9242 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9243 %}
9244 ins_pipe(pipe_serial);
9245 %}
9246
9247 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9248 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9249 match(Set dummy (GetAndAddI mem incr));
9250 ins_cost(VOLATILE_REF_COST);
9251 format %{ "get_and_addI_acq [$mem], $incr" %}
9252 ins_encode %{
9253 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9254 %}
9255 ins_pipe(pipe_serial);
9256 %}
9257
9258 // Manifest a CmpU result in an integer register.
9259 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9260 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9261 %{
9262 match(Set dst (CmpU3 src1 src2));
9263 effect(KILL flags);
9264
9265 ins_cost(INSN_COST * 3);
9266 format %{
9267 "cmpw $src1, $src2\n\t"
9268 "csetw $dst, ne\n\t"
9269 "cnegw $dst, lo\t# CmpU3(reg)"
9270 %}
9271 ins_encode %{
9272 __ cmpw($src1$$Register, $src2$$Register);
9273 __ csetw($dst$$Register, Assembler::NE);
9274 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9275 %}
9276
9277 ins_pipe(pipe_class_default);
9278 %}
9279
9280 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9281 %{
9282 match(Set dst (CmpU3 src1 src2));
9283 effect(KILL flags);
9284
9285 ins_cost(INSN_COST * 3);
9286 format %{
9287 "subsw zr, $src1, $src2\n\t"
9288 "csetw $dst, ne\n\t"
9289 "cnegw $dst, lo\t# CmpU3(imm)"
9290 %}
9291 ins_encode %{
9292 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9293 __ csetw($dst$$Register, Assembler::NE);
9294 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9295 %}
9296
9297 ins_pipe(pipe_class_default);
9298 %}
9299
9300 // Manifest a CmpUL result in an integer register.
9301 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9302 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9303 %{
9304 match(Set dst (CmpUL3 src1 src2));
9305 effect(KILL flags);
9306
9307 ins_cost(INSN_COST * 3);
9308 format %{
9309 "cmp $src1, $src2\n\t"
9310 "csetw $dst, ne\n\t"
9311 "cnegw $dst, lo\t# CmpUL3(reg)"
9312 %}
9313 ins_encode %{
9314 __ cmp($src1$$Register, $src2$$Register);
9315 __ csetw($dst$$Register, Assembler::NE);
9316 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9317 %}
9318
9319 ins_pipe(pipe_class_default);
9320 %}
9321
9322 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9323 %{
9324 match(Set dst (CmpUL3 src1 src2));
9325 effect(KILL flags);
9326
9327 ins_cost(INSN_COST * 3);
9328 format %{
9329 "subs zr, $src1, $src2\n\t"
9330 "csetw $dst, ne\n\t"
9331 "cnegw $dst, lo\t# CmpUL3(imm)"
9332 %}
9333 ins_encode %{
9334 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9335 __ csetw($dst$$Register, Assembler::NE);
9336 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9337 %}
9338
9339 ins_pipe(pipe_class_default);
9340 %}
9341
9342 // Manifest a CmpL result in an integer register.
9343 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9344 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9345 %{
9346 match(Set dst (CmpL3 src1 src2));
9347 effect(KILL flags);
9348
9349 ins_cost(INSN_COST * 3);
9350 format %{
9351 "cmp $src1, $src2\n\t"
9352 "csetw $dst, ne\n\t"
9353 "cnegw $dst, lt\t# CmpL3(reg)"
9354 %}
9355 ins_encode %{
9356 __ cmp($src1$$Register, $src2$$Register);
9357 __ csetw($dst$$Register, Assembler::NE);
9358 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9359 %}
9360
9361 ins_pipe(pipe_class_default);
9362 %}
9363
9364 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9365 %{
9366 match(Set dst (CmpL3 src1 src2));
9367 effect(KILL flags);
9368
9369 ins_cost(INSN_COST * 3);
9370 format %{
9371 "subs zr, $src1, $src2\n\t"
9372 "csetw $dst, ne\n\t"
9373 "cnegw $dst, lt\t# CmpL3(imm)"
9374 %}
9375 ins_encode %{
9376 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9377 __ csetw($dst$$Register, Assembler::NE);
9378 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9379 %}
9380
9381 ins_pipe(pipe_class_default);
9382 %}
9383
9384 // ============================================================================
9385 // Conditional Move Instructions
9386
9387 // n.b. we have identical rules for both a signed compare op (cmpOp)
9388 // and an unsigned compare op (cmpOpU). it would be nice if we could
9389 // define an op class which merged both inputs and use it to type the
9390 // argument to a single rule. unfortunatelyt his fails because the
9391 // opclass does not live up to the COND_INTER interface of its
9392 // component operands. When the generic code tries to negate the
9393 // operand it ends up running the generci Machoper::negate method
9394 // which throws a ShouldNotHappen. So, we have to provide two flavours
9395 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9396
9397 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9398 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9399
9400 ins_cost(INSN_COST * 2);
9401 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9402
9403 ins_encode %{
9404 __ cselw(as_Register($dst$$reg),
9405 as_Register($src2$$reg),
9406 as_Register($src1$$reg),
9407 (Assembler::Condition)$cmp$$cmpcode);
9408 %}
9409
9410 ins_pipe(icond_reg_reg);
9411 %}
9412
9413 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9414 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9415
9416 ins_cost(INSN_COST * 2);
9417 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9418
9419 ins_encode %{
9420 __ cselw(as_Register($dst$$reg),
9421 as_Register($src2$$reg),
9422 as_Register($src1$$reg),
9423 (Assembler::Condition)$cmp$$cmpcode);
9424 %}
9425
9426 ins_pipe(icond_reg_reg);
9427 %}
9428
9429 // special cases where one arg is zero
9430
9431 // n.b. this is selected in preference to the rule above because it
9432 // avoids loading constant 0 into a source register
9433
9434 // TODO
9435 // we ought only to be able to cull one of these variants as the ideal
9436 // transforms ought always to order the zero consistently (to left/right?)
9437
9438 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9439 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9440
9441 ins_cost(INSN_COST * 2);
9442 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9443
9444 ins_encode %{
9445 __ cselw(as_Register($dst$$reg),
9446 as_Register($src$$reg),
9447 zr,
9448 (Assembler::Condition)$cmp$$cmpcode);
9449 %}
9450
9451 ins_pipe(icond_reg);
9452 %}
9453
9454 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9455 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9456
9457 ins_cost(INSN_COST * 2);
9458 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9459
9460 ins_encode %{
9461 __ cselw(as_Register($dst$$reg),
9462 as_Register($src$$reg),
9463 zr,
9464 (Assembler::Condition)$cmp$$cmpcode);
9465 %}
9466
9467 ins_pipe(icond_reg);
9468 %}
9469
9470 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9471 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9472
9473 ins_cost(INSN_COST * 2);
9474 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9475
9476 ins_encode %{
9477 __ cselw(as_Register($dst$$reg),
9478 zr,
9479 as_Register($src$$reg),
9480 (Assembler::Condition)$cmp$$cmpcode);
9481 %}
9482
9483 ins_pipe(icond_reg);
9484 %}
9485
9486 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9487 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9488
9489 ins_cost(INSN_COST * 2);
9490 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9491
9492 ins_encode %{
9493 __ cselw(as_Register($dst$$reg),
9494 zr,
9495 as_Register($src$$reg),
9496 (Assembler::Condition)$cmp$$cmpcode);
9497 %}
9498
9499 ins_pipe(icond_reg);
9500 %}
9501
9502 // special case for creating a boolean 0 or 1
9503
9504 // n.b. this is selected in preference to the rule above because it
9505 // avoids loading constants 0 and 1 into a source register
9506
9507 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9508 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9509
9510 ins_cost(INSN_COST * 2);
9511 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9512
9513 ins_encode %{
9514 // equivalently
9515 // cset(as_Register($dst$$reg),
9516 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9517 __ csincw(as_Register($dst$$reg),
9518 zr,
9519 zr,
9520 (Assembler::Condition)$cmp$$cmpcode);
9521 %}
9522
9523 ins_pipe(icond_none);
9524 %}
9525
9526 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9527 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9528
9529 ins_cost(INSN_COST * 2);
9530 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9531
9532 ins_encode %{
9533 // equivalently
9534 // cset(as_Register($dst$$reg),
9535 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9536 __ csincw(as_Register($dst$$reg),
9537 zr,
9538 zr,
9539 (Assembler::Condition)$cmp$$cmpcode);
9540 %}
9541
9542 ins_pipe(icond_none);
9543 %}
9544
9545 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9546 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9547
9548 ins_cost(INSN_COST * 2);
9549 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9550
9551 ins_encode %{
9552 __ csel(as_Register($dst$$reg),
9553 as_Register($src2$$reg),
9554 as_Register($src1$$reg),
9555 (Assembler::Condition)$cmp$$cmpcode);
9556 %}
9557
9558 ins_pipe(icond_reg_reg);
9559 %}
9560
9561 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9562 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9563
9564 ins_cost(INSN_COST * 2);
9565 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9566
9567 ins_encode %{
9568 __ csel(as_Register($dst$$reg),
9569 as_Register($src2$$reg),
9570 as_Register($src1$$reg),
9571 (Assembler::Condition)$cmp$$cmpcode);
9572 %}
9573
9574 ins_pipe(icond_reg_reg);
9575 %}
9576
9577 // special cases where one arg is zero
9578
9579 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9580 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9581
9582 ins_cost(INSN_COST * 2);
9583 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9584
9585 ins_encode %{
9586 __ csel(as_Register($dst$$reg),
9587 zr,
9588 as_Register($src$$reg),
9589 (Assembler::Condition)$cmp$$cmpcode);
9590 %}
9591
9592 ins_pipe(icond_reg);
9593 %}
9594
9595 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9596 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9597
9598 ins_cost(INSN_COST * 2);
9599 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9600
9601 ins_encode %{
9602 __ csel(as_Register($dst$$reg),
9603 zr,
9604 as_Register($src$$reg),
9605 (Assembler::Condition)$cmp$$cmpcode);
9606 %}
9607
9608 ins_pipe(icond_reg);
9609 %}
9610
9611 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9612 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9613
9614 ins_cost(INSN_COST * 2);
9615 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9616
9617 ins_encode %{
9618 __ csel(as_Register($dst$$reg),
9619 as_Register($src$$reg),
9620 zr,
9621 (Assembler::Condition)$cmp$$cmpcode);
9622 %}
9623
9624 ins_pipe(icond_reg);
9625 %}
9626
9627 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9628 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9629
9630 ins_cost(INSN_COST * 2);
9631 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9632
9633 ins_encode %{
9634 __ csel(as_Register($dst$$reg),
9635 as_Register($src$$reg),
9636 zr,
9637 (Assembler::Condition)$cmp$$cmpcode);
9638 %}
9639
9640 ins_pipe(icond_reg);
9641 %}
9642
9643 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9644 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9645
9646 ins_cost(INSN_COST * 2);
9647 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9648
9649 ins_encode %{
9650 __ csel(as_Register($dst$$reg),
9651 as_Register($src2$$reg),
9652 as_Register($src1$$reg),
9653 (Assembler::Condition)$cmp$$cmpcode);
9654 %}
9655
9656 ins_pipe(icond_reg_reg);
9657 %}
9658
9659 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9660 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9661
9662 ins_cost(INSN_COST * 2);
9663 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9664
9665 ins_encode %{
9666 __ csel(as_Register($dst$$reg),
9667 as_Register($src2$$reg),
9668 as_Register($src1$$reg),
9669 (Assembler::Condition)$cmp$$cmpcode);
9670 %}
9671
9672 ins_pipe(icond_reg_reg);
9673 %}
9674
9675 // special cases where one arg is zero
9676
9677 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9678 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9679
9680 ins_cost(INSN_COST * 2);
9681 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9682
9683 ins_encode %{
9684 __ csel(as_Register($dst$$reg),
9685 zr,
9686 as_Register($src$$reg),
9687 (Assembler::Condition)$cmp$$cmpcode);
9688 %}
9689
9690 ins_pipe(icond_reg);
9691 %}
9692
9693 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9694 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9695
9696 ins_cost(INSN_COST * 2);
9697 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9698
9699 ins_encode %{
9700 __ csel(as_Register($dst$$reg),
9701 zr,
9702 as_Register($src$$reg),
9703 (Assembler::Condition)$cmp$$cmpcode);
9704 %}
9705
9706 ins_pipe(icond_reg);
9707 %}
9708
9709 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9710 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9711
9712 ins_cost(INSN_COST * 2);
9713 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9714
9715 ins_encode %{
9716 __ csel(as_Register($dst$$reg),
9717 as_Register($src$$reg),
9718 zr,
9719 (Assembler::Condition)$cmp$$cmpcode);
9720 %}
9721
9722 ins_pipe(icond_reg);
9723 %}
9724
9725 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9726 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9727
9728 ins_cost(INSN_COST * 2);
9729 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9730
9731 ins_encode %{
9732 __ csel(as_Register($dst$$reg),
9733 as_Register($src$$reg),
9734 zr,
9735 (Assembler::Condition)$cmp$$cmpcode);
9736 %}
9737
9738 ins_pipe(icond_reg);
9739 %}
9740
9741 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9742 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9743
9744 ins_cost(INSN_COST * 2);
9745 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9746
9747 ins_encode %{
9748 __ cselw(as_Register($dst$$reg),
9749 as_Register($src2$$reg),
9750 as_Register($src1$$reg),
9751 (Assembler::Condition)$cmp$$cmpcode);
9752 %}
9753
9754 ins_pipe(icond_reg_reg);
9755 %}
9756
9757 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9758 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9759
9760 ins_cost(INSN_COST * 2);
9761 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9762
9763 ins_encode %{
9764 __ cselw(as_Register($dst$$reg),
9765 as_Register($src2$$reg),
9766 as_Register($src1$$reg),
9767 (Assembler::Condition)$cmp$$cmpcode);
9768 %}
9769
9770 ins_pipe(icond_reg_reg);
9771 %}
9772
9773 // special cases where one arg is zero
9774
9775 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9776 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9777
9778 ins_cost(INSN_COST * 2);
9779 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9780
9781 ins_encode %{
9782 __ cselw(as_Register($dst$$reg),
9783 zr,
9784 as_Register($src$$reg),
9785 (Assembler::Condition)$cmp$$cmpcode);
9786 %}
9787
9788 ins_pipe(icond_reg);
9789 %}
9790
9791 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9792 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9793
9794 ins_cost(INSN_COST * 2);
9795 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9796
9797 ins_encode %{
9798 __ cselw(as_Register($dst$$reg),
9799 zr,
9800 as_Register($src$$reg),
9801 (Assembler::Condition)$cmp$$cmpcode);
9802 %}
9803
9804 ins_pipe(icond_reg);
9805 %}
9806
9807 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9808 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9809
9810 ins_cost(INSN_COST * 2);
9811 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9812
9813 ins_encode %{
9814 __ cselw(as_Register($dst$$reg),
9815 as_Register($src$$reg),
9816 zr,
9817 (Assembler::Condition)$cmp$$cmpcode);
9818 %}
9819
9820 ins_pipe(icond_reg);
9821 %}
9822
9823 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9824 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9825
9826 ins_cost(INSN_COST * 2);
9827 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9828
9829 ins_encode %{
9830 __ cselw(as_Register($dst$$reg),
9831 as_Register($src$$reg),
9832 zr,
9833 (Assembler::Condition)$cmp$$cmpcode);
9834 %}
9835
9836 ins_pipe(icond_reg);
9837 %}
9838
9839 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9840 %{
9841 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9842
9843 ins_cost(INSN_COST * 3);
9844
9845 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9846 ins_encode %{
9847 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9848 __ fcsels(as_FloatRegister($dst$$reg),
9849 as_FloatRegister($src2$$reg),
9850 as_FloatRegister($src1$$reg),
9851 cond);
9852 %}
9853
9854 ins_pipe(fp_cond_reg_reg_s);
9855 %}
9856
9857 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9858 %{
9859 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9860
9861 ins_cost(INSN_COST * 3);
9862
9863 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9864 ins_encode %{
9865 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9866 __ fcsels(as_FloatRegister($dst$$reg),
9867 as_FloatRegister($src2$$reg),
9868 as_FloatRegister($src1$$reg),
9869 cond);
9870 %}
9871
9872 ins_pipe(fp_cond_reg_reg_s);
9873 %}
9874
9875 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9876 %{
9877 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9878
9879 ins_cost(INSN_COST * 3);
9880
9881 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9882 ins_encode %{
9883 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9884 __ fcseld(as_FloatRegister($dst$$reg),
9885 as_FloatRegister($src2$$reg),
9886 as_FloatRegister($src1$$reg),
9887 cond);
9888 %}
9889
9890 ins_pipe(fp_cond_reg_reg_d);
9891 %}
9892
9893 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9894 %{
9895 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9896
9897 ins_cost(INSN_COST * 3);
9898
9899 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9900 ins_encode %{
9901 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9902 __ fcseld(as_FloatRegister($dst$$reg),
9903 as_FloatRegister($src2$$reg),
9904 as_FloatRegister($src1$$reg),
9905 cond);
9906 %}
9907
9908 ins_pipe(fp_cond_reg_reg_d);
9909 %}
9910
9911 // ============================================================================
9912 // Arithmetic Instructions
9913 //
9914
9915 // Integer Addition
9916
9917 // TODO
9918 // these currently employ operations which do not set CR and hence are
9919 // not flagged as killing CR but we would like to isolate the cases
9920 // where we want to set flags from those where we don't. need to work
9921 // out how to do that.
9922
9923 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9924 match(Set dst (AddI src1 src2));
9925
9926 ins_cost(INSN_COST);
9927 format %{ "addw $dst, $src1, $src2" %}
9928
9929 ins_encode %{
9930 __ addw(as_Register($dst$$reg),
9931 as_Register($src1$$reg),
9932 as_Register($src2$$reg));
9933 %}
9934
9935 ins_pipe(ialu_reg_reg);
9936 %}
9937
9938 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9939 match(Set dst (AddI src1 src2));
9940
9941 ins_cost(INSN_COST);
9942 format %{ "addw $dst, $src1, $src2" %}
9943
9944 // use opcode to indicate that this is an add not a sub
9945 opcode(0x0);
9946
9947 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9948
9949 ins_pipe(ialu_reg_imm);
9950 %}
9951
9952 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9953 match(Set dst (AddI (ConvL2I src1) src2));
9954
9955 ins_cost(INSN_COST);
9956 format %{ "addw $dst, $src1, $src2" %}
9957
9958 // use opcode to indicate that this is an add not a sub
9959 opcode(0x0);
9960
9961 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9962
9963 ins_pipe(ialu_reg_imm);
9964 %}
9965
9966 // Pointer Addition
9967 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9968 match(Set dst (AddP src1 src2));
9969
9970 ins_cost(INSN_COST);
9971 format %{ "add $dst, $src1, $src2\t# ptr" %}
9972
9973 ins_encode %{
9974 __ add(as_Register($dst$$reg),
9975 as_Register($src1$$reg),
9976 as_Register($src2$$reg));
9977 %}
9978
9979 ins_pipe(ialu_reg_reg);
9980 %}
9981
9982 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9983 match(Set dst (AddP src1 (ConvI2L src2)));
9984
9985 ins_cost(1.9 * INSN_COST);
9986 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9987
9988 ins_encode %{
9989 __ add(as_Register($dst$$reg),
9990 as_Register($src1$$reg),
9991 as_Register($src2$$reg), ext::sxtw);
9992 %}
9993
9994 ins_pipe(ialu_reg_reg);
9995 %}
9996
9997 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9998 match(Set dst (AddP src1 (LShiftL src2 scale)));
9999
10000 ins_cost(1.9 * INSN_COST);
10001 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10002
10003 ins_encode %{
10004 __ lea(as_Register($dst$$reg),
10005 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10006 Address::lsl($scale$$constant)));
10007 %}
10008
10009 ins_pipe(ialu_reg_reg_shift);
10010 %}
10011
10012 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10013 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10014
10015 ins_cost(1.9 * INSN_COST);
10016 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10017
10018 ins_encode %{
10019 __ lea(as_Register($dst$$reg),
10020 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10021 Address::sxtw($scale$$constant)));
10022 %}
10023
10024 ins_pipe(ialu_reg_reg_shift);
10025 %}
10026
10027 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10028 match(Set dst (LShiftL (ConvI2L src) scale));
10029
10030 ins_cost(INSN_COST);
10031 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10032
10033 ins_encode %{
10034 __ sbfiz(as_Register($dst$$reg),
10035 as_Register($src$$reg),
10036 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10037 %}
10038
10039 ins_pipe(ialu_reg_shift);
10040 %}
10041
10042 // Pointer Immediate Addition
10043 // n.b. this needs to be more expensive than using an indirect memory
10044 // operand
10045 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10046 match(Set dst (AddP src1 src2));
10047
10048 ins_cost(INSN_COST);
10049 format %{ "add $dst, $src1, $src2\t# ptr" %}
10050
10051 // use opcode to indicate that this is an add not a sub
10052 opcode(0x0);
10053
10054 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10055
10056 ins_pipe(ialu_reg_imm);
10057 %}
10058
10059 // Long Addition
10060 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10061
10062 match(Set dst (AddL src1 src2));
10063
10064 ins_cost(INSN_COST);
10065 format %{ "add $dst, $src1, $src2" %}
10066
10067 ins_encode %{
10068 __ add(as_Register($dst$$reg),
10069 as_Register($src1$$reg),
10070 as_Register($src2$$reg));
10071 %}
10072
10073 ins_pipe(ialu_reg_reg);
10074 %}
10075
10076 // No constant pool entries requiredLong Immediate Addition.
10077 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10078 match(Set dst (AddL src1 src2));
10079
10080 ins_cost(INSN_COST);
10081 format %{ "add $dst, $src1, $src2" %}
10082
10083 // use opcode to indicate that this is an add not a sub
10084 opcode(0x0);
10085
10086 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10087
10088 ins_pipe(ialu_reg_imm);
10089 %}
10090
10091 // Integer Subtraction
10092 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10093 match(Set dst (SubI src1 src2));
10094
10095 ins_cost(INSN_COST);
10096 format %{ "subw $dst, $src1, $src2" %}
10097
10098 ins_encode %{
10099 __ subw(as_Register($dst$$reg),
10100 as_Register($src1$$reg),
10101 as_Register($src2$$reg));
10102 %}
10103
10104 ins_pipe(ialu_reg_reg);
10105 %}
10106
10107 // Immediate Subtraction
10108 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10109 match(Set dst (SubI src1 src2));
10110
10111 ins_cost(INSN_COST);
10112 format %{ "subw $dst, $src1, $src2" %}
10113
10114 // use opcode to indicate that this is a sub not an add
10115 opcode(0x1);
10116
10117 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10118
10119 ins_pipe(ialu_reg_imm);
10120 %}
10121
10122 // Long Subtraction
10123 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10124
10125 match(Set dst (SubL src1 src2));
10126
10127 ins_cost(INSN_COST);
10128 format %{ "sub $dst, $src1, $src2" %}
10129
10130 ins_encode %{
10131 __ sub(as_Register($dst$$reg),
10132 as_Register($src1$$reg),
10133 as_Register($src2$$reg));
10134 %}
10135
10136 ins_pipe(ialu_reg_reg);
10137 %}
10138
10139 // No constant pool entries requiredLong Immediate Subtraction.
10140 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10141 match(Set dst (SubL src1 src2));
10142
10143 ins_cost(INSN_COST);
10144 format %{ "sub$dst, $src1, $src2" %}
10145
10146 // use opcode to indicate that this is a sub not an add
10147 opcode(0x1);
10148
10149 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10150
10151 ins_pipe(ialu_reg_imm);
10152 %}
10153
10154 // Integer Negation (special case for sub)
10155
10156 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10157 match(Set dst (SubI zero src));
10158
10159 ins_cost(INSN_COST);
10160 format %{ "negw $dst, $src\t# int" %}
10161
10162 ins_encode %{
10163 __ negw(as_Register($dst$$reg),
10164 as_Register($src$$reg));
10165 %}
10166
10167 ins_pipe(ialu_reg);
10168 %}
10169
10170 // Long Negation
10171
10172 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10173 match(Set dst (SubL zero src));
10174
10175 ins_cost(INSN_COST);
10176 format %{ "neg $dst, $src\t# long" %}
10177
10178 ins_encode %{
10179 __ neg(as_Register($dst$$reg),
10180 as_Register($src$$reg));
10181 %}
10182
10183 ins_pipe(ialu_reg);
10184 %}
10185
10186 // Integer Multiply
10187
10188 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10189 match(Set dst (MulI src1 src2));
10190
10191 ins_cost(INSN_COST * 3);
10192 format %{ "mulw $dst, $src1, $src2" %}
10193
10194 ins_encode %{
10195 __ mulw(as_Register($dst$$reg),
10196 as_Register($src1$$reg),
10197 as_Register($src2$$reg));
10198 %}
10199
10200 ins_pipe(imul_reg_reg);
10201 %}
10202
10203 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10204 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10205
10206 ins_cost(INSN_COST * 3);
10207 format %{ "smull $dst, $src1, $src2" %}
10208
10209 ins_encode %{
10210 __ smull(as_Register($dst$$reg),
10211 as_Register($src1$$reg),
10212 as_Register($src2$$reg));
10213 %}
10214
10215 ins_pipe(imul_reg_reg);
10216 %}
10217
10218 // Long Multiply
10219
10220 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10221 match(Set dst (MulL src1 src2));
10222
10223 ins_cost(INSN_COST * 5);
10224 format %{ "mul $dst, $src1, $src2" %}
10225
10226 ins_encode %{
10227 __ mul(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg));
10230 %}
10231
10232 ins_pipe(lmul_reg_reg);
10233 %}
10234
10235 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10236 %{
10237 match(Set dst (MulHiL src1 src2));
10238
10239 ins_cost(INSN_COST * 7);
10240 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10241
10242 ins_encode %{
10243 __ smulh(as_Register($dst$$reg),
10244 as_Register($src1$$reg),
10245 as_Register($src2$$reg));
10246 %}
10247
10248 ins_pipe(lmul_reg_reg);
10249 %}
10250
10251 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10252 %{
10253 match(Set dst (UMulHiL src1 src2));
10254
10255 ins_cost(INSN_COST * 7);
10256 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10257
10258 ins_encode %{
10259 __ umulh(as_Register($dst$$reg),
10260 as_Register($src1$$reg),
10261 as_Register($src2$$reg));
10262 %}
10263
10264 ins_pipe(lmul_reg_reg);
10265 %}
10266
10267 // Combined Integer Multiply & Add/Sub
10268
10269 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10270 match(Set dst (AddI src3 (MulI src1 src2)));
10271
10272 ins_cost(INSN_COST * 3);
10273 format %{ "madd $dst, $src1, $src2, $src3" %}
10274
10275 ins_encode %{
10276 __ maddw(as_Register($dst$$reg),
10277 as_Register($src1$$reg),
10278 as_Register($src2$$reg),
10279 as_Register($src3$$reg));
10280 %}
10281
10282 ins_pipe(imac_reg_reg);
10283 %}
10284
10285 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10286 match(Set dst (SubI src3 (MulI src1 src2)));
10287
10288 ins_cost(INSN_COST * 3);
10289 format %{ "msub $dst, $src1, $src2, $src3" %}
10290
10291 ins_encode %{
10292 __ msubw(as_Register($dst$$reg),
10293 as_Register($src1$$reg),
10294 as_Register($src2$$reg),
10295 as_Register($src3$$reg));
10296 %}
10297
10298 ins_pipe(imac_reg_reg);
10299 %}
10300
10301 // Combined Integer Multiply & Neg
10302
10303 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10304 match(Set dst (MulI (SubI zero src1) src2));
10305
10306 ins_cost(INSN_COST * 3);
10307 format %{ "mneg $dst, $src1, $src2" %}
10308
10309 ins_encode %{
10310 __ mnegw(as_Register($dst$$reg),
10311 as_Register($src1$$reg),
10312 as_Register($src2$$reg));
10313 %}
10314
10315 ins_pipe(imac_reg_reg);
10316 %}
10317
10318 // Combined Long Multiply & Add/Sub
10319
10320 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10321 match(Set dst (AddL src3 (MulL src1 src2)));
10322
10323 ins_cost(INSN_COST * 5);
10324 format %{ "madd $dst, $src1, $src2, $src3" %}
10325
10326 ins_encode %{
10327 __ madd(as_Register($dst$$reg),
10328 as_Register($src1$$reg),
10329 as_Register($src2$$reg),
10330 as_Register($src3$$reg));
10331 %}
10332
10333 ins_pipe(lmac_reg_reg);
10334 %}
10335
10336 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10337 match(Set dst (SubL src3 (MulL src1 src2)));
10338
10339 ins_cost(INSN_COST * 5);
10340 format %{ "msub $dst, $src1, $src2, $src3" %}
10341
10342 ins_encode %{
10343 __ msub(as_Register($dst$$reg),
10344 as_Register($src1$$reg),
10345 as_Register($src2$$reg),
10346 as_Register($src3$$reg));
10347 %}
10348
10349 ins_pipe(lmac_reg_reg);
10350 %}
10351
10352 // Combined Long Multiply & Neg
10353
10354 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10355 match(Set dst (MulL (SubL zero src1) src2));
10356
10357 ins_cost(INSN_COST * 5);
10358 format %{ "mneg $dst, $src1, $src2" %}
10359
10360 ins_encode %{
10361 __ mneg(as_Register($dst$$reg),
10362 as_Register($src1$$reg),
10363 as_Register($src2$$reg));
10364 %}
10365
10366 ins_pipe(lmac_reg_reg);
10367 %}
10368
10369 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10370
10371 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10372 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10373
10374 ins_cost(INSN_COST * 3);
10375 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10376
10377 ins_encode %{
10378 __ smaddl(as_Register($dst$$reg),
10379 as_Register($src1$$reg),
10380 as_Register($src2$$reg),
10381 as_Register($src3$$reg));
10382 %}
10383
10384 ins_pipe(imac_reg_reg);
10385 %}
10386
10387 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10388 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10389
10390 ins_cost(INSN_COST * 3);
10391 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10392
10393 ins_encode %{
10394 __ smsubl(as_Register($dst$$reg),
10395 as_Register($src1$$reg),
10396 as_Register($src2$$reg),
10397 as_Register($src3$$reg));
10398 %}
10399
10400 ins_pipe(imac_reg_reg);
10401 %}
10402
10403 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10404 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10405
10406 ins_cost(INSN_COST * 3);
10407 format %{ "smnegl $dst, $src1, $src2" %}
10408
10409 ins_encode %{
10410 __ smnegl(as_Register($dst$$reg),
10411 as_Register($src1$$reg),
10412 as_Register($src2$$reg));
10413 %}
10414
10415 ins_pipe(imac_reg_reg);
10416 %}
10417
10418 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10419
10420 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10421 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10422
10423 ins_cost(INSN_COST * 5);
10424 format %{ "mulw rscratch1, $src1, $src2\n\t"
10425 "maddw $dst, $src3, $src4, rscratch1" %}
10426
10427 ins_encode %{
10428 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10429 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10430
10431 ins_pipe(imac_reg_reg);
10432 %}
10433
10434 // Integer Divide
10435
10436 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10437 match(Set dst (DivI src1 src2));
10438
10439 ins_cost(INSN_COST * 19);
10440 format %{ "sdivw $dst, $src1, $src2" %}
10441
10442 ins_encode(aarch64_enc_divw(dst, src1, src2));
10443 ins_pipe(idiv_reg_reg);
10444 %}
10445
10446 // Long Divide
10447
10448 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10449 match(Set dst (DivL src1 src2));
10450
10451 ins_cost(INSN_COST * 35);
10452 format %{ "sdiv $dst, $src1, $src2" %}
10453
10454 ins_encode(aarch64_enc_div(dst, src1, src2));
10455 ins_pipe(ldiv_reg_reg);
10456 %}
10457
10458 // Integer Remainder
10459
10460 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10461 match(Set dst (ModI src1 src2));
10462
10463 ins_cost(INSN_COST * 22);
10464 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10465 "msubw $dst, rscratch1, $src2, $src1" %}
10466
10467 ins_encode(aarch64_enc_modw(dst, src1, src2));
10468 ins_pipe(idiv_reg_reg);
10469 %}
10470
10471 // Long Remainder
10472
10473 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10474 match(Set dst (ModL src1 src2));
10475
10476 ins_cost(INSN_COST * 38);
10477 format %{ "sdiv rscratch1, $src1, $src2\n"
10478 "msub $dst, rscratch1, $src2, $src1" %}
10479
10480 ins_encode(aarch64_enc_mod(dst, src1, src2));
10481 ins_pipe(ldiv_reg_reg);
10482 %}
10483
10484 // Unsigned Integer Divide
10485
10486 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10487 match(Set dst (UDivI src1 src2));
10488
10489 ins_cost(INSN_COST * 19);
10490 format %{ "udivw $dst, $src1, $src2" %}
10491
10492 ins_encode %{
10493 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10494 %}
10495
10496 ins_pipe(idiv_reg_reg);
10497 %}
10498
10499 // Unsigned Long Divide
10500
10501 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10502 match(Set dst (UDivL src1 src2));
10503
10504 ins_cost(INSN_COST * 35);
10505 format %{ "udiv $dst, $src1, $src2" %}
10506
10507 ins_encode %{
10508 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10509 %}
10510
10511 ins_pipe(ldiv_reg_reg);
10512 %}
10513
10514 // Unsigned Integer Remainder
10515
10516 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10517 match(Set dst (UModI src1 src2));
10518
10519 ins_cost(INSN_COST * 22);
10520 format %{ "udivw rscratch1, $src1, $src2\n\t"
10521 "msubw $dst, rscratch1, $src2, $src1" %}
10522
10523 ins_encode %{
10524 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10525 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10526 %}
10527
10528 ins_pipe(idiv_reg_reg);
10529 %}
10530
10531 // Unsigned Long Remainder
10532
10533 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10534 match(Set dst (UModL src1 src2));
10535
10536 ins_cost(INSN_COST * 38);
10537 format %{ "udiv rscratch1, $src1, $src2\n"
10538 "msub $dst, rscratch1, $src2, $src1" %}
10539
10540 ins_encode %{
10541 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10542 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10543 %}
10544
10545 ins_pipe(ldiv_reg_reg);
10546 %}
10547
10548 // Integer Shifts
10549
10550 // Shift Left Register
10551 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10552 match(Set dst (LShiftI src1 src2));
10553
10554 ins_cost(INSN_COST * 2);
10555 format %{ "lslvw $dst, $src1, $src2" %}
10556
10557 ins_encode %{
10558 __ lslvw(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 as_Register($src2$$reg));
10561 %}
10562
10563 ins_pipe(ialu_reg_reg_vshift);
10564 %}
10565
10566 // Shift Left Immediate
10567 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10568 match(Set dst (LShiftI src1 src2));
10569
10570 ins_cost(INSN_COST);
10571 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10572
10573 ins_encode %{
10574 __ lslw(as_Register($dst$$reg),
10575 as_Register($src1$$reg),
10576 $src2$$constant & 0x1f);
10577 %}
10578
10579 ins_pipe(ialu_reg_shift);
10580 %}
10581
10582 // Shift Right Logical Register
10583 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10584 match(Set dst (URShiftI src1 src2));
10585
10586 ins_cost(INSN_COST * 2);
10587 format %{ "lsrvw $dst, $src1, $src2" %}
10588
10589 ins_encode %{
10590 __ lsrvw(as_Register($dst$$reg),
10591 as_Register($src1$$reg),
10592 as_Register($src2$$reg));
10593 %}
10594
10595 ins_pipe(ialu_reg_reg_vshift);
10596 %}
10597
10598 // Shift Right Logical Immediate
10599 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10600 match(Set dst (URShiftI src1 src2));
10601
10602 ins_cost(INSN_COST);
10603 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10604
10605 ins_encode %{
10606 __ lsrw(as_Register($dst$$reg),
10607 as_Register($src1$$reg),
10608 $src2$$constant & 0x1f);
10609 %}
10610
10611 ins_pipe(ialu_reg_shift);
10612 %}
10613
10614 // Shift Right Arithmetic Register
10615 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10616 match(Set dst (RShiftI src1 src2));
10617
10618 ins_cost(INSN_COST * 2);
10619 format %{ "asrvw $dst, $src1, $src2" %}
10620
10621 ins_encode %{
10622 __ asrvw(as_Register($dst$$reg),
10623 as_Register($src1$$reg),
10624 as_Register($src2$$reg));
10625 %}
10626
10627 ins_pipe(ialu_reg_reg_vshift);
10628 %}
10629
10630 // Shift Right Arithmetic Immediate
10631 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10632 match(Set dst (RShiftI src1 src2));
10633
10634 ins_cost(INSN_COST);
10635 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10636
10637 ins_encode %{
10638 __ asrw(as_Register($dst$$reg),
10639 as_Register($src1$$reg),
10640 $src2$$constant & 0x1f);
10641 %}
10642
10643 ins_pipe(ialu_reg_shift);
10644 %}
10645
10646 // Combined Int Mask and Right Shift (using UBFM)
10647 // TODO
10648
10649 // Long Shifts
10650
10651 // Shift Left Register
10652 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10653 match(Set dst (LShiftL src1 src2));
10654
10655 ins_cost(INSN_COST * 2);
10656 format %{ "lslv $dst, $src1, $src2" %}
10657
10658 ins_encode %{
10659 __ lslv(as_Register($dst$$reg),
10660 as_Register($src1$$reg),
10661 as_Register($src2$$reg));
10662 %}
10663
10664 ins_pipe(ialu_reg_reg_vshift);
10665 %}
10666
10667 // Shift Left Immediate
10668 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10669 match(Set dst (LShiftL src1 src2));
10670
10671 ins_cost(INSN_COST);
10672 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10673
10674 ins_encode %{
10675 __ lsl(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 $src2$$constant & 0x3f);
10678 %}
10679
10680 ins_pipe(ialu_reg_shift);
10681 %}
10682
10683 // Shift Right Logical Register
10684 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10685 match(Set dst (URShiftL src1 src2));
10686
10687 ins_cost(INSN_COST * 2);
10688 format %{ "lsrv $dst, $src1, $src2" %}
10689
10690 ins_encode %{
10691 __ lsrv(as_Register($dst$$reg),
10692 as_Register($src1$$reg),
10693 as_Register($src2$$reg));
10694 %}
10695
10696 ins_pipe(ialu_reg_reg_vshift);
10697 %}
10698
10699 // Shift Right Logical Immediate
10700 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10701 match(Set dst (URShiftL src1 src2));
10702
10703 ins_cost(INSN_COST);
10704 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10705
10706 ins_encode %{
10707 __ lsr(as_Register($dst$$reg),
10708 as_Register($src1$$reg),
10709 $src2$$constant & 0x3f);
10710 %}
10711
10712 ins_pipe(ialu_reg_shift);
10713 %}
10714
10715 // A special-case pattern for card table stores.
10716 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10717 match(Set dst (URShiftL (CastP2X src1) src2));
10718
10719 ins_cost(INSN_COST);
10720 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10721
10722 ins_encode %{
10723 __ lsr(as_Register($dst$$reg),
10724 as_Register($src1$$reg),
10725 $src2$$constant & 0x3f);
10726 %}
10727
10728 ins_pipe(ialu_reg_shift);
10729 %}
10730
10731 // Shift Right Arithmetic Register
10732 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10733 match(Set dst (RShiftL src1 src2));
10734
10735 ins_cost(INSN_COST * 2);
10736 format %{ "asrv $dst, $src1, $src2" %}
10737
10738 ins_encode %{
10739 __ asrv(as_Register($dst$$reg),
10740 as_Register($src1$$reg),
10741 as_Register($src2$$reg));
10742 %}
10743
10744 ins_pipe(ialu_reg_reg_vshift);
10745 %}
10746
10747 // Shift Right Arithmetic Immediate
10748 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10749 match(Set dst (RShiftL src1 src2));
10750
10751 ins_cost(INSN_COST);
10752 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10753
10754 ins_encode %{
10755 __ asr(as_Register($dst$$reg),
10756 as_Register($src1$$reg),
10757 $src2$$constant & 0x3f);
10758 %}
10759
10760 ins_pipe(ialu_reg_shift);
10761 %}
10762
10763 // BEGIN This section of the file is automatically generated. Do not edit --------------
10764 // This section is generated from aarch64_ad.m4
10765
10766 // This pattern is automatically generated from aarch64_ad.m4
10767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10768 instruct regL_not_reg(iRegLNoSp dst,
10769 iRegL src1, immL_M1 m1,
10770 rFlagsReg cr) %{
10771 match(Set dst (XorL src1 m1));
10772 ins_cost(INSN_COST);
10773 format %{ "eon $dst, $src1, zr" %}
10774
10775 ins_encode %{
10776 __ eon(as_Register($dst$$reg),
10777 as_Register($src1$$reg),
10778 zr,
10779 Assembler::LSL, 0);
10780 %}
10781
10782 ins_pipe(ialu_reg);
10783 %}
10784
10785 // This pattern is automatically generated from aarch64_ad.m4
10786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10787 instruct regI_not_reg(iRegINoSp dst,
10788 iRegIorL2I src1, immI_M1 m1,
10789 rFlagsReg cr) %{
10790 match(Set dst (XorI src1 m1));
10791 ins_cost(INSN_COST);
10792 format %{ "eonw $dst, $src1, zr" %}
10793
10794 ins_encode %{
10795 __ eonw(as_Register($dst$$reg),
10796 as_Register($src1$$reg),
10797 zr,
10798 Assembler::LSL, 0);
10799 %}
10800
10801 ins_pipe(ialu_reg);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10807 immI0 zero, iRegIorL2I src1, immI src2) %{
10808 match(Set dst (SubI zero (URShiftI src1 src2)));
10809
10810 ins_cost(1.9 * INSN_COST);
10811 format %{ "negw $dst, $src1, LSR $src2" %}
10812
10813 ins_encode %{
10814 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10815 Assembler::LSR, $src2$$constant & 0x1f);
10816 %}
10817
10818 ins_pipe(ialu_reg_shift);
10819 %}
10820
10821 // This pattern is automatically generated from aarch64_ad.m4
10822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10823 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10824 immI0 zero, iRegIorL2I src1, immI src2) %{
10825 match(Set dst (SubI zero (RShiftI src1 src2)));
10826
10827 ins_cost(1.9 * INSN_COST);
10828 format %{ "negw $dst, $src1, ASR $src2" %}
10829
10830 ins_encode %{
10831 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10832 Assembler::ASR, $src2$$constant & 0x1f);
10833 %}
10834
10835 ins_pipe(ialu_reg_shift);
10836 %}
10837
10838 // This pattern is automatically generated from aarch64_ad.m4
10839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10840 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10841 immI0 zero, iRegIorL2I src1, immI src2) %{
10842 match(Set dst (SubI zero (LShiftI src1 src2)));
10843
10844 ins_cost(1.9 * INSN_COST);
10845 format %{ "negw $dst, $src1, LSL $src2" %}
10846
10847 ins_encode %{
10848 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10849 Assembler::LSL, $src2$$constant & 0x1f);
10850 %}
10851
10852 ins_pipe(ialu_reg_shift);
10853 %}
10854
10855 // This pattern is automatically generated from aarch64_ad.m4
10856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10857 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10858 immL0 zero, iRegL src1, immI src2) %{
10859 match(Set dst (SubL zero (URShiftL src1 src2)));
10860
10861 ins_cost(1.9 * INSN_COST);
10862 format %{ "neg $dst, $src1, LSR $src2" %}
10863
10864 ins_encode %{
10865 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10866 Assembler::LSR, $src2$$constant & 0x3f);
10867 %}
10868
10869 ins_pipe(ialu_reg_shift);
10870 %}
10871
10872 // This pattern is automatically generated from aarch64_ad.m4
10873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10874 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10875 immL0 zero, iRegL src1, immI src2) %{
10876 match(Set dst (SubL zero (RShiftL src1 src2)));
10877
10878 ins_cost(1.9 * INSN_COST);
10879 format %{ "neg $dst, $src1, ASR $src2" %}
10880
10881 ins_encode %{
10882 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10883 Assembler::ASR, $src2$$constant & 0x3f);
10884 %}
10885
10886 ins_pipe(ialu_reg_shift);
10887 %}
10888
10889 // This pattern is automatically generated from aarch64_ad.m4
10890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10891 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10892 immL0 zero, iRegL src1, immI src2) %{
10893 match(Set dst (SubL zero (LShiftL src1 src2)));
10894
10895 ins_cost(1.9 * INSN_COST);
10896 format %{ "neg $dst, $src1, LSL $src2" %}
10897
10898 ins_encode %{
10899 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10900 Assembler::LSL, $src2$$constant & 0x3f);
10901 %}
10902
10903 ins_pipe(ialu_reg_shift);
10904 %}
10905
10906 // This pattern is automatically generated from aarch64_ad.m4
10907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10908 instruct AndI_reg_not_reg(iRegINoSp dst,
10909 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10910 match(Set dst (AndI src1 (XorI src2 m1)));
10911 ins_cost(INSN_COST);
10912 format %{ "bicw $dst, $src1, $src2" %}
10913
10914 ins_encode %{
10915 __ bicw(as_Register($dst$$reg),
10916 as_Register($src1$$reg),
10917 as_Register($src2$$reg),
10918 Assembler::LSL, 0);
10919 %}
10920
10921 ins_pipe(ialu_reg_reg);
10922 %}
10923
10924 // This pattern is automatically generated from aarch64_ad.m4
10925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10926 instruct AndL_reg_not_reg(iRegLNoSp dst,
10927 iRegL src1, iRegL src2, immL_M1 m1) %{
10928 match(Set dst (AndL src1 (XorL src2 m1)));
10929 ins_cost(INSN_COST);
10930 format %{ "bic $dst, $src1, $src2" %}
10931
10932 ins_encode %{
10933 __ bic(as_Register($dst$$reg),
10934 as_Register($src1$$reg),
10935 as_Register($src2$$reg),
10936 Assembler::LSL, 0);
10937 %}
10938
10939 ins_pipe(ialu_reg_reg);
10940 %}
10941
10942 // This pattern is automatically generated from aarch64_ad.m4
10943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10944 instruct OrI_reg_not_reg(iRegINoSp dst,
10945 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10946 match(Set dst (OrI src1 (XorI src2 m1)));
10947 ins_cost(INSN_COST);
10948 format %{ "ornw $dst, $src1, $src2" %}
10949
10950 ins_encode %{
10951 __ ornw(as_Register($dst$$reg),
10952 as_Register($src1$$reg),
10953 as_Register($src2$$reg),
10954 Assembler::LSL, 0);
10955 %}
10956
10957 ins_pipe(ialu_reg_reg);
10958 %}
10959
10960 // This pattern is automatically generated from aarch64_ad.m4
10961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10962 instruct OrL_reg_not_reg(iRegLNoSp dst,
10963 iRegL src1, iRegL src2, immL_M1 m1) %{
10964 match(Set dst (OrL src1 (XorL src2 m1)));
10965 ins_cost(INSN_COST);
10966 format %{ "orn $dst, $src1, $src2" %}
10967
10968 ins_encode %{
10969 __ orn(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::LSL, 0);
10973 %}
10974
10975 ins_pipe(ialu_reg_reg);
10976 %}
10977
10978 // This pattern is automatically generated from aarch64_ad.m4
10979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10980 instruct XorI_reg_not_reg(iRegINoSp dst,
10981 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10982 match(Set dst (XorI m1 (XorI src2 src1)));
10983 ins_cost(INSN_COST);
10984 format %{ "eonw $dst, $src1, $src2" %}
10985
10986 ins_encode %{
10987 __ eonw(as_Register($dst$$reg),
10988 as_Register($src1$$reg),
10989 as_Register($src2$$reg),
10990 Assembler::LSL, 0);
10991 %}
10992
10993 ins_pipe(ialu_reg_reg);
10994 %}
10995
10996 // This pattern is automatically generated from aarch64_ad.m4
10997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10998 instruct XorL_reg_not_reg(iRegLNoSp dst,
10999 iRegL src1, iRegL src2, immL_M1 m1) %{
11000 match(Set dst (XorL m1 (XorL src2 src1)));
11001 ins_cost(INSN_COST);
11002 format %{ "eon $dst, $src1, $src2" %}
11003
11004 ins_encode %{
11005 __ eon(as_Register($dst$$reg),
11006 as_Register($src1$$reg),
11007 as_Register($src2$$reg),
11008 Assembler::LSL, 0);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 // val & (-1 ^ (val >>> shift)) ==> bicw
11017 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11018 iRegIorL2I src1, iRegIorL2I src2,
11019 immI src3, immI_M1 src4) %{
11020 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11021 ins_cost(1.9 * INSN_COST);
11022 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11023
11024 ins_encode %{
11025 __ bicw(as_Register($dst$$reg),
11026 as_Register($src1$$reg),
11027 as_Register($src2$$reg),
11028 Assembler::LSR,
11029 $src3$$constant & 0x1f);
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 // val & (-1 ^ (val >>> shift)) ==> bic
11038 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11039 iRegL src1, iRegL src2,
11040 immI src3, immL_M1 src4) %{
11041 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11042 ins_cost(1.9 * INSN_COST);
11043 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11044
11045 ins_encode %{
11046 __ bic(as_Register($dst$$reg),
11047 as_Register($src1$$reg),
11048 as_Register($src2$$reg),
11049 Assembler::LSR,
11050 $src3$$constant & 0x3f);
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_shift);
11054 %}
11055
11056 // This pattern is automatically generated from aarch64_ad.m4
11057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11058 // val & (-1 ^ (val >> shift)) ==> bicw
11059 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11060 iRegIorL2I src1, iRegIorL2I src2,
11061 immI src3, immI_M1 src4) %{
11062 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11063 ins_cost(1.9 * INSN_COST);
11064 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11065
11066 ins_encode %{
11067 __ bicw(as_Register($dst$$reg),
11068 as_Register($src1$$reg),
11069 as_Register($src2$$reg),
11070 Assembler::ASR,
11071 $src3$$constant & 0x1f);
11072 %}
11073
11074 ins_pipe(ialu_reg_reg_shift);
11075 %}
11076
11077 // This pattern is automatically generated from aarch64_ad.m4
11078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11079 // val & (-1 ^ (val >> shift)) ==> bic
11080 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11081 iRegL src1, iRegL src2,
11082 immI src3, immL_M1 src4) %{
11083 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11084 ins_cost(1.9 * INSN_COST);
11085 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11086
11087 ins_encode %{
11088 __ bic(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg),
11091 Assembler::ASR,
11092 $src3$$constant & 0x3f);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg_shift);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 // val & (-1 ^ (val ror shift)) ==> bicw
11101 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11102 iRegIorL2I src1, iRegIorL2I src2,
11103 immI src3, immI_M1 src4) %{
11104 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11107
11108 ins_encode %{
11109 __ bicw(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::ROR,
11113 $src3$$constant & 0x1f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 // val & (-1 ^ (val ror shift)) ==> bic
11122 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11123 iRegL src1, iRegL src2,
11124 immI src3, immL_M1 src4) %{
11125 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11128
11129 ins_encode %{
11130 __ bic(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::ROR,
11134 $src3$$constant & 0x3f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 // val & (-1 ^ (val << shift)) ==> bicw
11143 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11144 iRegIorL2I src1, iRegIorL2I src2,
11145 immI src3, immI_M1 src4) %{
11146 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11149
11150 ins_encode %{
11151 __ bicw(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::LSL,
11155 $src3$$constant & 0x1f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 // val & (-1 ^ (val << shift)) ==> bic
11164 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11165 iRegL src1, iRegL src2,
11166 immI src3, immL_M1 src4) %{
11167 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11170
11171 ins_encode %{
11172 __ bic(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::LSL,
11176 $src3$$constant & 0x3f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11185 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11186 iRegIorL2I src1, iRegIorL2I src2,
11187 immI src3, immI_M1 src4) %{
11188 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11191
11192 ins_encode %{
11193 __ eonw(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::LSR,
11197 $src3$$constant & 0x1f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 // val ^ (-1 ^ (val >>> shift)) ==> eon
11206 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11207 iRegL src1, iRegL src2,
11208 immI src3, immL_M1 src4) %{
11209 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11212
11213 ins_encode %{
11214 __ eon(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::LSR,
11218 $src3$$constant & 0x3f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 // val ^ (-1 ^ (val >> shift)) ==> eonw
11227 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11228 iRegIorL2I src1, iRegIorL2I src2,
11229 immI src3, immI_M1 src4) %{
11230 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11233
11234 ins_encode %{
11235 __ eonw(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::ASR,
11239 $src3$$constant & 0x1f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 // val ^ (-1 ^ (val >> shift)) ==> eon
11248 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11249 iRegL src1, iRegL src2,
11250 immI src3, immL_M1 src4) %{
11251 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11254
11255 ins_encode %{
11256 __ eon(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::ASR,
11260 $src3$$constant & 0x3f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 // val ^ (-1 ^ (val ror shift)) ==> eonw
11269 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11270 iRegIorL2I src1, iRegIorL2I src2,
11271 immI src3, immI_M1 src4) %{
11272 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11275
11276 ins_encode %{
11277 __ eonw(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::ROR,
11281 $src3$$constant & 0x1f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 // val ^ (-1 ^ (val ror shift)) ==> eon
11290 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11291 iRegL src1, iRegL src2,
11292 immI src3, immL_M1 src4) %{
11293 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11296
11297 ins_encode %{
11298 __ eon(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::ROR,
11302 $src3$$constant & 0x3f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 // val ^ (-1 ^ (val << shift)) ==> eonw
11311 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11312 iRegIorL2I src1, iRegIorL2I src2,
11313 immI src3, immI_M1 src4) %{
11314 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11317
11318 ins_encode %{
11319 __ eonw(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::LSL,
11323 $src3$$constant & 0x1f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 // val ^ (-1 ^ (val << shift)) ==> eon
11332 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11333 iRegL src1, iRegL src2,
11334 immI src3, immL_M1 src4) %{
11335 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11338
11339 ins_encode %{
11340 __ eon(as_Register($dst$$reg),
11341 as_Register($src1$$reg),
11342 as_Register($src2$$reg),
11343 Assembler::LSL,
11344 $src3$$constant & 0x3f);
11345 %}
11346
11347 ins_pipe(ialu_reg_reg_shift);
11348 %}
11349
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 // val | (-1 ^ (val >>> shift)) ==> ornw
11353 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11354 iRegIorL2I src1, iRegIorL2I src2,
11355 immI src3, immI_M1 src4) %{
11356 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11357 ins_cost(1.9 * INSN_COST);
11358 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11359
11360 ins_encode %{
11361 __ ornw(as_Register($dst$$reg),
11362 as_Register($src1$$reg),
11363 as_Register($src2$$reg),
11364 Assembler::LSR,
11365 $src3$$constant & 0x1f);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg_shift);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 // val | (-1 ^ (val >>> shift)) ==> orn
11374 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11375 iRegL src1, iRegL src2,
11376 immI src3, immL_M1 src4) %{
11377 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11378 ins_cost(1.9 * INSN_COST);
11379 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11380
11381 ins_encode %{
11382 __ orn(as_Register($dst$$reg),
11383 as_Register($src1$$reg),
11384 as_Register($src2$$reg),
11385 Assembler::LSR,
11386 $src3$$constant & 0x3f);
11387 %}
11388
11389 ins_pipe(ialu_reg_reg_shift);
11390 %}
11391
11392 // This pattern is automatically generated from aarch64_ad.m4
11393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11394 // val | (-1 ^ (val >> shift)) ==> ornw
11395 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11396 iRegIorL2I src1, iRegIorL2I src2,
11397 immI src3, immI_M1 src4) %{
11398 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11399 ins_cost(1.9 * INSN_COST);
11400 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11401
11402 ins_encode %{
11403 __ ornw(as_Register($dst$$reg),
11404 as_Register($src1$$reg),
11405 as_Register($src2$$reg),
11406 Assembler::ASR,
11407 $src3$$constant & 0x1f);
11408 %}
11409
11410 ins_pipe(ialu_reg_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 // val | (-1 ^ (val >> shift)) ==> orn
11416 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11417 iRegL src1, iRegL src2,
11418 immI src3, immL_M1 src4) %{
11419 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11420 ins_cost(1.9 * INSN_COST);
11421 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11422
11423 ins_encode %{
11424 __ orn(as_Register($dst$$reg),
11425 as_Register($src1$$reg),
11426 as_Register($src2$$reg),
11427 Assembler::ASR,
11428 $src3$$constant & 0x3f);
11429 %}
11430
11431 ins_pipe(ialu_reg_reg_shift);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436 // val | (-1 ^ (val ror shift)) ==> ornw
11437 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11438 iRegIorL2I src1, iRegIorL2I src2,
11439 immI src3, immI_M1 src4) %{
11440 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11441 ins_cost(1.9 * INSN_COST);
11442 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11443
11444 ins_encode %{
11445 __ ornw(as_Register($dst$$reg),
11446 as_Register($src1$$reg),
11447 as_Register($src2$$reg),
11448 Assembler::ROR,
11449 $src3$$constant & 0x1f);
11450 %}
11451
11452 ins_pipe(ialu_reg_reg_shift);
11453 %}
11454
11455 // This pattern is automatically generated from aarch64_ad.m4
11456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11457 // val | (-1 ^ (val ror shift)) ==> orn
11458 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11459 iRegL src1, iRegL src2,
11460 immI src3, immL_M1 src4) %{
11461 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11462 ins_cost(1.9 * INSN_COST);
11463 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11464
11465 ins_encode %{
11466 __ orn(as_Register($dst$$reg),
11467 as_Register($src1$$reg),
11468 as_Register($src2$$reg),
11469 Assembler::ROR,
11470 $src3$$constant & 0x3f);
11471 %}
11472
11473 ins_pipe(ialu_reg_reg_shift);
11474 %}
11475
11476 // This pattern is automatically generated from aarch64_ad.m4
11477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11478 // val | (-1 ^ (val << shift)) ==> ornw
11479 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11480 iRegIorL2I src1, iRegIorL2I src2,
11481 immI src3, immI_M1 src4) %{
11482 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11483 ins_cost(1.9 * INSN_COST);
11484 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11485
11486 ins_encode %{
11487 __ ornw(as_Register($dst$$reg),
11488 as_Register($src1$$reg),
11489 as_Register($src2$$reg),
11490 Assembler::LSL,
11491 $src3$$constant & 0x1f);
11492 %}
11493
11494 ins_pipe(ialu_reg_reg_shift);
11495 %}
11496
11497 // This pattern is automatically generated from aarch64_ad.m4
11498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11499 // val | (-1 ^ (val << shift)) ==> orn
11500 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11501 iRegL src1, iRegL src2,
11502 immI src3, immL_M1 src4) %{
11503 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11504 ins_cost(1.9 * INSN_COST);
11505 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11506
11507 ins_encode %{
11508 __ orn(as_Register($dst$$reg),
11509 as_Register($src1$$reg),
11510 as_Register($src2$$reg),
11511 Assembler::LSL,
11512 $src3$$constant & 0x3f);
11513 %}
11514
11515 ins_pipe(ialu_reg_reg_shift);
11516 %}
11517
11518 // This pattern is automatically generated from aarch64_ad.m4
11519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11520 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11521 iRegIorL2I src1, iRegIorL2I src2,
11522 immI src3) %{
11523 match(Set dst (AndI src1 (URShiftI src2 src3)));
11524
11525 ins_cost(1.9 * INSN_COST);
11526 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11527
11528 ins_encode %{
11529 __ andw(as_Register($dst$$reg),
11530 as_Register($src1$$reg),
11531 as_Register($src2$$reg),
11532 Assembler::LSR,
11533 $src3$$constant & 0x1f);
11534 %}
11535
11536 ins_pipe(ialu_reg_reg_shift);
11537 %}
11538
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11542 iRegL src1, iRegL src2,
11543 immI src3) %{
11544 match(Set dst (AndL src1 (URShiftL src2 src3)));
11545
11546 ins_cost(1.9 * INSN_COST);
11547 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11548
11549 ins_encode %{
11550 __ andr(as_Register($dst$$reg),
11551 as_Register($src1$$reg),
11552 as_Register($src2$$reg),
11553 Assembler::LSR,
11554 $src3$$constant & 0x3f);
11555 %}
11556
11557 ins_pipe(ialu_reg_reg_shift);
11558 %}
11559
11560 // This pattern is automatically generated from aarch64_ad.m4
11561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11562 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11563 iRegIorL2I src1, iRegIorL2I src2,
11564 immI src3) %{
11565 match(Set dst (AndI src1 (RShiftI src2 src3)));
11566
11567 ins_cost(1.9 * INSN_COST);
11568 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11569
11570 ins_encode %{
11571 __ andw(as_Register($dst$$reg),
11572 as_Register($src1$$reg),
11573 as_Register($src2$$reg),
11574 Assembler::ASR,
11575 $src3$$constant & 0x1f);
11576 %}
11577
11578 ins_pipe(ialu_reg_reg_shift);
11579 %}
11580
11581 // This pattern is automatically generated from aarch64_ad.m4
11582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11583 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11584 iRegL src1, iRegL src2,
11585 immI src3) %{
11586 match(Set dst (AndL src1 (RShiftL src2 src3)));
11587
11588 ins_cost(1.9 * INSN_COST);
11589 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11590
11591 ins_encode %{
11592 __ andr(as_Register($dst$$reg),
11593 as_Register($src1$$reg),
11594 as_Register($src2$$reg),
11595 Assembler::ASR,
11596 $src3$$constant & 0x3f);
11597 %}
11598
11599 ins_pipe(ialu_reg_reg_shift);
11600 %}
11601
11602 // This pattern is automatically generated from aarch64_ad.m4
11603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11604 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11605 iRegIorL2I src1, iRegIorL2I src2,
11606 immI src3) %{
11607 match(Set dst (AndI src1 (LShiftI src2 src3)));
11608
11609 ins_cost(1.9 * INSN_COST);
11610 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11611
11612 ins_encode %{
11613 __ andw(as_Register($dst$$reg),
11614 as_Register($src1$$reg),
11615 as_Register($src2$$reg),
11616 Assembler::LSL,
11617 $src3$$constant & 0x1f);
11618 %}
11619
11620 ins_pipe(ialu_reg_reg_shift);
11621 %}
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11626 iRegL src1, iRegL src2,
11627 immI src3) %{
11628 match(Set dst (AndL src1 (LShiftL src2 src3)));
11629
11630 ins_cost(1.9 * INSN_COST);
11631 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11632
11633 ins_encode %{
11634 __ andr(as_Register($dst$$reg),
11635 as_Register($src1$$reg),
11636 as_Register($src2$$reg),
11637 Assembler::LSL,
11638 $src3$$constant & 0x3f);
11639 %}
11640
11641 ins_pipe(ialu_reg_reg_shift);
11642 %}
11643
11644 // This pattern is automatically generated from aarch64_ad.m4
11645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11646 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11647 iRegIorL2I src1, iRegIorL2I src2,
11648 immI src3) %{
11649 match(Set dst (AndI src1 (RotateRight src2 src3)));
11650
11651 ins_cost(1.9 * INSN_COST);
11652 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11653
11654 ins_encode %{
11655 __ andw(as_Register($dst$$reg),
11656 as_Register($src1$$reg),
11657 as_Register($src2$$reg),
11658 Assembler::ROR,
11659 $src3$$constant & 0x1f);
11660 %}
11661
11662 ins_pipe(ialu_reg_reg_shift);
11663 %}
11664
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11668 iRegL src1, iRegL src2,
11669 immI src3) %{
11670 match(Set dst (AndL src1 (RotateRight src2 src3)));
11671
11672 ins_cost(1.9 * INSN_COST);
11673 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11674
11675 ins_encode %{
11676 __ andr(as_Register($dst$$reg),
11677 as_Register($src1$$reg),
11678 as_Register($src2$$reg),
11679 Assembler::ROR,
11680 $src3$$constant & 0x3f);
11681 %}
11682
11683 ins_pipe(ialu_reg_reg_shift);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11689 iRegIorL2I src1, iRegIorL2I src2,
11690 immI src3) %{
11691 match(Set dst (XorI src1 (URShiftI src2 src3)));
11692
11693 ins_cost(1.9 * INSN_COST);
11694 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11695
11696 ins_encode %{
11697 __ eorw(as_Register($dst$$reg),
11698 as_Register($src1$$reg),
11699 as_Register($src2$$reg),
11700 Assembler::LSR,
11701 $src3$$constant & 0x1f);
11702 %}
11703
11704 ins_pipe(ialu_reg_reg_shift);
11705 %}
11706
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11710 iRegL src1, iRegL src2,
11711 immI src3) %{
11712 match(Set dst (XorL src1 (URShiftL src2 src3)));
11713
11714 ins_cost(1.9 * INSN_COST);
11715 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11716
11717 ins_encode %{
11718 __ eor(as_Register($dst$$reg),
11719 as_Register($src1$$reg),
11720 as_Register($src2$$reg),
11721 Assembler::LSR,
11722 $src3$$constant & 0x3f);
11723 %}
11724
11725 ins_pipe(ialu_reg_reg_shift);
11726 %}
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11731 iRegIorL2I src1, iRegIorL2I src2,
11732 immI src3) %{
11733 match(Set dst (XorI src1 (RShiftI src2 src3)));
11734
11735 ins_cost(1.9 * INSN_COST);
11736 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11737
11738 ins_encode %{
11739 __ eorw(as_Register($dst$$reg),
11740 as_Register($src1$$reg),
11741 as_Register($src2$$reg),
11742 Assembler::ASR,
11743 $src3$$constant & 0x1f);
11744 %}
11745
11746 ins_pipe(ialu_reg_reg_shift);
11747 %}
11748
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11752 iRegL src1, iRegL src2,
11753 immI src3) %{
11754 match(Set dst (XorL src1 (RShiftL src2 src3)));
11755
11756 ins_cost(1.9 * INSN_COST);
11757 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11758
11759 ins_encode %{
11760 __ eor(as_Register($dst$$reg),
11761 as_Register($src1$$reg),
11762 as_Register($src2$$reg),
11763 Assembler::ASR,
11764 $src3$$constant & 0x3f);
11765 %}
11766
11767 ins_pipe(ialu_reg_reg_shift);
11768 %}
11769
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11773 iRegIorL2I src1, iRegIorL2I src2,
11774 immI src3) %{
11775 match(Set dst (XorI src1 (LShiftI src2 src3)));
11776
11777 ins_cost(1.9 * INSN_COST);
11778 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11779
11780 ins_encode %{
11781 __ eorw(as_Register($dst$$reg),
11782 as_Register($src1$$reg),
11783 as_Register($src2$$reg),
11784 Assembler::LSL,
11785 $src3$$constant & 0x1f);
11786 %}
11787
11788 ins_pipe(ialu_reg_reg_shift);
11789 %}
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11794 iRegL src1, iRegL src2,
11795 immI src3) %{
11796 match(Set dst (XorL src1 (LShiftL src2 src3)));
11797
11798 ins_cost(1.9 * INSN_COST);
11799 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11800
11801 ins_encode %{
11802 __ eor(as_Register($dst$$reg),
11803 as_Register($src1$$reg),
11804 as_Register($src2$$reg),
11805 Assembler::LSL,
11806 $src3$$constant & 0x3f);
11807 %}
11808
11809 ins_pipe(ialu_reg_reg_shift);
11810 %}
11811
11812 // This pattern is automatically generated from aarch64_ad.m4
11813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11814 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11815 iRegIorL2I src1, iRegIorL2I src2,
11816 immI src3) %{
11817 match(Set dst (XorI src1 (RotateRight src2 src3)));
11818
11819 ins_cost(1.9 * INSN_COST);
11820 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11821
11822 ins_encode %{
11823 __ eorw(as_Register($dst$$reg),
11824 as_Register($src1$$reg),
11825 as_Register($src2$$reg),
11826 Assembler::ROR,
11827 $src3$$constant & 0x1f);
11828 %}
11829
11830 ins_pipe(ialu_reg_reg_shift);
11831 %}
11832
11833 // This pattern is automatically generated from aarch64_ad.m4
11834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11835 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11836 iRegL src1, iRegL src2,
11837 immI src3) %{
11838 match(Set dst (XorL src1 (RotateRight src2 src3)));
11839
11840 ins_cost(1.9 * INSN_COST);
11841 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11842
11843 ins_encode %{
11844 __ eor(as_Register($dst$$reg),
11845 as_Register($src1$$reg),
11846 as_Register($src2$$reg),
11847 Assembler::ROR,
11848 $src3$$constant & 0x3f);
11849 %}
11850
11851 ins_pipe(ialu_reg_reg_shift);
11852 %}
11853
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11857 iRegIorL2I src1, iRegIorL2I src2,
11858 immI src3) %{
11859 match(Set dst (OrI src1 (URShiftI src2 src3)));
11860
11861 ins_cost(1.9 * INSN_COST);
11862 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11863
11864 ins_encode %{
11865 __ orrw(as_Register($dst$$reg),
11866 as_Register($src1$$reg),
11867 as_Register($src2$$reg),
11868 Assembler::LSR,
11869 $src3$$constant & 0x1f);
11870 %}
11871
11872 ins_pipe(ialu_reg_reg_shift);
11873 %}
11874
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11878 iRegL src1, iRegL src2,
11879 immI src3) %{
11880 match(Set dst (OrL src1 (URShiftL src2 src3)));
11881
11882 ins_cost(1.9 * INSN_COST);
11883 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11884
11885 ins_encode %{
11886 __ orr(as_Register($dst$$reg),
11887 as_Register($src1$$reg),
11888 as_Register($src2$$reg),
11889 Assembler::LSR,
11890 $src3$$constant & 0x3f);
11891 %}
11892
11893 ins_pipe(ialu_reg_reg_shift);
11894 %}
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11899 iRegIorL2I src1, iRegIorL2I src2,
11900 immI src3) %{
11901 match(Set dst (OrI src1 (RShiftI src2 src3)));
11902
11903 ins_cost(1.9 * INSN_COST);
11904 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11905
11906 ins_encode %{
11907 __ orrw(as_Register($dst$$reg),
11908 as_Register($src1$$reg),
11909 as_Register($src2$$reg),
11910 Assembler::ASR,
11911 $src3$$constant & 0x1f);
11912 %}
11913
11914 ins_pipe(ialu_reg_reg_shift);
11915 %}
11916
11917 // This pattern is automatically generated from aarch64_ad.m4
11918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11919 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11920 iRegL src1, iRegL src2,
11921 immI src3) %{
11922 match(Set dst (OrL src1 (RShiftL src2 src3)));
11923
11924 ins_cost(1.9 * INSN_COST);
11925 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11926
11927 ins_encode %{
11928 __ orr(as_Register($dst$$reg),
11929 as_Register($src1$$reg),
11930 as_Register($src2$$reg),
11931 Assembler::ASR,
11932 $src3$$constant & 0x3f);
11933 %}
11934
11935 ins_pipe(ialu_reg_reg_shift);
11936 %}
11937
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11941 iRegIorL2I src1, iRegIorL2I src2,
11942 immI src3) %{
11943 match(Set dst (OrI src1 (LShiftI src2 src3)));
11944
11945 ins_cost(1.9 * INSN_COST);
11946 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11947
11948 ins_encode %{
11949 __ orrw(as_Register($dst$$reg),
11950 as_Register($src1$$reg),
11951 as_Register($src2$$reg),
11952 Assembler::LSL,
11953 $src3$$constant & 0x1f);
11954 %}
11955
11956 ins_pipe(ialu_reg_reg_shift);
11957 %}
11958
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11962 iRegL src1, iRegL src2,
11963 immI src3) %{
11964 match(Set dst (OrL src1 (LShiftL src2 src3)));
11965
11966 ins_cost(1.9 * INSN_COST);
11967 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11968
11969 ins_encode %{
11970 __ orr(as_Register($dst$$reg),
11971 as_Register($src1$$reg),
11972 as_Register($src2$$reg),
11973 Assembler::LSL,
11974 $src3$$constant & 0x3f);
11975 %}
11976
11977 ins_pipe(ialu_reg_reg_shift);
11978 %}
11979
11980 // This pattern is automatically generated from aarch64_ad.m4
11981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11982 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11983 iRegIorL2I src1, iRegIorL2I src2,
11984 immI src3) %{
11985 match(Set dst (OrI src1 (RotateRight src2 src3)));
11986
11987 ins_cost(1.9 * INSN_COST);
11988 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11989
11990 ins_encode %{
11991 __ orrw(as_Register($dst$$reg),
11992 as_Register($src1$$reg),
11993 as_Register($src2$$reg),
11994 Assembler::ROR,
11995 $src3$$constant & 0x1f);
11996 %}
11997
11998 ins_pipe(ialu_reg_reg_shift);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12004 iRegL src1, iRegL src2,
12005 immI src3) %{
12006 match(Set dst (OrL src1 (RotateRight src2 src3)));
12007
12008 ins_cost(1.9 * INSN_COST);
12009 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12010
12011 ins_encode %{
12012 __ orr(as_Register($dst$$reg),
12013 as_Register($src1$$reg),
12014 as_Register($src2$$reg),
12015 Assembler::ROR,
12016 $src3$$constant & 0x3f);
12017 %}
12018
12019 ins_pipe(ialu_reg_reg_shift);
12020 %}
12021
12022 // This pattern is automatically generated from aarch64_ad.m4
12023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12024 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12025 iRegIorL2I src1, iRegIorL2I src2,
12026 immI src3) %{
12027 match(Set dst (AddI src1 (URShiftI src2 src3)));
12028
12029 ins_cost(1.9 * INSN_COST);
12030 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12031
12032 ins_encode %{
12033 __ addw(as_Register($dst$$reg),
12034 as_Register($src1$$reg),
12035 as_Register($src2$$reg),
12036 Assembler::LSR,
12037 $src3$$constant & 0x1f);
12038 %}
12039
12040 ins_pipe(ialu_reg_reg_shift);
12041 %}
12042
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12046 iRegL src1, iRegL src2,
12047 immI src3) %{
12048 match(Set dst (AddL src1 (URShiftL src2 src3)));
12049
12050 ins_cost(1.9 * INSN_COST);
12051 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12052
12053 ins_encode %{
12054 __ add(as_Register($dst$$reg),
12055 as_Register($src1$$reg),
12056 as_Register($src2$$reg),
12057 Assembler::LSR,
12058 $src3$$constant & 0x3f);
12059 %}
12060
12061 ins_pipe(ialu_reg_reg_shift);
12062 %}
12063
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12067 iRegIorL2I src1, iRegIorL2I src2,
12068 immI src3) %{
12069 match(Set dst (AddI src1 (RShiftI src2 src3)));
12070
12071 ins_cost(1.9 * INSN_COST);
12072 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12073
12074 ins_encode %{
12075 __ addw(as_Register($dst$$reg),
12076 as_Register($src1$$reg),
12077 as_Register($src2$$reg),
12078 Assembler::ASR,
12079 $src3$$constant & 0x1f);
12080 %}
12081
12082 ins_pipe(ialu_reg_reg_shift);
12083 %}
12084
12085 // This pattern is automatically generated from aarch64_ad.m4
12086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12087 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12088 iRegL src1, iRegL src2,
12089 immI src3) %{
12090 match(Set dst (AddL src1 (RShiftL src2 src3)));
12091
12092 ins_cost(1.9 * INSN_COST);
12093 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12094
12095 ins_encode %{
12096 __ add(as_Register($dst$$reg),
12097 as_Register($src1$$reg),
12098 as_Register($src2$$reg),
12099 Assembler::ASR,
12100 $src3$$constant & 0x3f);
12101 %}
12102
12103 ins_pipe(ialu_reg_reg_shift);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12109 iRegIorL2I src1, iRegIorL2I src2,
12110 immI src3) %{
12111 match(Set dst (AddI src1 (LShiftI src2 src3)));
12112
12113 ins_cost(1.9 * INSN_COST);
12114 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12115
12116 ins_encode %{
12117 __ addw(as_Register($dst$$reg),
12118 as_Register($src1$$reg),
12119 as_Register($src2$$reg),
12120 Assembler::LSL,
12121 $src3$$constant & 0x1f);
12122 %}
12123
12124 ins_pipe(ialu_reg_reg_shift);
12125 %}
12126
12127 // This pattern is automatically generated from aarch64_ad.m4
12128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12129 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12130 iRegL src1, iRegL src2,
12131 immI src3) %{
12132 match(Set dst (AddL src1 (LShiftL src2 src3)));
12133
12134 ins_cost(1.9 * INSN_COST);
12135 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12136
12137 ins_encode %{
12138 __ add(as_Register($dst$$reg),
12139 as_Register($src1$$reg),
12140 as_Register($src2$$reg),
12141 Assembler::LSL,
12142 $src3$$constant & 0x3f);
12143 %}
12144
12145 ins_pipe(ialu_reg_reg_shift);
12146 %}
12147
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12151 iRegIorL2I src1, iRegIorL2I src2,
12152 immI src3) %{
12153 match(Set dst (SubI src1 (URShiftI src2 src3)));
12154
12155 ins_cost(1.9 * INSN_COST);
12156 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12157
12158 ins_encode %{
12159 __ subw(as_Register($dst$$reg),
12160 as_Register($src1$$reg),
12161 as_Register($src2$$reg),
12162 Assembler::LSR,
12163 $src3$$constant & 0x1f);
12164 %}
12165
12166 ins_pipe(ialu_reg_reg_shift);
12167 %}
12168
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12172 iRegL src1, iRegL src2,
12173 immI src3) %{
12174 match(Set dst (SubL src1 (URShiftL src2 src3)));
12175
12176 ins_cost(1.9 * INSN_COST);
12177 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12178
12179 ins_encode %{
12180 __ sub(as_Register($dst$$reg),
12181 as_Register($src1$$reg),
12182 as_Register($src2$$reg),
12183 Assembler::LSR,
12184 $src3$$constant & 0x3f);
12185 %}
12186
12187 ins_pipe(ialu_reg_reg_shift);
12188 %}
12189
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12193 iRegIorL2I src1, iRegIorL2I src2,
12194 immI src3) %{
12195 match(Set dst (SubI src1 (RShiftI src2 src3)));
12196
12197 ins_cost(1.9 * INSN_COST);
12198 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12199
12200 ins_encode %{
12201 __ subw(as_Register($dst$$reg),
12202 as_Register($src1$$reg),
12203 as_Register($src2$$reg),
12204 Assembler::ASR,
12205 $src3$$constant & 0x1f);
12206 %}
12207
12208 ins_pipe(ialu_reg_reg_shift);
12209 %}
12210
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12214 iRegL src1, iRegL src2,
12215 immI src3) %{
12216 match(Set dst (SubL src1 (RShiftL src2 src3)));
12217
12218 ins_cost(1.9 * INSN_COST);
12219 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12220
12221 ins_encode %{
12222 __ sub(as_Register($dst$$reg),
12223 as_Register($src1$$reg),
12224 as_Register($src2$$reg),
12225 Assembler::ASR,
12226 $src3$$constant & 0x3f);
12227 %}
12228
12229 ins_pipe(ialu_reg_reg_shift);
12230 %}
12231
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12235 iRegIorL2I src1, iRegIorL2I src2,
12236 immI src3) %{
12237 match(Set dst (SubI src1 (LShiftI src2 src3)));
12238
12239 ins_cost(1.9 * INSN_COST);
12240 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12241
12242 ins_encode %{
12243 __ subw(as_Register($dst$$reg),
12244 as_Register($src1$$reg),
12245 as_Register($src2$$reg),
12246 Assembler::LSL,
12247 $src3$$constant & 0x1f);
12248 %}
12249
12250 ins_pipe(ialu_reg_reg_shift);
12251 %}
12252
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12256 iRegL src1, iRegL src2,
12257 immI src3) %{
12258 match(Set dst (SubL src1 (LShiftL src2 src3)));
12259
12260 ins_cost(1.9 * INSN_COST);
12261 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12262
12263 ins_encode %{
12264 __ sub(as_Register($dst$$reg),
12265 as_Register($src1$$reg),
12266 as_Register($src2$$reg),
12267 Assembler::LSL,
12268 $src3$$constant & 0x3f);
12269 %}
12270
12271 ins_pipe(ialu_reg_reg_shift);
12272 %}
12273
12274 // This pattern is automatically generated from aarch64_ad.m4
12275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12276
12277 // Shift Left followed by Shift Right.
12278 // This idiom is used by the compiler for the i2b bytecode etc.
12279 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12280 %{
12281 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12282 ins_cost(INSN_COST * 2);
12283 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12284 ins_encode %{
12285 int lshift = $lshift_count$$constant & 63;
12286 int rshift = $rshift_count$$constant & 63;
12287 int s = 63 - lshift;
12288 int r = (rshift - lshift) & 63;
12289 __ sbfm(as_Register($dst$$reg),
12290 as_Register($src$$reg),
12291 r, s);
12292 %}
12293
12294 ins_pipe(ialu_reg_shift);
12295 %}
12296
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299
12300 // Shift Left followed by Shift Right.
12301 // This idiom is used by the compiler for the i2b bytecode etc.
12302 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12303 %{
12304 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12305 ins_cost(INSN_COST * 2);
12306 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12307 ins_encode %{
12308 int lshift = $lshift_count$$constant & 31;
12309 int rshift = $rshift_count$$constant & 31;
12310 int s = 31 - lshift;
12311 int r = (rshift - lshift) & 31;
12312 __ sbfmw(as_Register($dst$$reg),
12313 as_Register($src$$reg),
12314 r, s);
12315 %}
12316
12317 ins_pipe(ialu_reg_shift);
12318 %}
12319
12320 // This pattern is automatically generated from aarch64_ad.m4
12321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12322
12323 // Shift Left followed by Shift Right.
12324 // This idiom is used by the compiler for the i2b bytecode etc.
12325 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12326 %{
12327 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12328 ins_cost(INSN_COST * 2);
12329 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12330 ins_encode %{
12331 int lshift = $lshift_count$$constant & 63;
12332 int rshift = $rshift_count$$constant & 63;
12333 int s = 63 - lshift;
12334 int r = (rshift - lshift) & 63;
12335 __ ubfm(as_Register($dst$$reg),
12336 as_Register($src$$reg),
12337 r, s);
12338 %}
12339
12340 ins_pipe(ialu_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345
12346 // Shift Left followed by Shift Right.
12347 // This idiom is used by the compiler for the i2b bytecode etc.
12348 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12349 %{
12350 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12351 ins_cost(INSN_COST * 2);
12352 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12353 ins_encode %{
12354 int lshift = $lshift_count$$constant & 31;
12355 int rshift = $rshift_count$$constant & 31;
12356 int s = 31 - lshift;
12357 int r = (rshift - lshift) & 31;
12358 __ ubfmw(as_Register($dst$$reg),
12359 as_Register($src$$reg),
12360 r, s);
12361 %}
12362
12363 ins_pipe(ialu_reg_shift);
12364 %}
12365
12366 // Bitfield extract with shift & mask
12367
12368 // This pattern is automatically generated from aarch64_ad.m4
12369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12370 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12371 %{
12372 match(Set dst (AndI (URShiftI src rshift) mask));
12373 // Make sure we are not going to exceed what ubfxw can do.
12374 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12375
12376 ins_cost(INSN_COST);
12377 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12378 ins_encode %{
12379 int rshift = $rshift$$constant & 31;
12380 intptr_t mask = $mask$$constant;
12381 int width = exact_log2(mask+1);
12382 __ ubfxw(as_Register($dst$$reg),
12383 as_Register($src$$reg), rshift, width);
12384 %}
12385 ins_pipe(ialu_reg_shift);
12386 %}
12387
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12391 %{
12392 match(Set dst (AndL (URShiftL src rshift) mask));
12393 // Make sure we are not going to exceed what ubfx can do.
12394 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12395
12396 ins_cost(INSN_COST);
12397 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12398 ins_encode %{
12399 int rshift = $rshift$$constant & 63;
12400 intptr_t mask = $mask$$constant;
12401 int width = exact_log2_long(mask+1);
12402 __ ubfx(as_Register($dst$$reg),
12403 as_Register($src$$reg), rshift, width);
12404 %}
12405 ins_pipe(ialu_reg_shift);
12406 %}
12407
12408
12409 // This pattern is automatically generated from aarch64_ad.m4
12410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12411
12412 // We can use ubfx when extending an And with a mask when we know mask
12413 // is positive. We know that because immI_bitmask guarantees it.
12414 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12415 %{
12416 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12417 // Make sure we are not going to exceed what ubfxw can do.
12418 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12419
12420 ins_cost(INSN_COST * 2);
12421 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12422 ins_encode %{
12423 int rshift = $rshift$$constant & 31;
12424 intptr_t mask = $mask$$constant;
12425 int width = exact_log2(mask+1);
12426 __ ubfx(as_Register($dst$$reg),
12427 as_Register($src$$reg), rshift, width);
12428 %}
12429 ins_pipe(ialu_reg_shift);
12430 %}
12431
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435
12436 // We can use ubfiz when masking by a positive number and then left shifting the result.
12437 // We know that the mask is positive because immI_bitmask guarantees it.
12438 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12439 %{
12440 match(Set dst (LShiftI (AndI src mask) lshift));
12441 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12442
12443 ins_cost(INSN_COST);
12444 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12445 ins_encode %{
12446 int lshift = $lshift$$constant & 31;
12447 intptr_t mask = $mask$$constant;
12448 int width = exact_log2(mask+1);
12449 __ ubfizw(as_Register($dst$$reg),
12450 as_Register($src$$reg), lshift, width);
12451 %}
12452 ins_pipe(ialu_reg_shift);
12453 %}
12454
12455 // This pattern is automatically generated from aarch64_ad.m4
12456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12457
12458 // We can use ubfiz when masking by a positive number and then left shifting the result.
12459 // We know that the mask is positive because immL_bitmask guarantees it.
12460 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12461 %{
12462 match(Set dst (LShiftL (AndL src mask) lshift));
12463 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12464
12465 ins_cost(INSN_COST);
12466 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12467 ins_encode %{
12468 int lshift = $lshift$$constant & 63;
12469 intptr_t mask = $mask$$constant;
12470 int width = exact_log2_long(mask+1);
12471 __ ubfiz(as_Register($dst$$reg),
12472 as_Register($src$$reg), lshift, width);
12473 %}
12474 ins_pipe(ialu_reg_shift);
12475 %}
12476
12477 // This pattern is automatically generated from aarch64_ad.m4
12478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12479
12480 // We can use ubfiz when masking by a positive number and then left shifting the result.
12481 // We know that the mask is positive because immI_bitmask guarantees it.
12482 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12483 %{
12484 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12485 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12486
12487 ins_cost(INSN_COST);
12488 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12489 ins_encode %{
12490 int lshift = $lshift$$constant & 31;
12491 intptr_t mask = $mask$$constant;
12492 int width = exact_log2(mask+1);
12493 __ ubfizw(as_Register($dst$$reg),
12494 as_Register($src$$reg), lshift, width);
12495 %}
12496 ins_pipe(ialu_reg_shift);
12497 %}
12498
12499 // This pattern is automatically generated from aarch64_ad.m4
12500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12501
12502 // We can use ubfiz when masking by a positive number and then left shifting the result.
12503 // We know that the mask is positive because immL_bitmask guarantees it.
12504 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12505 %{
12506 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12507 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12508
12509 ins_cost(INSN_COST);
12510 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12511 ins_encode %{
12512 int lshift = $lshift$$constant & 63;
12513 intptr_t mask = $mask$$constant;
12514 int width = exact_log2_long(mask+1);
12515 __ ubfiz(as_Register($dst$$reg),
12516 as_Register($src$$reg), lshift, width);
12517 %}
12518 ins_pipe(ialu_reg_shift);
12519 %}
12520
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524
12525 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12526 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12527 %{
12528 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12529 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12530
12531 ins_cost(INSN_COST);
12532 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12533 ins_encode %{
12534 int lshift = $lshift$$constant & 63;
12535 intptr_t mask = $mask$$constant;
12536 int width = exact_log2(mask+1);
12537 __ ubfiz(as_Register($dst$$reg),
12538 as_Register($src$$reg), lshift, width);
12539 %}
12540 ins_pipe(ialu_reg_shift);
12541 %}
12542
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545
12546 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12547 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12548 %{
12549 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12550 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12551
12552 ins_cost(INSN_COST);
12553 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12554 ins_encode %{
12555 int lshift = $lshift$$constant & 31;
12556 intptr_t mask = $mask$$constant;
12557 int width = exact_log2(mask+1);
12558 __ ubfiz(as_Register($dst$$reg),
12559 as_Register($src$$reg), lshift, width);
12560 %}
12561 ins_pipe(ialu_reg_shift);
12562 %}
12563
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566
12567 // Can skip int2long conversions after AND with small bitmask
12568 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12569 %{
12570 match(Set dst (ConvI2L (AndI src msk)));
12571 ins_cost(INSN_COST);
12572 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12573 ins_encode %{
12574 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12575 %}
12576 ins_pipe(ialu_reg_shift);
12577 %}
12578
12579
12580 // Rotations
12581
12582 // This pattern is automatically generated from aarch64_ad.m4
12583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12584 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12585 %{
12586 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12587 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12588
12589 ins_cost(INSN_COST);
12590 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12591
12592 ins_encode %{
12593 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12594 $rshift$$constant & 63);
12595 %}
12596 ins_pipe(ialu_reg_reg_extr);
12597 %}
12598
12599
12600 // This pattern is automatically generated from aarch64_ad.m4
12601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12602 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12603 %{
12604 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12605 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12606
12607 ins_cost(INSN_COST);
12608 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12609
12610 ins_encode %{
12611 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12612 $rshift$$constant & 31);
12613 %}
12614 ins_pipe(ialu_reg_reg_extr);
12615 %}
12616
12617
12618 // This pattern is automatically generated from aarch64_ad.m4
12619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12620 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12621 %{
12622 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12623 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12624
12625 ins_cost(INSN_COST);
12626 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12627
12628 ins_encode %{
12629 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12630 $rshift$$constant & 63);
12631 %}
12632 ins_pipe(ialu_reg_reg_extr);
12633 %}
12634
12635
12636 // This pattern is automatically generated from aarch64_ad.m4
12637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12638 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12639 %{
12640 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12641 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12642
12643 ins_cost(INSN_COST);
12644 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12645
12646 ins_encode %{
12647 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12648 $rshift$$constant & 31);
12649 %}
12650 ins_pipe(ialu_reg_reg_extr);
12651 %}
12652
12653 // This pattern is automatically generated from aarch64_ad.m4
12654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12655 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12656 %{
12657 match(Set dst (RotateRight src shift));
12658
12659 ins_cost(INSN_COST);
12660 format %{ "ror $dst, $src, $shift" %}
12661
12662 ins_encode %{
12663 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12664 $shift$$constant & 0x1f);
12665 %}
12666 ins_pipe(ialu_reg_reg_vshift);
12667 %}
12668
12669 // This pattern is automatically generated from aarch64_ad.m4
12670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12671 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12672 %{
12673 match(Set dst (RotateRight src shift));
12674
12675 ins_cost(INSN_COST);
12676 format %{ "ror $dst, $src, $shift" %}
12677
12678 ins_encode %{
12679 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12680 $shift$$constant & 0x3f);
12681 %}
12682 ins_pipe(ialu_reg_reg_vshift);
12683 %}
12684
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12688 %{
12689 match(Set dst (RotateRight src shift));
12690
12691 ins_cost(INSN_COST);
12692 format %{ "ror $dst, $src, $shift" %}
12693
12694 ins_encode %{
12695 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12696 %}
12697 ins_pipe(ialu_reg_reg_vshift);
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12703 %{
12704 match(Set dst (RotateRight src shift));
12705
12706 ins_cost(INSN_COST);
12707 format %{ "ror $dst, $src, $shift" %}
12708
12709 ins_encode %{
12710 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12711 %}
12712 ins_pipe(ialu_reg_reg_vshift);
12713 %}
12714
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12718 %{
12719 match(Set dst (RotateLeft src shift));
12720
12721 ins_cost(INSN_COST);
12722 format %{ "rol $dst, $src, $shift" %}
12723
12724 ins_encode %{
12725 __ subw(rscratch1, zr, as_Register($shift$$reg));
12726 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12727 %}
12728 ins_pipe(ialu_reg_reg_vshift);
12729 %}
12730
12731 // This pattern is automatically generated from aarch64_ad.m4
12732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12733 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12734 %{
12735 match(Set dst (RotateLeft src shift));
12736
12737 ins_cost(INSN_COST);
12738 format %{ "rol $dst, $src, $shift" %}
12739
12740 ins_encode %{
12741 __ subw(rscratch1, zr, as_Register($shift$$reg));
12742 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12743 %}
12744 ins_pipe(ialu_reg_reg_vshift);
12745 %}
12746
12747
12748 // Add/subtract (extended)
12749
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12753 %{
12754 match(Set dst (AddL src1 (ConvI2L src2)));
12755 ins_cost(INSN_COST);
12756 format %{ "add $dst, $src1, $src2, sxtw" %}
12757
12758 ins_encode %{
12759 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12760 as_Register($src2$$reg), ext::sxtw);
12761 %}
12762 ins_pipe(ialu_reg_reg);
12763 %}
12764
12765 // This pattern is automatically generated from aarch64_ad.m4
12766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12767 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12768 %{
12769 match(Set dst (SubL src1 (ConvI2L src2)));
12770 ins_cost(INSN_COST);
12771 format %{ "sub $dst, $src1, $src2, sxtw" %}
12772
12773 ins_encode %{
12774 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12775 as_Register($src2$$reg), ext::sxtw);
12776 %}
12777 ins_pipe(ialu_reg_reg);
12778 %}
12779
12780 // This pattern is automatically generated from aarch64_ad.m4
12781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12782 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12783 %{
12784 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12785 ins_cost(INSN_COST);
12786 format %{ "add $dst, $src1, $src2, sxth" %}
12787
12788 ins_encode %{
12789 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12790 as_Register($src2$$reg), ext::sxth);
12791 %}
12792 ins_pipe(ialu_reg_reg);
12793 %}
12794
12795 // This pattern is automatically generated from aarch64_ad.m4
12796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12797 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12798 %{
12799 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12800 ins_cost(INSN_COST);
12801 format %{ "add $dst, $src1, $src2, sxtb" %}
12802
12803 ins_encode %{
12804 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12805 as_Register($src2$$reg), ext::sxtb);
12806 %}
12807 ins_pipe(ialu_reg_reg);
12808 %}
12809
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12813 %{
12814 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12815 ins_cost(INSN_COST);
12816 format %{ "add $dst, $src1, $src2, uxtb" %}
12817
12818 ins_encode %{
12819 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12820 as_Register($src2$$reg), ext::uxtb);
12821 %}
12822 ins_pipe(ialu_reg_reg);
12823 %}
12824
12825 // This pattern is automatically generated from aarch64_ad.m4
12826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12827 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12828 %{
12829 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12830 ins_cost(INSN_COST);
12831 format %{ "add $dst, $src1, $src2, sxth" %}
12832
12833 ins_encode %{
12834 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12835 as_Register($src2$$reg), ext::sxth);
12836 %}
12837 ins_pipe(ialu_reg_reg);
12838 %}
12839
12840 // This pattern is automatically generated from aarch64_ad.m4
12841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12842 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12843 %{
12844 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12845 ins_cost(INSN_COST);
12846 format %{ "add $dst, $src1, $src2, sxtw" %}
12847
12848 ins_encode %{
12849 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12850 as_Register($src2$$reg), ext::sxtw);
12851 %}
12852 ins_pipe(ialu_reg_reg);
12853 %}
12854
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12858 %{
12859 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12860 ins_cost(INSN_COST);
12861 format %{ "add $dst, $src1, $src2, sxtb" %}
12862
12863 ins_encode %{
12864 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12865 as_Register($src2$$reg), ext::sxtb);
12866 %}
12867 ins_pipe(ialu_reg_reg);
12868 %}
12869
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12873 %{
12874 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12875 ins_cost(INSN_COST);
12876 format %{ "add $dst, $src1, $src2, uxtb" %}
12877
12878 ins_encode %{
12879 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12880 as_Register($src2$$reg), ext::uxtb);
12881 %}
12882 ins_pipe(ialu_reg_reg);
12883 %}
12884
12885 // This pattern is automatically generated from aarch64_ad.m4
12886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12887 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12888 %{
12889 match(Set dst (AddI src1 (AndI src2 mask)));
12890 ins_cost(INSN_COST);
12891 format %{ "addw $dst, $src1, $src2, uxtb" %}
12892
12893 ins_encode %{
12894 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12895 as_Register($src2$$reg), ext::uxtb);
12896 %}
12897 ins_pipe(ialu_reg_reg);
12898 %}
12899
12900 // This pattern is automatically generated from aarch64_ad.m4
12901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12902 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12903 %{
12904 match(Set dst (AddI src1 (AndI src2 mask)));
12905 ins_cost(INSN_COST);
12906 format %{ "addw $dst, $src1, $src2, uxth" %}
12907
12908 ins_encode %{
12909 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12910 as_Register($src2$$reg), ext::uxth);
12911 %}
12912 ins_pipe(ialu_reg_reg);
12913 %}
12914
12915 // This pattern is automatically generated from aarch64_ad.m4
12916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12917 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12918 %{
12919 match(Set dst (AddL src1 (AndL src2 mask)));
12920 ins_cost(INSN_COST);
12921 format %{ "add $dst, $src1, $src2, uxtb" %}
12922
12923 ins_encode %{
12924 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12925 as_Register($src2$$reg), ext::uxtb);
12926 %}
12927 ins_pipe(ialu_reg_reg);
12928 %}
12929
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12933 %{
12934 match(Set dst (AddL src1 (AndL src2 mask)));
12935 ins_cost(INSN_COST);
12936 format %{ "add $dst, $src1, $src2, uxth" %}
12937
12938 ins_encode %{
12939 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12940 as_Register($src2$$reg), ext::uxth);
12941 %}
12942 ins_pipe(ialu_reg_reg);
12943 %}
12944
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12948 %{
12949 match(Set dst (AddL src1 (AndL src2 mask)));
12950 ins_cost(INSN_COST);
12951 format %{ "add $dst, $src1, $src2, uxtw" %}
12952
12953 ins_encode %{
12954 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12955 as_Register($src2$$reg), ext::uxtw);
12956 %}
12957 ins_pipe(ialu_reg_reg);
12958 %}
12959
12960 // This pattern is automatically generated from aarch64_ad.m4
12961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12962 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12963 %{
12964 match(Set dst (SubI src1 (AndI src2 mask)));
12965 ins_cost(INSN_COST);
12966 format %{ "subw $dst, $src1, $src2, uxtb" %}
12967
12968 ins_encode %{
12969 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12970 as_Register($src2$$reg), ext::uxtb);
12971 %}
12972 ins_pipe(ialu_reg_reg);
12973 %}
12974
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12978 %{
12979 match(Set dst (SubI src1 (AndI src2 mask)));
12980 ins_cost(INSN_COST);
12981 format %{ "subw $dst, $src1, $src2, uxth" %}
12982
12983 ins_encode %{
12984 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12985 as_Register($src2$$reg), ext::uxth);
12986 %}
12987 ins_pipe(ialu_reg_reg);
12988 %}
12989
12990 // This pattern is automatically generated from aarch64_ad.m4
12991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12992 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12993 %{
12994 match(Set dst (SubL src1 (AndL src2 mask)));
12995 ins_cost(INSN_COST);
12996 format %{ "sub $dst, $src1, $src2, uxtb" %}
12997
12998 ins_encode %{
12999 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13000 as_Register($src2$$reg), ext::uxtb);
13001 %}
13002 ins_pipe(ialu_reg_reg);
13003 %}
13004
13005 // This pattern is automatically generated from aarch64_ad.m4
13006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13007 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13008 %{
13009 match(Set dst (SubL src1 (AndL src2 mask)));
13010 ins_cost(INSN_COST);
13011 format %{ "sub $dst, $src1, $src2, uxth" %}
13012
13013 ins_encode %{
13014 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13015 as_Register($src2$$reg), ext::uxth);
13016 %}
13017 ins_pipe(ialu_reg_reg);
13018 %}
13019
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13023 %{
13024 match(Set dst (SubL src1 (AndL src2 mask)));
13025 ins_cost(INSN_COST);
13026 format %{ "sub $dst, $src1, $src2, uxtw" %}
13027
13028 ins_encode %{
13029 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13030 as_Register($src2$$reg), ext::uxtw);
13031 %}
13032 ins_pipe(ialu_reg_reg);
13033 %}
13034
13035
13036 // This pattern is automatically generated from aarch64_ad.m4
13037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13038 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13039 %{
13040 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13041 ins_cost(1.9 * INSN_COST);
13042 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13043
13044 ins_encode %{
13045 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13046 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13047 %}
13048 ins_pipe(ialu_reg_reg_shift);
13049 %}
13050
13051 // This pattern is automatically generated from aarch64_ad.m4
13052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13053 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13054 %{
13055 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13056 ins_cost(1.9 * INSN_COST);
13057 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13058
13059 ins_encode %{
13060 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13061 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13062 %}
13063 ins_pipe(ialu_reg_reg_shift);
13064 %}
13065
13066 // This pattern is automatically generated from aarch64_ad.m4
13067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13068 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13069 %{
13070 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13071 ins_cost(1.9 * INSN_COST);
13072 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13073
13074 ins_encode %{
13075 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13076 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13077 %}
13078 ins_pipe(ialu_reg_reg_shift);
13079 %}
13080
13081 // This pattern is automatically generated from aarch64_ad.m4
13082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13083 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13084 %{
13085 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13086 ins_cost(1.9 * INSN_COST);
13087 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13088
13089 ins_encode %{
13090 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13091 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13092 %}
13093 ins_pipe(ialu_reg_reg_shift);
13094 %}
13095
13096 // This pattern is automatically generated from aarch64_ad.m4
13097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13098 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13099 %{
13100 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13101 ins_cost(1.9 * INSN_COST);
13102 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13103
13104 ins_encode %{
13105 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13106 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13107 %}
13108 ins_pipe(ialu_reg_reg_shift);
13109 %}
13110
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13114 %{
13115 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13116 ins_cost(1.9 * INSN_COST);
13117 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13118
13119 ins_encode %{
13120 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13121 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13122 %}
13123 ins_pipe(ialu_reg_reg_shift);
13124 %}
13125
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13129 %{
13130 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13131 ins_cost(1.9 * INSN_COST);
13132 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13133
13134 ins_encode %{
13135 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13136 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13137 %}
13138 ins_pipe(ialu_reg_reg_shift);
13139 %}
13140
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13144 %{
13145 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13146 ins_cost(1.9 * INSN_COST);
13147 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13148
13149 ins_encode %{
13150 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13151 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13152 %}
13153 ins_pipe(ialu_reg_reg_shift);
13154 %}
13155
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13159 %{
13160 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13161 ins_cost(1.9 * INSN_COST);
13162 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13163
13164 ins_encode %{
13165 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13166 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13167 %}
13168 ins_pipe(ialu_reg_reg_shift);
13169 %}
13170
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13174 %{
13175 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13176 ins_cost(1.9 * INSN_COST);
13177 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13178
13179 ins_encode %{
13180 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13181 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13182 %}
13183 ins_pipe(ialu_reg_reg_shift);
13184 %}
13185
13186 // This pattern is automatically generated from aarch64_ad.m4
13187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13188 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13189 %{
13190 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13191 ins_cost(1.9 * INSN_COST);
13192 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13193
13194 ins_encode %{
13195 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13196 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13197 %}
13198 ins_pipe(ialu_reg_reg_shift);
13199 %}
13200
13201 // This pattern is automatically generated from aarch64_ad.m4
13202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13203 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13204 %{
13205 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13206 ins_cost(1.9 * INSN_COST);
13207 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13208
13209 ins_encode %{
13210 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13211 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13212 %}
13213 ins_pipe(ialu_reg_reg_shift);
13214 %}
13215
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13219 %{
13220 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13221 ins_cost(1.9 * INSN_COST);
13222 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13223
13224 ins_encode %{
13225 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13226 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13227 %}
13228 ins_pipe(ialu_reg_reg_shift);
13229 %}
13230
13231 // This pattern is automatically generated from aarch64_ad.m4
13232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13233 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13234 %{
13235 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13236 ins_cost(1.9 * INSN_COST);
13237 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13238
13239 ins_encode %{
13240 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13241 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13242 %}
13243 ins_pipe(ialu_reg_reg_shift);
13244 %}
13245
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13249 %{
13250 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13251 ins_cost(1.9 * INSN_COST);
13252 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13253
13254 ins_encode %{
13255 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13256 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13257 %}
13258 ins_pipe(ialu_reg_reg_shift);
13259 %}
13260
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13264 %{
13265 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13266 ins_cost(1.9 * INSN_COST);
13267 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13268
13269 ins_encode %{
13270 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13271 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13272 %}
13273 ins_pipe(ialu_reg_reg_shift);
13274 %}
13275
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13279 %{
13280 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13281 ins_cost(1.9 * INSN_COST);
13282 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13283
13284 ins_encode %{
13285 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13286 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13287 %}
13288 ins_pipe(ialu_reg_reg_shift);
13289 %}
13290
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13294 %{
13295 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13296 ins_cost(1.9 * INSN_COST);
13297 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13298
13299 ins_encode %{
13300 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13301 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13302 %}
13303 ins_pipe(ialu_reg_reg_shift);
13304 %}
13305
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13309 %{
13310 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13311 ins_cost(1.9 * INSN_COST);
13312 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13313
13314 ins_encode %{
13315 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13316 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13317 %}
13318 ins_pipe(ialu_reg_reg_shift);
13319 %}
13320
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13324 %{
13325 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13326 ins_cost(1.9 * INSN_COST);
13327 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13328
13329 ins_encode %{
13330 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13331 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13332 %}
13333 ins_pipe(ialu_reg_reg_shift);
13334 %}
13335
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13339 %{
13340 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13341 ins_cost(1.9 * INSN_COST);
13342 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13343
13344 ins_encode %{
13345 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13346 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13347 %}
13348 ins_pipe(ialu_reg_reg_shift);
13349 %}
13350
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13354 %{
13355 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13356 ins_cost(1.9 * INSN_COST);
13357 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13358
13359 ins_encode %{
13360 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13361 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13362 %}
13363 ins_pipe(ialu_reg_reg_shift);
13364 %}
13365
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13369 %{
13370 effect(DEF dst, USE src1, USE src2, USE cr);
13371 ins_cost(INSN_COST * 2);
13372 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13373
13374 ins_encode %{
13375 __ cselw($dst$$Register,
13376 $src1$$Register,
13377 $src2$$Register,
13378 Assembler::LT);
13379 %}
13380 ins_pipe(icond_reg_reg);
13381 %}
13382
13383 // This pattern is automatically generated from aarch64_ad.m4
13384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13385 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13386 %{
13387 effect(DEF dst, USE src1, USE src2, USE cr);
13388 ins_cost(INSN_COST * 2);
13389 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13390
13391 ins_encode %{
13392 __ cselw($dst$$Register,
13393 $src1$$Register,
13394 $src2$$Register,
13395 Assembler::GT);
13396 %}
13397 ins_pipe(icond_reg_reg);
13398 %}
13399
13400 // This pattern is automatically generated from aarch64_ad.m4
13401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13402 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13403 %{
13404 effect(DEF dst, USE src1, USE cr);
13405 ins_cost(INSN_COST * 2);
13406 format %{ "cselw $dst, $src1, zr lt\t" %}
13407
13408 ins_encode %{
13409 __ cselw($dst$$Register,
13410 $src1$$Register,
13411 zr,
13412 Assembler::LT);
13413 %}
13414 ins_pipe(icond_reg);
13415 %}
13416
13417 // This pattern is automatically generated from aarch64_ad.m4
13418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13419 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13420 %{
13421 effect(DEF dst, USE src1, USE cr);
13422 ins_cost(INSN_COST * 2);
13423 format %{ "cselw $dst, $src1, zr gt\t" %}
13424
13425 ins_encode %{
13426 __ cselw($dst$$Register,
13427 $src1$$Register,
13428 zr,
13429 Assembler::GT);
13430 %}
13431 ins_pipe(icond_reg);
13432 %}
13433
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13437 %{
13438 effect(DEF dst, USE src1, USE cr);
13439 ins_cost(INSN_COST * 2);
13440 format %{ "csincw $dst, $src1, zr le\t" %}
13441
13442 ins_encode %{
13443 __ csincw($dst$$Register,
13444 $src1$$Register,
13445 zr,
13446 Assembler::LE);
13447 %}
13448 ins_pipe(icond_reg);
13449 %}
13450
13451 // This pattern is automatically generated from aarch64_ad.m4
13452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13453 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13454 %{
13455 effect(DEF dst, USE src1, USE cr);
13456 ins_cost(INSN_COST * 2);
13457 format %{ "csincw $dst, $src1, zr gt\t" %}
13458
13459 ins_encode %{
13460 __ csincw($dst$$Register,
13461 $src1$$Register,
13462 zr,
13463 Assembler::GT);
13464 %}
13465 ins_pipe(icond_reg);
13466 %}
13467
13468 // This pattern is automatically generated from aarch64_ad.m4
13469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13470 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13471 %{
13472 effect(DEF dst, USE src1, USE cr);
13473 ins_cost(INSN_COST * 2);
13474 format %{ "csinvw $dst, $src1, zr lt\t" %}
13475
13476 ins_encode %{
13477 __ csinvw($dst$$Register,
13478 $src1$$Register,
13479 zr,
13480 Assembler::LT);
13481 %}
13482 ins_pipe(icond_reg);
13483 %}
13484
13485 // This pattern is automatically generated from aarch64_ad.m4
13486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13487 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13488 %{
13489 effect(DEF dst, USE src1, USE cr);
13490 ins_cost(INSN_COST * 2);
13491 format %{ "csinvw $dst, $src1, zr ge\t" %}
13492
13493 ins_encode %{
13494 __ csinvw($dst$$Register,
13495 $src1$$Register,
13496 zr,
13497 Assembler::GE);
13498 %}
13499 ins_pipe(icond_reg);
13500 %}
13501
13502 // This pattern is automatically generated from aarch64_ad.m4
13503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13504 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13505 %{
13506 match(Set dst (MinI src imm));
13507 ins_cost(INSN_COST * 3);
13508 expand %{
13509 rFlagsReg cr;
13510 compI_reg_imm0(cr, src);
13511 cmovI_reg_imm0_lt(dst, src, cr);
13512 %}
13513 %}
13514
13515 // This pattern is automatically generated from aarch64_ad.m4
13516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13517 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13518 %{
13519 match(Set dst (MinI imm src));
13520 ins_cost(INSN_COST * 3);
13521 expand %{
13522 rFlagsReg cr;
13523 compI_reg_imm0(cr, src);
13524 cmovI_reg_imm0_lt(dst, src, cr);
13525 %}
13526 %}
13527
13528 // This pattern is automatically generated from aarch64_ad.m4
13529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13530 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13531 %{
13532 match(Set dst (MinI src imm));
13533 ins_cost(INSN_COST * 3);
13534 expand %{
13535 rFlagsReg cr;
13536 compI_reg_imm0(cr, src);
13537 cmovI_reg_imm1_le(dst, src, cr);
13538 %}
13539 %}
13540
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13544 %{
13545 match(Set dst (MinI imm src));
13546 ins_cost(INSN_COST * 3);
13547 expand %{
13548 rFlagsReg cr;
13549 compI_reg_imm0(cr, src);
13550 cmovI_reg_imm1_le(dst, src, cr);
13551 %}
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13557 %{
13558 match(Set dst (MinI src imm));
13559 ins_cost(INSN_COST * 3);
13560 expand %{
13561 rFlagsReg cr;
13562 compI_reg_imm0(cr, src);
13563 cmovI_reg_immM1_lt(dst, src, cr);
13564 %}
13565 %}
13566
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13570 %{
13571 match(Set dst (MinI imm src));
13572 ins_cost(INSN_COST * 3);
13573 expand %{
13574 rFlagsReg cr;
13575 compI_reg_imm0(cr, src);
13576 cmovI_reg_immM1_lt(dst, src, cr);
13577 %}
13578 %}
13579
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13583 %{
13584 match(Set dst (MaxI src imm));
13585 ins_cost(INSN_COST * 3);
13586 expand %{
13587 rFlagsReg cr;
13588 compI_reg_imm0(cr, src);
13589 cmovI_reg_imm0_gt(dst, src, cr);
13590 %}
13591 %}
13592
13593 // This pattern is automatically generated from aarch64_ad.m4
13594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13595 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13596 %{
13597 match(Set dst (MaxI imm src));
13598 ins_cost(INSN_COST * 3);
13599 expand %{
13600 rFlagsReg cr;
13601 compI_reg_imm0(cr, src);
13602 cmovI_reg_imm0_gt(dst, src, cr);
13603 %}
13604 %}
13605
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13609 %{
13610 match(Set dst (MaxI src imm));
13611 ins_cost(INSN_COST * 3);
13612 expand %{
13613 rFlagsReg cr;
13614 compI_reg_imm0(cr, src);
13615 cmovI_reg_imm1_gt(dst, src, cr);
13616 %}
13617 %}
13618
13619 // This pattern is automatically generated from aarch64_ad.m4
13620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13621 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13622 %{
13623 match(Set dst (MaxI imm src));
13624 ins_cost(INSN_COST * 3);
13625 expand %{
13626 rFlagsReg cr;
13627 compI_reg_imm0(cr, src);
13628 cmovI_reg_imm1_gt(dst, src, cr);
13629 %}
13630 %}
13631
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13635 %{
13636 match(Set dst (MaxI src imm));
13637 ins_cost(INSN_COST * 3);
13638 expand %{
13639 rFlagsReg cr;
13640 compI_reg_imm0(cr, src);
13641 cmovI_reg_immM1_ge(dst, src, cr);
13642 %}
13643 %}
13644
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13648 %{
13649 match(Set dst (MaxI imm src));
13650 ins_cost(INSN_COST * 3);
13651 expand %{
13652 rFlagsReg cr;
13653 compI_reg_imm0(cr, src);
13654 cmovI_reg_immM1_ge(dst, src, cr);
13655 %}
13656 %}
13657
13658 // This pattern is automatically generated from aarch64_ad.m4
13659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13660 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13661 %{
13662 match(Set dst (ReverseI src));
13663 ins_cost(INSN_COST);
13664 format %{ "rbitw $dst, $src" %}
13665 ins_encode %{
13666 __ rbitw($dst$$Register, $src$$Register);
13667 %}
13668 ins_pipe(ialu_reg);
13669 %}
13670
13671 // This pattern is automatically generated from aarch64_ad.m4
13672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13673 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13674 %{
13675 match(Set dst (ReverseL src));
13676 ins_cost(INSN_COST);
13677 format %{ "rbit $dst, $src" %}
13678 ins_encode %{
13679 __ rbit($dst$$Register, $src$$Register);
13680 %}
13681 ins_pipe(ialu_reg);
13682 %}
13683
13684
13685 // END This section of the file is automatically generated. Do not edit --------------
13686
13687
13688 // ============================================================================
13689 // Floating Point Arithmetic Instructions
13690
13691 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13692 match(Set dst (AddHF src1 src2));
13693 format %{ "faddh $dst, $src1, $src2" %}
13694 ins_encode %{
13695 __ faddh($dst$$FloatRegister,
13696 $src1$$FloatRegister,
13697 $src2$$FloatRegister);
13698 %}
13699 ins_pipe(fp_dop_reg_reg_s);
13700 %}
13701
13702 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13703 match(Set dst (AddF src1 src2));
13704
13705 ins_cost(INSN_COST * 5);
13706 format %{ "fadds $dst, $src1, $src2" %}
13707
13708 ins_encode %{
13709 __ fadds(as_FloatRegister($dst$$reg),
13710 as_FloatRegister($src1$$reg),
13711 as_FloatRegister($src2$$reg));
13712 %}
13713
13714 ins_pipe(fp_dop_reg_reg_s);
13715 %}
13716
13717 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13718 match(Set dst (AddD src1 src2));
13719
13720 ins_cost(INSN_COST * 5);
13721 format %{ "faddd $dst, $src1, $src2" %}
13722
13723 ins_encode %{
13724 __ faddd(as_FloatRegister($dst$$reg),
13725 as_FloatRegister($src1$$reg),
13726 as_FloatRegister($src2$$reg));
13727 %}
13728
13729 ins_pipe(fp_dop_reg_reg_d);
13730 %}
13731
13732 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13733 match(Set dst (SubHF src1 src2));
13734 format %{ "fsubh $dst, $src1, $src2" %}
13735 ins_encode %{
13736 __ fsubh($dst$$FloatRegister,
13737 $src1$$FloatRegister,
13738 $src2$$FloatRegister);
13739 %}
13740 ins_pipe(fp_dop_reg_reg_s);
13741 %}
13742
13743 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13744 match(Set dst (SubF src1 src2));
13745
13746 ins_cost(INSN_COST * 5);
13747 format %{ "fsubs $dst, $src1, $src2" %}
13748
13749 ins_encode %{
13750 __ fsubs(as_FloatRegister($dst$$reg),
13751 as_FloatRegister($src1$$reg),
13752 as_FloatRegister($src2$$reg));
13753 %}
13754
13755 ins_pipe(fp_dop_reg_reg_s);
13756 %}
13757
13758 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13759 match(Set dst (SubD src1 src2));
13760
13761 ins_cost(INSN_COST * 5);
13762 format %{ "fsubd $dst, $src1, $src2" %}
13763
13764 ins_encode %{
13765 __ fsubd(as_FloatRegister($dst$$reg),
13766 as_FloatRegister($src1$$reg),
13767 as_FloatRegister($src2$$reg));
13768 %}
13769
13770 ins_pipe(fp_dop_reg_reg_d);
13771 %}
13772
13773 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13774 match(Set dst (MulHF src1 src2));
13775 format %{ "fmulh $dst, $src1, $src2" %}
13776 ins_encode %{
13777 __ fmulh($dst$$FloatRegister,
13778 $src1$$FloatRegister,
13779 $src2$$FloatRegister);
13780 %}
13781 ins_pipe(fp_dop_reg_reg_s);
13782 %}
13783
13784 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13785 match(Set dst (MulF src1 src2));
13786
13787 ins_cost(INSN_COST * 6);
13788 format %{ "fmuls $dst, $src1, $src2" %}
13789
13790 ins_encode %{
13791 __ fmuls(as_FloatRegister($dst$$reg),
13792 as_FloatRegister($src1$$reg),
13793 as_FloatRegister($src2$$reg));
13794 %}
13795
13796 ins_pipe(fp_dop_reg_reg_s);
13797 %}
13798
13799 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13800 match(Set dst (MulD src1 src2));
13801
13802 ins_cost(INSN_COST * 6);
13803 format %{ "fmuld $dst, $src1, $src2" %}
13804
13805 ins_encode %{
13806 __ fmuld(as_FloatRegister($dst$$reg),
13807 as_FloatRegister($src1$$reg),
13808 as_FloatRegister($src2$$reg));
13809 %}
13810
13811 ins_pipe(fp_dop_reg_reg_d);
13812 %}
13813
13814 // src1 * src2 + src3 (half-precision float)
13815 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13816 match(Set dst (FmaHF src3 (Binary src1 src2)));
13817 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13818 ins_encode %{
13819 assert(UseFMA, "Needs FMA instructions support.");
13820 __ fmaddh($dst$$FloatRegister,
13821 $src1$$FloatRegister,
13822 $src2$$FloatRegister,
13823 $src3$$FloatRegister);
13824 %}
13825 ins_pipe(pipe_class_default);
13826 %}
13827
13828 // src1 * src2 + src3
13829 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13830 match(Set dst (FmaF src3 (Binary src1 src2)));
13831
13832 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13833
13834 ins_encode %{
13835 assert(UseFMA, "Needs FMA instructions support.");
13836 __ fmadds(as_FloatRegister($dst$$reg),
13837 as_FloatRegister($src1$$reg),
13838 as_FloatRegister($src2$$reg),
13839 as_FloatRegister($src3$$reg));
13840 %}
13841
13842 ins_pipe(pipe_class_default);
13843 %}
13844
13845 // src1 * src2 + src3
13846 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13847 match(Set dst (FmaD src3 (Binary src1 src2)));
13848
13849 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13850
13851 ins_encode %{
13852 assert(UseFMA, "Needs FMA instructions support.");
13853 __ fmaddd(as_FloatRegister($dst$$reg),
13854 as_FloatRegister($src1$$reg),
13855 as_FloatRegister($src2$$reg),
13856 as_FloatRegister($src3$$reg));
13857 %}
13858
13859 ins_pipe(pipe_class_default);
13860 %}
13861
13862 // src1 * (-src2) + src3
13863 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13864 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13865 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13866
13867 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13868
13869 ins_encode %{
13870 assert(UseFMA, "Needs FMA instructions support.");
13871 __ fmsubs(as_FloatRegister($dst$$reg),
13872 as_FloatRegister($src1$$reg),
13873 as_FloatRegister($src2$$reg),
13874 as_FloatRegister($src3$$reg));
13875 %}
13876
13877 ins_pipe(pipe_class_default);
13878 %}
13879
13880 // src1 * (-src2) + src3
13881 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13882 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13883 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13884
13885 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13886
13887 ins_encode %{
13888 assert(UseFMA, "Needs FMA instructions support.");
13889 __ fmsubd(as_FloatRegister($dst$$reg),
13890 as_FloatRegister($src1$$reg),
13891 as_FloatRegister($src2$$reg),
13892 as_FloatRegister($src3$$reg));
13893 %}
13894
13895 ins_pipe(pipe_class_default);
13896 %}
13897
13898 // src1 * (-src2) - src3
13899 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13900 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13901 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13902
13903 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13904
13905 ins_encode %{
13906 assert(UseFMA, "Needs FMA instructions support.");
13907 __ fnmadds(as_FloatRegister($dst$$reg),
13908 as_FloatRegister($src1$$reg),
13909 as_FloatRegister($src2$$reg),
13910 as_FloatRegister($src3$$reg));
13911 %}
13912
13913 ins_pipe(pipe_class_default);
13914 %}
13915
13916 // src1 * (-src2) - src3
13917 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13918 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13919 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13920
13921 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13922
13923 ins_encode %{
13924 assert(UseFMA, "Needs FMA instructions support.");
13925 __ fnmaddd(as_FloatRegister($dst$$reg),
13926 as_FloatRegister($src1$$reg),
13927 as_FloatRegister($src2$$reg),
13928 as_FloatRegister($src3$$reg));
13929 %}
13930
13931 ins_pipe(pipe_class_default);
13932 %}
13933
13934 // src1 * src2 - src3
13935 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13936 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13937
13938 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13939
13940 ins_encode %{
13941 assert(UseFMA, "Needs FMA instructions support.");
13942 __ fnmsubs(as_FloatRegister($dst$$reg),
13943 as_FloatRegister($src1$$reg),
13944 as_FloatRegister($src2$$reg),
13945 as_FloatRegister($src3$$reg));
13946 %}
13947
13948 ins_pipe(pipe_class_default);
13949 %}
13950
13951 // src1 * src2 - src3
13952 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13953 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13954
13955 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13956
13957 ins_encode %{
13958 assert(UseFMA, "Needs FMA instructions support.");
13959 // n.b. insn name should be fnmsubd
13960 __ fnmsub(as_FloatRegister($dst$$reg),
13961 as_FloatRegister($src1$$reg),
13962 as_FloatRegister($src2$$reg),
13963 as_FloatRegister($src3$$reg));
13964 %}
13965
13966 ins_pipe(pipe_class_default);
13967 %}
13968
13969 // Math.max(HH)H (half-precision float)
13970 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13971 match(Set dst (MaxHF src1 src2));
13972 format %{ "fmaxh $dst, $src1, $src2" %}
13973 ins_encode %{
13974 __ fmaxh($dst$$FloatRegister,
13975 $src1$$FloatRegister,
13976 $src2$$FloatRegister);
13977 %}
13978 ins_pipe(fp_dop_reg_reg_s);
13979 %}
13980
13981 // Math.min(HH)H (half-precision float)
13982 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13983 match(Set dst (MinHF src1 src2));
13984 format %{ "fminh $dst, $src1, $src2" %}
13985 ins_encode %{
13986 __ fminh($dst$$FloatRegister,
13987 $src1$$FloatRegister,
13988 $src2$$FloatRegister);
13989 %}
13990 ins_pipe(fp_dop_reg_reg_s);
13991 %}
13992
13993 // Math.max(FF)F
13994 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13995 match(Set dst (MaxF src1 src2));
13996
13997 format %{ "fmaxs $dst, $src1, $src2" %}
13998 ins_encode %{
13999 __ fmaxs(as_FloatRegister($dst$$reg),
14000 as_FloatRegister($src1$$reg),
14001 as_FloatRegister($src2$$reg));
14002 %}
14003
14004 ins_pipe(fp_dop_reg_reg_s);
14005 %}
14006
14007 // Math.min(FF)F
14008 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14009 match(Set dst (MinF src1 src2));
14010
14011 format %{ "fmins $dst, $src1, $src2" %}
14012 ins_encode %{
14013 __ fmins(as_FloatRegister($dst$$reg),
14014 as_FloatRegister($src1$$reg),
14015 as_FloatRegister($src2$$reg));
14016 %}
14017
14018 ins_pipe(fp_dop_reg_reg_s);
14019 %}
14020
14021 // Math.max(DD)D
14022 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14023 match(Set dst (MaxD src1 src2));
14024
14025 format %{ "fmaxd $dst, $src1, $src2" %}
14026 ins_encode %{
14027 __ fmaxd(as_FloatRegister($dst$$reg),
14028 as_FloatRegister($src1$$reg),
14029 as_FloatRegister($src2$$reg));
14030 %}
14031
14032 ins_pipe(fp_dop_reg_reg_d);
14033 %}
14034
14035 // Math.min(DD)D
14036 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14037 match(Set dst (MinD src1 src2));
14038
14039 format %{ "fmind $dst, $src1, $src2" %}
14040 ins_encode %{
14041 __ fmind(as_FloatRegister($dst$$reg),
14042 as_FloatRegister($src1$$reg),
14043 as_FloatRegister($src2$$reg));
14044 %}
14045
14046 ins_pipe(fp_dop_reg_reg_d);
14047 %}
14048
14049 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14050 match(Set dst (DivHF src1 src2));
14051 format %{ "fdivh $dst, $src1, $src2" %}
14052 ins_encode %{
14053 __ fdivh($dst$$FloatRegister,
14054 $src1$$FloatRegister,
14055 $src2$$FloatRegister);
14056 %}
14057 ins_pipe(fp_div_s);
14058 %}
14059
14060 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14061 match(Set dst (DivF src1 src2));
14062
14063 ins_cost(INSN_COST * 18);
14064 format %{ "fdivs $dst, $src1, $src2" %}
14065
14066 ins_encode %{
14067 __ fdivs(as_FloatRegister($dst$$reg),
14068 as_FloatRegister($src1$$reg),
14069 as_FloatRegister($src2$$reg));
14070 %}
14071
14072 ins_pipe(fp_div_s);
14073 %}
14074
14075 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14076 match(Set dst (DivD src1 src2));
14077
14078 ins_cost(INSN_COST * 32);
14079 format %{ "fdivd $dst, $src1, $src2" %}
14080
14081 ins_encode %{
14082 __ fdivd(as_FloatRegister($dst$$reg),
14083 as_FloatRegister($src1$$reg),
14084 as_FloatRegister($src2$$reg));
14085 %}
14086
14087 ins_pipe(fp_div_d);
14088 %}
14089
14090 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14091 match(Set dst (NegF src));
14092
14093 ins_cost(INSN_COST * 3);
14094 format %{ "fneg $dst, $src" %}
14095
14096 ins_encode %{
14097 __ fnegs(as_FloatRegister($dst$$reg),
14098 as_FloatRegister($src$$reg));
14099 %}
14100
14101 ins_pipe(fp_uop_s);
14102 %}
14103
14104 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14105 match(Set dst (NegD src));
14106
14107 ins_cost(INSN_COST * 3);
14108 format %{ "fnegd $dst, $src" %}
14109
14110 ins_encode %{
14111 __ fnegd(as_FloatRegister($dst$$reg),
14112 as_FloatRegister($src$$reg));
14113 %}
14114
14115 ins_pipe(fp_uop_d);
14116 %}
14117
14118 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14119 %{
14120 match(Set dst (AbsI src));
14121
14122 effect(KILL cr);
14123 ins_cost(INSN_COST * 2);
14124 format %{ "cmpw $src, zr\n\t"
14125 "cnegw $dst, $src, Assembler::LT\t# int abs"
14126 %}
14127
14128 ins_encode %{
14129 __ cmpw(as_Register($src$$reg), zr);
14130 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14131 %}
14132 ins_pipe(pipe_class_default);
14133 %}
14134
14135 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14136 %{
14137 match(Set dst (AbsL src));
14138
14139 effect(KILL cr);
14140 ins_cost(INSN_COST * 2);
14141 format %{ "cmp $src, zr\n\t"
14142 "cneg $dst, $src, Assembler::LT\t# long abs"
14143 %}
14144
14145 ins_encode %{
14146 __ cmp(as_Register($src$$reg), zr);
14147 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14148 %}
14149 ins_pipe(pipe_class_default);
14150 %}
14151
14152 instruct absF_reg(vRegF dst, vRegF src) %{
14153 match(Set dst (AbsF src));
14154
14155 ins_cost(INSN_COST * 3);
14156 format %{ "fabss $dst, $src" %}
14157 ins_encode %{
14158 __ fabss(as_FloatRegister($dst$$reg),
14159 as_FloatRegister($src$$reg));
14160 %}
14161
14162 ins_pipe(fp_uop_s);
14163 %}
14164
14165 instruct absD_reg(vRegD dst, vRegD src) %{
14166 match(Set dst (AbsD src));
14167
14168 ins_cost(INSN_COST * 3);
14169 format %{ "fabsd $dst, $src" %}
14170 ins_encode %{
14171 __ fabsd(as_FloatRegister($dst$$reg),
14172 as_FloatRegister($src$$reg));
14173 %}
14174
14175 ins_pipe(fp_uop_d);
14176 %}
14177
14178 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14179 match(Set dst (AbsF (SubF src1 src2)));
14180
14181 ins_cost(INSN_COST * 3);
14182 format %{ "fabds $dst, $src1, $src2" %}
14183 ins_encode %{
14184 __ fabds(as_FloatRegister($dst$$reg),
14185 as_FloatRegister($src1$$reg),
14186 as_FloatRegister($src2$$reg));
14187 %}
14188
14189 ins_pipe(fp_uop_s);
14190 %}
14191
14192 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14193 match(Set dst (AbsD (SubD src1 src2)));
14194
14195 ins_cost(INSN_COST * 3);
14196 format %{ "fabdd $dst, $src1, $src2" %}
14197 ins_encode %{
14198 __ fabdd(as_FloatRegister($dst$$reg),
14199 as_FloatRegister($src1$$reg),
14200 as_FloatRegister($src2$$reg));
14201 %}
14202
14203 ins_pipe(fp_uop_d);
14204 %}
14205
14206 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14207 match(Set dst (SqrtD src));
14208
14209 ins_cost(INSN_COST * 50);
14210 format %{ "fsqrtd $dst, $src" %}
14211 ins_encode %{
14212 __ fsqrtd(as_FloatRegister($dst$$reg),
14213 as_FloatRegister($src$$reg));
14214 %}
14215
14216 ins_pipe(fp_div_s);
14217 %}
14218
14219 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14220 match(Set dst (SqrtF src));
14221
14222 ins_cost(INSN_COST * 50);
14223 format %{ "fsqrts $dst, $src" %}
14224 ins_encode %{
14225 __ fsqrts(as_FloatRegister($dst$$reg),
14226 as_FloatRegister($src$$reg));
14227 %}
14228
14229 ins_pipe(fp_div_d);
14230 %}
14231
14232 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14233 match(Set dst (SqrtHF src));
14234 format %{ "fsqrth $dst, $src" %}
14235 ins_encode %{
14236 __ fsqrth($dst$$FloatRegister,
14237 $src$$FloatRegister);
14238 %}
14239 ins_pipe(fp_div_s);
14240 %}
14241
14242 // Math.rint, floor, ceil
14243 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14244 match(Set dst (RoundDoubleMode src rmode));
14245 format %{ "frint $dst, $src, $rmode" %}
14246 ins_encode %{
14247 switch ($rmode$$constant) {
14248 case RoundDoubleModeNode::rmode_rint:
14249 __ frintnd(as_FloatRegister($dst$$reg),
14250 as_FloatRegister($src$$reg));
14251 break;
14252 case RoundDoubleModeNode::rmode_floor:
14253 __ frintmd(as_FloatRegister($dst$$reg),
14254 as_FloatRegister($src$$reg));
14255 break;
14256 case RoundDoubleModeNode::rmode_ceil:
14257 __ frintpd(as_FloatRegister($dst$$reg),
14258 as_FloatRegister($src$$reg));
14259 break;
14260 }
14261 %}
14262 ins_pipe(fp_uop_d);
14263 %}
14264
14265 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14266 match(Set dst (CopySignD src1 (Binary src2 zero)));
14267 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14268 format %{ "CopySignD $dst $src1 $src2" %}
14269 ins_encode %{
14270 FloatRegister dst = as_FloatRegister($dst$$reg),
14271 src1 = as_FloatRegister($src1$$reg),
14272 src2 = as_FloatRegister($src2$$reg),
14273 zero = as_FloatRegister($zero$$reg);
14274 __ fnegd(dst, zero);
14275 __ bsl(dst, __ T8B, src2, src1);
14276 %}
14277 ins_pipe(fp_uop_d);
14278 %}
14279
14280 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14281 match(Set dst (CopySignF src1 src2));
14282 effect(TEMP_DEF dst, USE src1, USE src2);
14283 format %{ "CopySignF $dst $src1 $src2" %}
14284 ins_encode %{
14285 FloatRegister dst = as_FloatRegister($dst$$reg),
14286 src1 = as_FloatRegister($src1$$reg),
14287 src2 = as_FloatRegister($src2$$reg);
14288 __ movi(dst, __ T2S, 0x80, 24);
14289 __ bsl(dst, __ T8B, src2, src1);
14290 %}
14291 ins_pipe(fp_uop_d);
14292 %}
14293
14294 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14295 match(Set dst (SignumD src (Binary zero one)));
14296 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14297 format %{ "signumD $dst, $src" %}
14298 ins_encode %{
14299 FloatRegister src = as_FloatRegister($src$$reg),
14300 dst = as_FloatRegister($dst$$reg),
14301 zero = as_FloatRegister($zero$$reg),
14302 one = as_FloatRegister($one$$reg);
14303 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14304 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14305 // Bit selection instruction gets bit from "one" for each enabled bit in
14306 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14307 // NaN the whole "src" will be copied because "dst" is zero. For all other
14308 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14309 // from "src", and all other bits are copied from 1.0.
14310 __ bsl(dst, __ T8B, one, src);
14311 %}
14312 ins_pipe(fp_uop_d);
14313 %}
14314
14315 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14316 match(Set dst (SignumF src (Binary zero one)));
14317 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14318 format %{ "signumF $dst, $src" %}
14319 ins_encode %{
14320 FloatRegister src = as_FloatRegister($src$$reg),
14321 dst = as_FloatRegister($dst$$reg),
14322 zero = as_FloatRegister($zero$$reg),
14323 one = as_FloatRegister($one$$reg);
14324 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14325 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14326 // Bit selection instruction gets bit from "one" for each enabled bit in
14327 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14328 // NaN the whole "src" will be copied because "dst" is zero. For all other
14329 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14330 // from "src", and all other bits are copied from 1.0.
14331 __ bsl(dst, __ T8B, one, src);
14332 %}
14333 ins_pipe(fp_uop_d);
14334 %}
14335
14336 instruct onspinwait() %{
14337 match(OnSpinWait);
14338 ins_cost(INSN_COST);
14339
14340 format %{ "onspinwait" %}
14341
14342 ins_encode %{
14343 __ spin_wait();
14344 %}
14345 ins_pipe(pipe_class_empty);
14346 %}
14347
14348 // ============================================================================
14349 // Logical Instructions
14350
14351 // Integer Logical Instructions
14352
14353 // And Instructions
14354
14355
14356 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14357 match(Set dst (AndI src1 src2));
14358
14359 format %{ "andw $dst, $src1, $src2\t# int" %}
14360
14361 ins_cost(INSN_COST);
14362 ins_encode %{
14363 __ andw(as_Register($dst$$reg),
14364 as_Register($src1$$reg),
14365 as_Register($src2$$reg));
14366 %}
14367
14368 ins_pipe(ialu_reg_reg);
14369 %}
14370
14371 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14372 match(Set dst (AndI src1 src2));
14373
14374 format %{ "andsw $dst, $src1, $src2\t# int" %}
14375
14376 ins_cost(INSN_COST);
14377 ins_encode %{
14378 __ andw(as_Register($dst$$reg),
14379 as_Register($src1$$reg),
14380 (uint64_t)($src2$$constant));
14381 %}
14382
14383 ins_pipe(ialu_reg_imm);
14384 %}
14385
14386 // Or Instructions
14387
14388 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14389 match(Set dst (OrI src1 src2));
14390
14391 format %{ "orrw $dst, $src1, $src2\t# int" %}
14392
14393 ins_cost(INSN_COST);
14394 ins_encode %{
14395 __ orrw(as_Register($dst$$reg),
14396 as_Register($src1$$reg),
14397 as_Register($src2$$reg));
14398 %}
14399
14400 ins_pipe(ialu_reg_reg);
14401 %}
14402
14403 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14404 match(Set dst (OrI src1 src2));
14405
14406 format %{ "orrw $dst, $src1, $src2\t# int" %}
14407
14408 ins_cost(INSN_COST);
14409 ins_encode %{
14410 __ orrw(as_Register($dst$$reg),
14411 as_Register($src1$$reg),
14412 (uint64_t)($src2$$constant));
14413 %}
14414
14415 ins_pipe(ialu_reg_imm);
14416 %}
14417
14418 // Xor Instructions
14419
14420 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14421 match(Set dst (XorI src1 src2));
14422
14423 format %{ "eorw $dst, $src1, $src2\t# int" %}
14424
14425 ins_cost(INSN_COST);
14426 ins_encode %{
14427 __ eorw(as_Register($dst$$reg),
14428 as_Register($src1$$reg),
14429 as_Register($src2$$reg));
14430 %}
14431
14432 ins_pipe(ialu_reg_reg);
14433 %}
14434
14435 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14436 match(Set dst (XorI src1 src2));
14437
14438 format %{ "eorw $dst, $src1, $src2\t# int" %}
14439
14440 ins_cost(INSN_COST);
14441 ins_encode %{
14442 __ eorw(as_Register($dst$$reg),
14443 as_Register($src1$$reg),
14444 (uint64_t)($src2$$constant));
14445 %}
14446
14447 ins_pipe(ialu_reg_imm);
14448 %}
14449
14450 // Long Logical Instructions
14451 // TODO
14452
14453 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14454 match(Set dst (AndL src1 src2));
14455
14456 format %{ "and $dst, $src1, $src2\t# int" %}
14457
14458 ins_cost(INSN_COST);
14459 ins_encode %{
14460 __ andr(as_Register($dst$$reg),
14461 as_Register($src1$$reg),
14462 as_Register($src2$$reg));
14463 %}
14464
14465 ins_pipe(ialu_reg_reg);
14466 %}
14467
14468 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14469 match(Set dst (AndL src1 src2));
14470
14471 format %{ "and $dst, $src1, $src2\t# int" %}
14472
14473 ins_cost(INSN_COST);
14474 ins_encode %{
14475 __ andr(as_Register($dst$$reg),
14476 as_Register($src1$$reg),
14477 (uint64_t)($src2$$constant));
14478 %}
14479
14480 ins_pipe(ialu_reg_imm);
14481 %}
14482
14483 // Or Instructions
14484
14485 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14486 match(Set dst (OrL src1 src2));
14487
14488 format %{ "orr $dst, $src1, $src2\t# int" %}
14489
14490 ins_cost(INSN_COST);
14491 ins_encode %{
14492 __ orr(as_Register($dst$$reg),
14493 as_Register($src1$$reg),
14494 as_Register($src2$$reg));
14495 %}
14496
14497 ins_pipe(ialu_reg_reg);
14498 %}
14499
14500 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14501 match(Set dst (OrL src1 src2));
14502
14503 format %{ "orr $dst, $src1, $src2\t# int" %}
14504
14505 ins_cost(INSN_COST);
14506 ins_encode %{
14507 __ orr(as_Register($dst$$reg),
14508 as_Register($src1$$reg),
14509 (uint64_t)($src2$$constant));
14510 %}
14511
14512 ins_pipe(ialu_reg_imm);
14513 %}
14514
14515 // Xor Instructions
14516
14517 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14518 match(Set dst (XorL src1 src2));
14519
14520 format %{ "eor $dst, $src1, $src2\t# int" %}
14521
14522 ins_cost(INSN_COST);
14523 ins_encode %{
14524 __ eor(as_Register($dst$$reg),
14525 as_Register($src1$$reg),
14526 as_Register($src2$$reg));
14527 %}
14528
14529 ins_pipe(ialu_reg_reg);
14530 %}
14531
14532 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14533 match(Set dst (XorL src1 src2));
14534
14535 ins_cost(INSN_COST);
14536 format %{ "eor $dst, $src1, $src2\t# int" %}
14537
14538 ins_encode %{
14539 __ eor(as_Register($dst$$reg),
14540 as_Register($src1$$reg),
14541 (uint64_t)($src2$$constant));
14542 %}
14543
14544 ins_pipe(ialu_reg_imm);
14545 %}
14546
14547 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14548 %{
14549 match(Set dst (ConvI2L src));
14550
14551 ins_cost(INSN_COST);
14552 format %{ "sxtw $dst, $src\t# i2l" %}
14553 ins_encode %{
14554 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14555 %}
14556 ins_pipe(ialu_reg_shift);
14557 %}
14558
14559 // this pattern occurs in bigmath arithmetic
14560 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14561 %{
14562 match(Set dst (AndL (ConvI2L src) mask));
14563
14564 ins_cost(INSN_COST);
14565 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14566 ins_encode %{
14567 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14568 %}
14569
14570 ins_pipe(ialu_reg_shift);
14571 %}
14572
14573 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14574 match(Set dst (ConvL2I src));
14575
14576 ins_cost(INSN_COST);
14577 format %{ "movw $dst, $src \t// l2i" %}
14578
14579 ins_encode %{
14580 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14581 %}
14582
14583 ins_pipe(ialu_reg);
14584 %}
14585
14586 instruct convD2F_reg(vRegF dst, vRegD src) %{
14587 match(Set dst (ConvD2F src));
14588
14589 ins_cost(INSN_COST * 5);
14590 format %{ "fcvtd $dst, $src \t// d2f" %}
14591
14592 ins_encode %{
14593 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14594 %}
14595
14596 ins_pipe(fp_d2f);
14597 %}
14598
14599 instruct convF2D_reg(vRegD dst, vRegF src) %{
14600 match(Set dst (ConvF2D src));
14601
14602 ins_cost(INSN_COST * 5);
14603 format %{ "fcvts $dst, $src \t// f2d" %}
14604
14605 ins_encode %{
14606 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14607 %}
14608
14609 ins_pipe(fp_f2d);
14610 %}
14611
14612 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14613 match(Set dst (ConvF2I src));
14614
14615 ins_cost(INSN_COST * 5);
14616 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14617
14618 ins_encode %{
14619 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14620 %}
14621
14622 ins_pipe(fp_f2i);
14623 %}
14624
14625 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14626 match(Set dst (ConvF2L src));
14627
14628 ins_cost(INSN_COST * 5);
14629 format %{ "fcvtzs $dst, $src \t// f2l" %}
14630
14631 ins_encode %{
14632 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14633 %}
14634
14635 ins_pipe(fp_f2l);
14636 %}
14637
14638 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14639 match(Set dst (ConvF2HF src));
14640 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14641 "smov $dst, $tmp\t# move result from $tmp to $dst"
14642 %}
14643 effect(TEMP tmp);
14644 ins_encode %{
14645 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14646 %}
14647 ins_pipe(pipe_slow);
14648 %}
14649
14650 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14651 match(Set dst (ConvHF2F src));
14652 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14653 "fcvt $dst, $tmp\t# convert half to single precision"
14654 %}
14655 effect(TEMP tmp);
14656 ins_encode %{
14657 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14658 %}
14659 ins_pipe(pipe_slow);
14660 %}
14661
14662 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14663 match(Set dst (ConvI2F src));
14664
14665 ins_cost(INSN_COST * 5);
14666 format %{ "scvtfws $dst, $src \t// i2f" %}
14667
14668 ins_encode %{
14669 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14670 %}
14671
14672 ins_pipe(fp_i2f);
14673 %}
14674
14675 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14676 match(Set dst (ConvL2F src));
14677
14678 ins_cost(INSN_COST * 5);
14679 format %{ "scvtfs $dst, $src \t// l2f" %}
14680
14681 ins_encode %{
14682 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14683 %}
14684
14685 ins_pipe(fp_l2f);
14686 %}
14687
14688 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14689 match(Set dst (ConvD2I src));
14690
14691 ins_cost(INSN_COST * 5);
14692 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14693
14694 ins_encode %{
14695 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14696 %}
14697
14698 ins_pipe(fp_d2i);
14699 %}
14700
14701 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14702 match(Set dst (ConvD2L src));
14703
14704 ins_cost(INSN_COST * 5);
14705 format %{ "fcvtzd $dst, $src \t// d2l" %}
14706
14707 ins_encode %{
14708 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14709 %}
14710
14711 ins_pipe(fp_d2l);
14712 %}
14713
14714 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14715 match(Set dst (ConvI2D src));
14716
14717 ins_cost(INSN_COST * 5);
14718 format %{ "scvtfwd $dst, $src \t// i2d" %}
14719
14720 ins_encode %{
14721 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14722 %}
14723
14724 ins_pipe(fp_i2d);
14725 %}
14726
14727 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14728 match(Set dst (ConvL2D src));
14729
14730 ins_cost(INSN_COST * 5);
14731 format %{ "scvtfd $dst, $src \t// l2d" %}
14732
14733 ins_encode %{
14734 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14735 %}
14736
14737 ins_pipe(fp_l2d);
14738 %}
14739
14740 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14741 %{
14742 match(Set dst (RoundD src));
14743 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14744 format %{ "java_round_double $dst,$src"%}
14745 ins_encode %{
14746 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14747 as_FloatRegister($ftmp$$reg));
14748 %}
14749 ins_pipe(pipe_slow);
14750 %}
14751
14752 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14753 %{
14754 match(Set dst (RoundF src));
14755 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14756 format %{ "java_round_float $dst,$src"%}
14757 ins_encode %{
14758 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14759 as_FloatRegister($ftmp$$reg));
14760 %}
14761 ins_pipe(pipe_slow);
14762 %}
14763
14764 // stack <-> reg and reg <-> reg shuffles with no conversion
14765
14766 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14767
14768 match(Set dst (MoveF2I src));
14769
14770 effect(DEF dst, USE src);
14771
14772 ins_cost(4 * INSN_COST);
14773
14774 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14775
14776 ins_encode %{
14777 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14778 %}
14779
14780 ins_pipe(iload_reg_reg);
14781
14782 %}
14783
14784 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14785
14786 match(Set dst (MoveI2F src));
14787
14788 effect(DEF dst, USE src);
14789
14790 ins_cost(4 * INSN_COST);
14791
14792 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14793
14794 ins_encode %{
14795 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14796 %}
14797
14798 ins_pipe(pipe_class_memory);
14799
14800 %}
14801
14802 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14803
14804 match(Set dst (MoveD2L src));
14805
14806 effect(DEF dst, USE src);
14807
14808 ins_cost(4 * INSN_COST);
14809
14810 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14811
14812 ins_encode %{
14813 __ ldr($dst$$Register, Address(sp, $src$$disp));
14814 %}
14815
14816 ins_pipe(iload_reg_reg);
14817
14818 %}
14819
14820 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14821
14822 match(Set dst (MoveL2D src));
14823
14824 effect(DEF dst, USE src);
14825
14826 ins_cost(4 * INSN_COST);
14827
14828 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14829
14830 ins_encode %{
14831 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14832 %}
14833
14834 ins_pipe(pipe_class_memory);
14835
14836 %}
14837
14838 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14839
14840 match(Set dst (MoveF2I src));
14841
14842 effect(DEF dst, USE src);
14843
14844 ins_cost(INSN_COST);
14845
14846 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14847
14848 ins_encode %{
14849 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14850 %}
14851
14852 ins_pipe(pipe_class_memory);
14853
14854 %}
14855
14856 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14857
14858 match(Set dst (MoveI2F src));
14859
14860 effect(DEF dst, USE src);
14861
14862 ins_cost(INSN_COST);
14863
14864 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14865
14866 ins_encode %{
14867 __ strw($src$$Register, Address(sp, $dst$$disp));
14868 %}
14869
14870 ins_pipe(istore_reg_reg);
14871
14872 %}
14873
14874 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14875
14876 match(Set dst (MoveD2L src));
14877
14878 effect(DEF dst, USE src);
14879
14880 ins_cost(INSN_COST);
14881
14882 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14883
14884 ins_encode %{
14885 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14886 %}
14887
14888 ins_pipe(pipe_class_memory);
14889
14890 %}
14891
14892 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14893
14894 match(Set dst (MoveL2D src));
14895
14896 effect(DEF dst, USE src);
14897
14898 ins_cost(INSN_COST);
14899
14900 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14901
14902 ins_encode %{
14903 __ str($src$$Register, Address(sp, $dst$$disp));
14904 %}
14905
14906 ins_pipe(istore_reg_reg);
14907
14908 %}
14909
14910 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14911
14912 match(Set dst (MoveF2I src));
14913
14914 effect(DEF dst, USE src);
14915
14916 ins_cost(INSN_COST);
14917
14918 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14919
14920 ins_encode %{
14921 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14922 %}
14923
14924 ins_pipe(fp_f2i);
14925
14926 %}
14927
14928 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14929
14930 match(Set dst (MoveI2F src));
14931
14932 effect(DEF dst, USE src);
14933
14934 ins_cost(INSN_COST);
14935
14936 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14937
14938 ins_encode %{
14939 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14940 %}
14941
14942 ins_pipe(fp_i2f);
14943
14944 %}
14945
14946 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14947
14948 match(Set dst (MoveD2L src));
14949
14950 effect(DEF dst, USE src);
14951
14952 ins_cost(INSN_COST);
14953
14954 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14955
14956 ins_encode %{
14957 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14958 %}
14959
14960 ins_pipe(fp_d2l);
14961
14962 %}
14963
14964 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14965
14966 match(Set dst (MoveL2D src));
14967
14968 effect(DEF dst, USE src);
14969
14970 ins_cost(INSN_COST);
14971
14972 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14973
14974 ins_encode %{
14975 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14976 %}
14977
14978 ins_pipe(fp_l2d);
14979
14980 %}
14981
14982 // ============================================================================
14983 // clearing of an array
14984
14985 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14986 %{
14987 match(Set dummy (ClearArray cnt base));
14988 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14989
14990 ins_cost(4 * INSN_COST);
14991 format %{ "ClearArray $cnt, $base" %}
14992
14993 ins_encode %{
14994 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14995 if (tpc == nullptr) {
14996 ciEnv::current()->record_failure("CodeCache is full");
14997 return;
14998 }
14999 %}
15000
15001 ins_pipe(pipe_class_memory);
15002 %}
15003
15004 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15005 %{
15006 predicate((uint64_t)n->in(2)->get_long()
15007 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15008 match(Set dummy (ClearArray cnt base));
15009 effect(TEMP temp, USE_KILL base, KILL cr);
15010
15011 ins_cost(4 * INSN_COST);
15012 format %{ "ClearArray $cnt, $base" %}
15013
15014 ins_encode %{
15015 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15016 if (tpc == nullptr) {
15017 ciEnv::current()->record_failure("CodeCache is full");
15018 return;
15019 }
15020 %}
15021
15022 ins_pipe(pipe_class_memory);
15023 %}
15024
15025 // ============================================================================
15026 // Overflow Math Instructions
15027
15028 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15029 %{
15030 match(Set cr (OverflowAddI op1 op2));
15031
15032 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15033 ins_cost(INSN_COST);
15034 ins_encode %{
15035 __ cmnw($op1$$Register, $op2$$Register);
15036 %}
15037
15038 ins_pipe(icmp_reg_reg);
15039 %}
15040
15041 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15042 %{
15043 match(Set cr (OverflowAddI op1 op2));
15044
15045 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15046 ins_cost(INSN_COST);
15047 ins_encode %{
15048 __ cmnw($op1$$Register, $op2$$constant);
15049 %}
15050
15051 ins_pipe(icmp_reg_imm);
15052 %}
15053
15054 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15055 %{
15056 match(Set cr (OverflowAddL op1 op2));
15057
15058 format %{ "cmn $op1, $op2\t# overflow check long" %}
15059 ins_cost(INSN_COST);
15060 ins_encode %{
15061 __ cmn($op1$$Register, $op2$$Register);
15062 %}
15063
15064 ins_pipe(icmp_reg_reg);
15065 %}
15066
15067 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15068 %{
15069 match(Set cr (OverflowAddL op1 op2));
15070
15071 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15072 ins_cost(INSN_COST);
15073 ins_encode %{
15074 __ adds(zr, $op1$$Register, $op2$$constant);
15075 %}
15076
15077 ins_pipe(icmp_reg_imm);
15078 %}
15079
15080 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15081 %{
15082 match(Set cr (OverflowSubI op1 op2));
15083
15084 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15085 ins_cost(INSN_COST);
15086 ins_encode %{
15087 __ cmpw($op1$$Register, $op2$$Register);
15088 %}
15089
15090 ins_pipe(icmp_reg_reg);
15091 %}
15092
15093 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15094 %{
15095 match(Set cr (OverflowSubI op1 op2));
15096
15097 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15098 ins_cost(INSN_COST);
15099 ins_encode %{
15100 __ cmpw($op1$$Register, $op2$$constant);
15101 %}
15102
15103 ins_pipe(icmp_reg_imm);
15104 %}
15105
15106 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15107 %{
15108 match(Set cr (OverflowSubL op1 op2));
15109
15110 format %{ "cmp $op1, $op2\t# overflow check long" %}
15111 ins_cost(INSN_COST);
15112 ins_encode %{
15113 __ cmp($op1$$Register, $op2$$Register);
15114 %}
15115
15116 ins_pipe(icmp_reg_reg);
15117 %}
15118
15119 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15120 %{
15121 match(Set cr (OverflowSubL op1 op2));
15122
15123 format %{ "cmp $op1, $op2\t# overflow check long" %}
15124 ins_cost(INSN_COST);
15125 ins_encode %{
15126 __ subs(zr, $op1$$Register, $op2$$constant);
15127 %}
15128
15129 ins_pipe(icmp_reg_imm);
15130 %}
15131
15132 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15133 %{
15134 match(Set cr (OverflowSubI zero op1));
15135
15136 format %{ "cmpw zr, $op1\t# overflow check int" %}
15137 ins_cost(INSN_COST);
15138 ins_encode %{
15139 __ cmpw(zr, $op1$$Register);
15140 %}
15141
15142 ins_pipe(icmp_reg_imm);
15143 %}
15144
15145 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15146 %{
15147 match(Set cr (OverflowSubL zero op1));
15148
15149 format %{ "cmp zr, $op1\t# overflow check long" %}
15150 ins_cost(INSN_COST);
15151 ins_encode %{
15152 __ cmp(zr, $op1$$Register);
15153 %}
15154
15155 ins_pipe(icmp_reg_imm);
15156 %}
15157
15158 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15159 %{
15160 match(Set cr (OverflowMulI op1 op2));
15161
15162 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15163 "cmp rscratch1, rscratch1, sxtw\n\t"
15164 "movw rscratch1, #0x80000000\n\t"
15165 "cselw rscratch1, rscratch1, zr, NE\n\t"
15166 "cmpw rscratch1, #1" %}
15167 ins_cost(5 * INSN_COST);
15168 ins_encode %{
15169 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15170 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15171 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15172 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15173 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15174 %}
15175
15176 ins_pipe(pipe_slow);
15177 %}
15178
15179 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15180 %{
15181 match(If cmp (OverflowMulI op1 op2));
15182 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15183 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15184 effect(USE labl, KILL cr);
15185
15186 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15187 "cmp rscratch1, rscratch1, sxtw\n\t"
15188 "b$cmp $labl" %}
15189 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15190 ins_encode %{
15191 Label* L = $labl$$label;
15192 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15193 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15194 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15195 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15196 %}
15197
15198 ins_pipe(pipe_serial);
15199 %}
15200
15201 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15202 %{
15203 match(Set cr (OverflowMulL op1 op2));
15204
15205 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15206 "smulh rscratch2, $op1, $op2\n\t"
15207 "cmp rscratch2, rscratch1, ASR #63\n\t"
15208 "movw rscratch1, #0x80000000\n\t"
15209 "cselw rscratch1, rscratch1, zr, NE\n\t"
15210 "cmpw rscratch1, #1" %}
15211 ins_cost(6 * INSN_COST);
15212 ins_encode %{
15213 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15214 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15215 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15216 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15217 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15218 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15219 %}
15220
15221 ins_pipe(pipe_slow);
15222 %}
15223
15224 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15225 %{
15226 match(If cmp (OverflowMulL op1 op2));
15227 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15228 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15229 effect(USE labl, KILL cr);
15230
15231 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15232 "smulh rscratch2, $op1, $op2\n\t"
15233 "cmp rscratch2, rscratch1, ASR #63\n\t"
15234 "b$cmp $labl" %}
15235 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15236 ins_encode %{
15237 Label* L = $labl$$label;
15238 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15239 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15240 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15241 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15242 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15243 %}
15244
15245 ins_pipe(pipe_serial);
15246 %}
15247
15248 // ============================================================================
15249 // Compare Instructions
15250
15251 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15252 %{
15253 match(Set cr (CmpI op1 op2));
15254
15255 effect(DEF cr, USE op1, USE op2);
15256
15257 ins_cost(INSN_COST);
15258 format %{ "cmpw $op1, $op2" %}
15259
15260 ins_encode(aarch64_enc_cmpw(op1, op2));
15261
15262 ins_pipe(icmp_reg_reg);
15263 %}
15264
15265 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15266 %{
15267 match(Set cr (CmpI op1 zero));
15268
15269 effect(DEF cr, USE op1);
15270
15271 ins_cost(INSN_COST);
15272 format %{ "cmpw $op1, 0" %}
15273
15274 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15275
15276 ins_pipe(icmp_reg_imm);
15277 %}
15278
15279 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15280 %{
15281 match(Set cr (CmpI op1 op2));
15282
15283 effect(DEF cr, USE op1);
15284
15285 ins_cost(INSN_COST);
15286 format %{ "cmpw $op1, $op2" %}
15287
15288 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15289
15290 ins_pipe(icmp_reg_imm);
15291 %}
15292
15293 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15294 %{
15295 match(Set cr (CmpI op1 op2));
15296
15297 effect(DEF cr, USE op1);
15298
15299 ins_cost(INSN_COST * 2);
15300 format %{ "cmpw $op1, $op2" %}
15301
15302 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15303
15304 ins_pipe(icmp_reg_imm);
15305 %}
15306
15307 // Unsigned compare Instructions; really, same as signed compare
15308 // except it should only be used to feed an If or a CMovI which takes a
15309 // cmpOpU.
15310
15311 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15312 %{
15313 match(Set cr (CmpU op1 op2));
15314
15315 effect(DEF cr, USE op1, USE op2);
15316
15317 ins_cost(INSN_COST);
15318 format %{ "cmpw $op1, $op2\t# unsigned" %}
15319
15320 ins_encode(aarch64_enc_cmpw(op1, op2));
15321
15322 ins_pipe(icmp_reg_reg);
15323 %}
15324
15325 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15326 %{
15327 match(Set cr (CmpU op1 zero));
15328
15329 effect(DEF cr, USE op1);
15330
15331 ins_cost(INSN_COST);
15332 format %{ "cmpw $op1, #0\t# unsigned" %}
15333
15334 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15335
15336 ins_pipe(icmp_reg_imm);
15337 %}
15338
15339 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15340 %{
15341 match(Set cr (CmpU op1 op2));
15342
15343 effect(DEF cr, USE op1);
15344
15345 ins_cost(INSN_COST);
15346 format %{ "cmpw $op1, $op2\t# unsigned" %}
15347
15348 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15349
15350 ins_pipe(icmp_reg_imm);
15351 %}
15352
15353 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15354 %{
15355 match(Set cr (CmpU op1 op2));
15356
15357 effect(DEF cr, USE op1);
15358
15359 ins_cost(INSN_COST * 2);
15360 format %{ "cmpw $op1, $op2\t# unsigned" %}
15361
15362 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15363
15364 ins_pipe(icmp_reg_imm);
15365 %}
15366
15367 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15368 %{
15369 match(Set cr (CmpL op1 op2));
15370
15371 effect(DEF cr, USE op1, USE op2);
15372
15373 ins_cost(INSN_COST);
15374 format %{ "cmp $op1, $op2" %}
15375
15376 ins_encode(aarch64_enc_cmp(op1, op2));
15377
15378 ins_pipe(icmp_reg_reg);
15379 %}
15380
15381 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15382 %{
15383 match(Set cr (CmpL op1 zero));
15384
15385 effect(DEF cr, USE op1);
15386
15387 ins_cost(INSN_COST);
15388 format %{ "tst $op1" %}
15389
15390 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15391
15392 ins_pipe(icmp_reg_imm);
15393 %}
15394
15395 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15396 %{
15397 match(Set cr (CmpL op1 op2));
15398
15399 effect(DEF cr, USE op1);
15400
15401 ins_cost(INSN_COST);
15402 format %{ "cmp $op1, $op2" %}
15403
15404 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15405
15406 ins_pipe(icmp_reg_imm);
15407 %}
15408
15409 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15410 %{
15411 match(Set cr (CmpL op1 op2));
15412
15413 effect(DEF cr, USE op1);
15414
15415 ins_cost(INSN_COST * 2);
15416 format %{ "cmp $op1, $op2" %}
15417
15418 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15419
15420 ins_pipe(icmp_reg_imm);
15421 %}
15422
15423 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15424 %{
15425 match(Set cr (CmpUL op1 op2));
15426
15427 effect(DEF cr, USE op1, USE op2);
15428
15429 ins_cost(INSN_COST);
15430 format %{ "cmp $op1, $op2" %}
15431
15432 ins_encode(aarch64_enc_cmp(op1, op2));
15433
15434 ins_pipe(icmp_reg_reg);
15435 %}
15436
15437 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15438 %{
15439 match(Set cr (CmpUL op1 zero));
15440
15441 effect(DEF cr, USE op1);
15442
15443 ins_cost(INSN_COST);
15444 format %{ "tst $op1" %}
15445
15446 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15447
15448 ins_pipe(icmp_reg_imm);
15449 %}
15450
15451 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15452 %{
15453 match(Set cr (CmpUL op1 op2));
15454
15455 effect(DEF cr, USE op1);
15456
15457 ins_cost(INSN_COST);
15458 format %{ "cmp $op1, $op2" %}
15459
15460 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15461
15462 ins_pipe(icmp_reg_imm);
15463 %}
15464
15465 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15466 %{
15467 match(Set cr (CmpUL op1 op2));
15468
15469 effect(DEF cr, USE op1);
15470
15471 ins_cost(INSN_COST * 2);
15472 format %{ "cmp $op1, $op2" %}
15473
15474 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15475
15476 ins_pipe(icmp_reg_imm);
15477 %}
15478
15479 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15480 %{
15481 match(Set cr (CmpP op1 op2));
15482
15483 effect(DEF cr, USE op1, USE op2);
15484
15485 ins_cost(INSN_COST);
15486 format %{ "cmp $op1, $op2\t // ptr" %}
15487
15488 ins_encode(aarch64_enc_cmpp(op1, op2));
15489
15490 ins_pipe(icmp_reg_reg);
15491 %}
15492
15493 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15494 %{
15495 match(Set cr (CmpN op1 op2));
15496
15497 effect(DEF cr, USE op1, USE op2);
15498
15499 ins_cost(INSN_COST);
15500 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15501
15502 ins_encode(aarch64_enc_cmpn(op1, op2));
15503
15504 ins_pipe(icmp_reg_reg);
15505 %}
15506
15507 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15508 %{
15509 match(Set cr (CmpP op1 zero));
15510
15511 effect(DEF cr, USE op1, USE zero);
15512
15513 ins_cost(INSN_COST);
15514 format %{ "cmp $op1, 0\t // ptr" %}
15515
15516 ins_encode(aarch64_enc_testp(op1));
15517
15518 ins_pipe(icmp_reg_imm);
15519 %}
15520
15521 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15522 %{
15523 match(Set cr (CmpN op1 zero));
15524
15525 effect(DEF cr, USE op1, USE zero);
15526
15527 ins_cost(INSN_COST);
15528 format %{ "cmp $op1, 0\t // compressed ptr" %}
15529
15530 ins_encode(aarch64_enc_testn(op1));
15531
15532 ins_pipe(icmp_reg_imm);
15533 %}
15534
15535 // FP comparisons
15536 //
15537 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15538 // using normal cmpOp. See declaration of rFlagsReg for details.
15539
15540 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15541 %{
15542 match(Set cr (CmpF src1 src2));
15543
15544 ins_cost(3 * INSN_COST);
15545 format %{ "fcmps $src1, $src2" %}
15546
15547 ins_encode %{
15548 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15549 %}
15550
15551 ins_pipe(pipe_class_compare);
15552 %}
15553
15554 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15555 %{
15556 match(Set cr (CmpF src1 src2));
15557
15558 ins_cost(3 * INSN_COST);
15559 format %{ "fcmps $src1, 0.0" %}
15560
15561 ins_encode %{
15562 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15563 %}
15564
15565 ins_pipe(pipe_class_compare);
15566 %}
15567 // FROM HERE
15568
15569 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15570 %{
15571 match(Set cr (CmpD src1 src2));
15572
15573 ins_cost(3 * INSN_COST);
15574 format %{ "fcmpd $src1, $src2" %}
15575
15576 ins_encode %{
15577 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15578 %}
15579
15580 ins_pipe(pipe_class_compare);
15581 %}
15582
15583 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15584 %{
15585 match(Set cr (CmpD src1 src2));
15586
15587 ins_cost(3 * INSN_COST);
15588 format %{ "fcmpd $src1, 0.0" %}
15589
15590 ins_encode %{
15591 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15592 %}
15593
15594 ins_pipe(pipe_class_compare);
15595 %}
15596
15597 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15598 %{
15599 match(Set dst (CmpF3 src1 src2));
15600 effect(KILL cr);
15601
15602 ins_cost(5 * INSN_COST);
15603 format %{ "fcmps $src1, $src2\n\t"
15604 "csinvw($dst, zr, zr, eq\n\t"
15605 "csnegw($dst, $dst, $dst, lt)"
15606 %}
15607
15608 ins_encode %{
15609 Label done;
15610 FloatRegister s1 = as_FloatRegister($src1$$reg);
15611 FloatRegister s2 = as_FloatRegister($src2$$reg);
15612 Register d = as_Register($dst$$reg);
15613 __ fcmps(s1, s2);
15614 // installs 0 if EQ else -1
15615 __ csinvw(d, zr, zr, Assembler::EQ);
15616 // keeps -1 if less or unordered else installs 1
15617 __ csnegw(d, d, d, Assembler::LT);
15618 __ bind(done);
15619 %}
15620
15621 ins_pipe(pipe_class_default);
15622
15623 %}
15624
15625 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15626 %{
15627 match(Set dst (CmpD3 src1 src2));
15628 effect(KILL cr);
15629
15630 ins_cost(5 * INSN_COST);
15631 format %{ "fcmpd $src1, $src2\n\t"
15632 "csinvw($dst, zr, zr, eq\n\t"
15633 "csnegw($dst, $dst, $dst, lt)"
15634 %}
15635
15636 ins_encode %{
15637 Label done;
15638 FloatRegister s1 = as_FloatRegister($src1$$reg);
15639 FloatRegister s2 = as_FloatRegister($src2$$reg);
15640 Register d = as_Register($dst$$reg);
15641 __ fcmpd(s1, s2);
15642 // installs 0 if EQ else -1
15643 __ csinvw(d, zr, zr, Assembler::EQ);
15644 // keeps -1 if less or unordered else installs 1
15645 __ csnegw(d, d, d, Assembler::LT);
15646 __ bind(done);
15647 %}
15648 ins_pipe(pipe_class_default);
15649
15650 %}
15651
15652 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15653 %{
15654 match(Set dst (CmpF3 src1 zero));
15655 effect(KILL cr);
15656
15657 ins_cost(5 * INSN_COST);
15658 format %{ "fcmps $src1, 0.0\n\t"
15659 "csinvw($dst, zr, zr, eq\n\t"
15660 "csnegw($dst, $dst, $dst, lt)"
15661 %}
15662
15663 ins_encode %{
15664 Label done;
15665 FloatRegister s1 = as_FloatRegister($src1$$reg);
15666 Register d = as_Register($dst$$reg);
15667 __ fcmps(s1, 0.0);
15668 // installs 0 if EQ else -1
15669 __ csinvw(d, zr, zr, Assembler::EQ);
15670 // keeps -1 if less or unordered else installs 1
15671 __ csnegw(d, d, d, Assembler::LT);
15672 __ bind(done);
15673 %}
15674
15675 ins_pipe(pipe_class_default);
15676
15677 %}
15678
15679 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15680 %{
15681 match(Set dst (CmpD3 src1 zero));
15682 effect(KILL cr);
15683
15684 ins_cost(5 * INSN_COST);
15685 format %{ "fcmpd $src1, 0.0\n\t"
15686 "csinvw($dst, zr, zr, eq\n\t"
15687 "csnegw($dst, $dst, $dst, lt)"
15688 %}
15689
15690 ins_encode %{
15691 Label done;
15692 FloatRegister s1 = as_FloatRegister($src1$$reg);
15693 Register d = as_Register($dst$$reg);
15694 __ fcmpd(s1, 0.0);
15695 // installs 0 if EQ else -1
15696 __ csinvw(d, zr, zr, Assembler::EQ);
15697 // keeps -1 if less or unordered else installs 1
15698 __ csnegw(d, d, d, Assembler::LT);
15699 __ bind(done);
15700 %}
15701 ins_pipe(pipe_class_default);
15702
15703 %}
15704
15705 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15706 %{
15707 match(Set dst (CmpLTMask p q));
15708 effect(KILL cr);
15709
15710 ins_cost(3 * INSN_COST);
15711
15712 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15713 "csetw $dst, lt\n\t"
15714 "subw $dst, zr, $dst"
15715 %}
15716
15717 ins_encode %{
15718 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15719 __ csetw(as_Register($dst$$reg), Assembler::LT);
15720 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15721 %}
15722
15723 ins_pipe(ialu_reg_reg);
15724 %}
15725
15726 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15727 %{
15728 match(Set dst (CmpLTMask src zero));
15729 effect(KILL cr);
15730
15731 ins_cost(INSN_COST);
15732
15733 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15734
15735 ins_encode %{
15736 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15737 %}
15738
15739 ins_pipe(ialu_reg_shift);
15740 %}
15741
15742 // ============================================================================
15743 // Max and Min
15744
15745 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15746
15747 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15748 %{
15749 effect(DEF cr, USE src);
15750 ins_cost(INSN_COST);
15751 format %{ "cmpw $src, 0" %}
15752
15753 ins_encode %{
15754 __ cmpw($src$$Register, 0);
15755 %}
15756 ins_pipe(icmp_reg_imm);
15757 %}
15758
15759 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15760 %{
15761 match(Set dst (MinI src1 src2));
15762 ins_cost(INSN_COST * 3);
15763
15764 expand %{
15765 rFlagsReg cr;
15766 compI_reg_reg(cr, src1, src2);
15767 cmovI_reg_reg_lt(dst, src1, src2, cr);
15768 %}
15769 %}
15770
15771 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15772 %{
15773 match(Set dst (MaxI src1 src2));
15774 ins_cost(INSN_COST * 3);
15775
15776 expand %{
15777 rFlagsReg cr;
15778 compI_reg_reg(cr, src1, src2);
15779 cmovI_reg_reg_gt(dst, src1, src2, cr);
15780 %}
15781 %}
15782
15783
15784 // ============================================================================
15785 // Branch Instructions
15786
15787 // Direct Branch.
15788 instruct branch(label lbl)
15789 %{
15790 match(Goto);
15791
15792 effect(USE lbl);
15793
15794 ins_cost(BRANCH_COST);
15795 format %{ "b $lbl" %}
15796
15797 ins_encode(aarch64_enc_b(lbl));
15798
15799 ins_pipe(pipe_branch);
15800 %}
15801
15802 // Conditional Near Branch
15803 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15804 %{
15805 // Same match rule as `branchConFar'.
15806 match(If cmp cr);
15807
15808 effect(USE lbl);
15809
15810 ins_cost(BRANCH_COST);
15811 // If set to 1 this indicates that the current instruction is a
15812 // short variant of a long branch. This avoids using this
15813 // instruction in first-pass matching. It will then only be used in
15814 // the `Shorten_branches' pass.
15815 // ins_short_branch(1);
15816 format %{ "b$cmp $lbl" %}
15817
15818 ins_encode(aarch64_enc_br_con(cmp, lbl));
15819
15820 ins_pipe(pipe_branch_cond);
15821 %}
15822
15823 // Conditional Near Branch Unsigned
15824 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15825 %{
15826 // Same match rule as `branchConFar'.
15827 match(If cmp cr);
15828
15829 effect(USE lbl);
15830
15831 ins_cost(BRANCH_COST);
15832 // If set to 1 this indicates that the current instruction is a
15833 // short variant of a long branch. This avoids using this
15834 // instruction in first-pass matching. It will then only be used in
15835 // the `Shorten_branches' pass.
15836 // ins_short_branch(1);
15837 format %{ "b$cmp $lbl\t# unsigned" %}
15838
15839 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15840
15841 ins_pipe(pipe_branch_cond);
15842 %}
15843
15844 // Make use of CBZ and CBNZ. These instructions, as well as being
15845 // shorter than (cmp; branch), have the additional benefit of not
15846 // killing the flags.
15847
15848 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15849 match(If cmp (CmpI op1 op2));
15850 effect(USE labl);
15851
15852 ins_cost(BRANCH_COST);
15853 format %{ "cbw$cmp $op1, $labl" %}
15854 ins_encode %{
15855 Label* L = $labl$$label;
15856 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15857 if (cond == Assembler::EQ)
15858 __ cbzw($op1$$Register, *L);
15859 else
15860 __ cbnzw($op1$$Register, *L);
15861 %}
15862 ins_pipe(pipe_cmp_branch);
15863 %}
15864
15865 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15866 match(If cmp (CmpL op1 op2));
15867 effect(USE labl);
15868
15869 ins_cost(BRANCH_COST);
15870 format %{ "cb$cmp $op1, $labl" %}
15871 ins_encode %{
15872 Label* L = $labl$$label;
15873 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15874 if (cond == Assembler::EQ)
15875 __ cbz($op1$$Register, *L);
15876 else
15877 __ cbnz($op1$$Register, *L);
15878 %}
15879 ins_pipe(pipe_cmp_branch);
15880 %}
15881
15882 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15883 match(If cmp (CmpP op1 op2));
15884 effect(USE labl);
15885
15886 ins_cost(BRANCH_COST);
15887 format %{ "cb$cmp $op1, $labl" %}
15888 ins_encode %{
15889 Label* L = $labl$$label;
15890 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15891 if (cond == Assembler::EQ)
15892 __ cbz($op1$$Register, *L);
15893 else
15894 __ cbnz($op1$$Register, *L);
15895 %}
15896 ins_pipe(pipe_cmp_branch);
15897 %}
15898
15899 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15900 match(If cmp (CmpN op1 op2));
15901 effect(USE labl);
15902
15903 ins_cost(BRANCH_COST);
15904 format %{ "cbw$cmp $op1, $labl" %}
15905 ins_encode %{
15906 Label* L = $labl$$label;
15907 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15908 if (cond == Assembler::EQ)
15909 __ cbzw($op1$$Register, *L);
15910 else
15911 __ cbnzw($op1$$Register, *L);
15912 %}
15913 ins_pipe(pipe_cmp_branch);
15914 %}
15915
15916 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15917 match(If cmp (CmpP (DecodeN oop) zero));
15918 effect(USE labl);
15919
15920 ins_cost(BRANCH_COST);
15921 format %{ "cb$cmp $oop, $labl" %}
15922 ins_encode %{
15923 Label* L = $labl$$label;
15924 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15925 if (cond == Assembler::EQ)
15926 __ cbzw($oop$$Register, *L);
15927 else
15928 __ cbnzw($oop$$Register, *L);
15929 %}
15930 ins_pipe(pipe_cmp_branch);
15931 %}
15932
15933 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15934 match(If cmp (CmpU op1 op2));
15935 effect(USE labl);
15936
15937 ins_cost(BRANCH_COST);
15938 format %{ "cbw$cmp $op1, $labl" %}
15939 ins_encode %{
15940 Label* L = $labl$$label;
15941 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15942 if (cond == Assembler::EQ || cond == Assembler::LS) {
15943 __ cbzw($op1$$Register, *L);
15944 } else {
15945 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15946 __ cbnzw($op1$$Register, *L);
15947 }
15948 %}
15949 ins_pipe(pipe_cmp_branch);
15950 %}
15951
15952 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15953 match(If cmp (CmpUL op1 op2));
15954 effect(USE labl);
15955
15956 ins_cost(BRANCH_COST);
15957 format %{ "cb$cmp $op1, $labl" %}
15958 ins_encode %{
15959 Label* L = $labl$$label;
15960 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15961 if (cond == Assembler::EQ || cond == Assembler::LS) {
15962 __ cbz($op1$$Register, *L);
15963 } else {
15964 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15965 __ cbnz($op1$$Register, *L);
15966 }
15967 %}
15968 ins_pipe(pipe_cmp_branch);
15969 %}
15970
15971 // Test bit and Branch
15972
15973 // Patterns for short (< 32KiB) variants
15974 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15975 match(If cmp (CmpL op1 op2));
15976 effect(USE labl);
15977
15978 ins_cost(BRANCH_COST);
15979 format %{ "cb$cmp $op1, $labl # long" %}
15980 ins_encode %{
15981 Label* L = $labl$$label;
15982 Assembler::Condition cond =
15983 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15984 __ tbr(cond, $op1$$Register, 63, *L);
15985 %}
15986 ins_pipe(pipe_cmp_branch);
15987 ins_short_branch(1);
15988 %}
15989
15990 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15991 match(If cmp (CmpI op1 op2));
15992 effect(USE labl);
15993
15994 ins_cost(BRANCH_COST);
15995 format %{ "cb$cmp $op1, $labl # int" %}
15996 ins_encode %{
15997 Label* L = $labl$$label;
15998 Assembler::Condition cond =
15999 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16000 __ tbr(cond, $op1$$Register, 31, *L);
16001 %}
16002 ins_pipe(pipe_cmp_branch);
16003 ins_short_branch(1);
16004 %}
16005
16006 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16007 match(If cmp (CmpL (AndL op1 op2) op3));
16008 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16009 effect(USE labl);
16010
16011 ins_cost(BRANCH_COST);
16012 format %{ "tb$cmp $op1, $op2, $labl" %}
16013 ins_encode %{
16014 Label* L = $labl$$label;
16015 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16016 int bit = exact_log2_long($op2$$constant);
16017 __ tbr(cond, $op1$$Register, bit, *L);
16018 %}
16019 ins_pipe(pipe_cmp_branch);
16020 ins_short_branch(1);
16021 %}
16022
16023 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16024 match(If cmp (CmpI (AndI op1 op2) op3));
16025 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16026 effect(USE labl);
16027
16028 ins_cost(BRANCH_COST);
16029 format %{ "tb$cmp $op1, $op2, $labl" %}
16030 ins_encode %{
16031 Label* L = $labl$$label;
16032 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16033 int bit = exact_log2((juint)$op2$$constant);
16034 __ tbr(cond, $op1$$Register, bit, *L);
16035 %}
16036 ins_pipe(pipe_cmp_branch);
16037 ins_short_branch(1);
16038 %}
16039
16040 // And far variants
16041 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16042 match(If cmp (CmpL op1 op2));
16043 effect(USE labl);
16044
16045 ins_cost(BRANCH_COST);
16046 format %{ "cb$cmp $op1, $labl # long" %}
16047 ins_encode %{
16048 Label* L = $labl$$label;
16049 Assembler::Condition cond =
16050 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16051 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16052 %}
16053 ins_pipe(pipe_cmp_branch);
16054 %}
16055
16056 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16057 match(If cmp (CmpI op1 op2));
16058 effect(USE labl);
16059
16060 ins_cost(BRANCH_COST);
16061 format %{ "cb$cmp $op1, $labl # int" %}
16062 ins_encode %{
16063 Label* L = $labl$$label;
16064 Assembler::Condition cond =
16065 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16066 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16067 %}
16068 ins_pipe(pipe_cmp_branch);
16069 %}
16070
16071 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16072 match(If cmp (CmpL (AndL op1 op2) op3));
16073 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16074 effect(USE labl);
16075
16076 ins_cost(BRANCH_COST);
16077 format %{ "tb$cmp $op1, $op2, $labl" %}
16078 ins_encode %{
16079 Label* L = $labl$$label;
16080 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16081 int bit = exact_log2_long($op2$$constant);
16082 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16083 %}
16084 ins_pipe(pipe_cmp_branch);
16085 %}
16086
16087 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16088 match(If cmp (CmpI (AndI op1 op2) op3));
16089 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16090 effect(USE labl);
16091
16092 ins_cost(BRANCH_COST);
16093 format %{ "tb$cmp $op1, $op2, $labl" %}
16094 ins_encode %{
16095 Label* L = $labl$$label;
16096 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16097 int bit = exact_log2((juint)$op2$$constant);
16098 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16099 %}
16100 ins_pipe(pipe_cmp_branch);
16101 %}
16102
16103 // Test bits
16104
16105 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16106 match(Set cr (CmpL (AndL op1 op2) op3));
16107 predicate(Assembler::operand_valid_for_logical_immediate
16108 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16109
16110 ins_cost(INSN_COST);
16111 format %{ "tst $op1, $op2 # long" %}
16112 ins_encode %{
16113 __ tst($op1$$Register, $op2$$constant);
16114 %}
16115 ins_pipe(ialu_reg_reg);
16116 %}
16117
16118 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16119 match(Set cr (CmpI (AndI op1 op2) op3));
16120 predicate(Assembler::operand_valid_for_logical_immediate
16121 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16122
16123 ins_cost(INSN_COST);
16124 format %{ "tst $op1, $op2 # int" %}
16125 ins_encode %{
16126 __ tstw($op1$$Register, $op2$$constant);
16127 %}
16128 ins_pipe(ialu_reg_reg);
16129 %}
16130
16131 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16132 match(Set cr (CmpL (AndL op1 op2) op3));
16133
16134 ins_cost(INSN_COST);
16135 format %{ "tst $op1, $op2 # long" %}
16136 ins_encode %{
16137 __ tst($op1$$Register, $op2$$Register);
16138 %}
16139 ins_pipe(ialu_reg_reg);
16140 %}
16141
16142 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16143 match(Set cr (CmpI (AndI op1 op2) op3));
16144
16145 ins_cost(INSN_COST);
16146 format %{ "tstw $op1, $op2 # int" %}
16147 ins_encode %{
16148 __ tstw($op1$$Register, $op2$$Register);
16149 %}
16150 ins_pipe(ialu_reg_reg);
16151 %}
16152
16153
16154 // Conditional Far Branch
16155 // Conditional Far Branch Unsigned
16156 // TODO: fixme
16157
16158 // counted loop end branch near
16159 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16160 %{
16161 match(CountedLoopEnd cmp cr);
16162
16163 effect(USE lbl);
16164
16165 ins_cost(BRANCH_COST);
16166 // short variant.
16167 // ins_short_branch(1);
16168 format %{ "b$cmp $lbl \t// counted loop end" %}
16169
16170 ins_encode(aarch64_enc_br_con(cmp, lbl));
16171
16172 ins_pipe(pipe_branch);
16173 %}
16174
16175 // counted loop end branch far
16176 // TODO: fixme
16177
16178 // ============================================================================
16179 // inlined locking and unlocking
16180
16181 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16182 %{
16183 predicate(LockingMode != LM_LIGHTWEIGHT);
16184 match(Set cr (FastLock object box));
16185 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16186
16187 ins_cost(5 * INSN_COST);
16188 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16189
16190 ins_encode %{
16191 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16192 %}
16193
16194 ins_pipe(pipe_serial);
16195 %}
16196
16197 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16198 %{
16199 predicate(LockingMode != LM_LIGHTWEIGHT);
16200 match(Set cr (FastUnlock object box));
16201 effect(TEMP tmp, TEMP tmp2);
16202
16203 ins_cost(5 * INSN_COST);
16204 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16205
16206 ins_encode %{
16207 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16208 %}
16209
16210 ins_pipe(pipe_serial);
16211 %}
16212
16213 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16214 %{
16215 predicate(LockingMode == LM_LIGHTWEIGHT);
16216 match(Set cr (FastLock object box));
16217 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16218
16219 ins_cost(5 * INSN_COST);
16220 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16221
16222 ins_encode %{
16223 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16224 %}
16225
16226 ins_pipe(pipe_serial);
16227 %}
16228
16229 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16230 %{
16231 predicate(LockingMode == LM_LIGHTWEIGHT);
16232 match(Set cr (FastUnlock object box));
16233 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16234
16235 ins_cost(5 * INSN_COST);
16236 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16237
16238 ins_encode %{
16239 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16240 %}
16241
16242 ins_pipe(pipe_serial);
16243 %}
16244
16245 // ============================================================================
16246 // Safepoint Instructions
16247
16248 // TODO
16249 // provide a near and far version of this code
16250
16251 instruct safePoint(rFlagsReg cr, iRegP poll)
16252 %{
16253 match(SafePoint poll);
16254 effect(KILL cr);
16255
16256 format %{
16257 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16258 %}
16259 ins_encode %{
16260 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16261 %}
16262 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16263 %}
16264
16265
16266 // ============================================================================
16267 // Procedure Call/Return Instructions
16268
16269 // Call Java Static Instruction
16270
16271 instruct CallStaticJavaDirect(method meth)
16272 %{
16273 match(CallStaticJava);
16274
16275 effect(USE meth);
16276
16277 ins_cost(CALL_COST);
16278
16279 format %{ "call,static $meth \t// ==> " %}
16280
16281 ins_encode(aarch64_enc_java_static_call(meth),
16282 aarch64_enc_call_epilog);
16283
16284 ins_pipe(pipe_class_call);
16285 %}
16286
16287 // TO HERE
16288
16289 // Call Java Dynamic Instruction
16290 instruct CallDynamicJavaDirect(method meth)
16291 %{
16292 match(CallDynamicJava);
16293
16294 effect(USE meth);
16295
16296 ins_cost(CALL_COST);
16297
16298 format %{ "CALL,dynamic $meth \t// ==> " %}
16299
16300 ins_encode(aarch64_enc_java_dynamic_call(meth),
16301 aarch64_enc_call_epilog);
16302
16303 ins_pipe(pipe_class_call);
16304 %}
16305
16306 // Call Runtime Instruction
16307
16308 instruct CallRuntimeDirect(method meth)
16309 %{
16310 match(CallRuntime);
16311
16312 effect(USE meth);
16313
16314 ins_cost(CALL_COST);
16315
16316 format %{ "CALL, runtime $meth" %}
16317
16318 ins_encode( aarch64_enc_java_to_runtime(meth) );
16319
16320 ins_pipe(pipe_class_call);
16321 %}
16322
16323 // Call Runtime Instruction
16324
16325 instruct CallLeafDirect(method meth)
16326 %{
16327 match(CallLeaf);
16328
16329 effect(USE meth);
16330
16331 ins_cost(CALL_COST);
16332
16333 format %{ "CALL, runtime leaf $meth" %}
16334
16335 ins_encode( aarch64_enc_java_to_runtime(meth) );
16336
16337 ins_pipe(pipe_class_call);
16338 %}
16339
16340 // Call Runtime Instruction without safepoint and with vector arguments
16341 instruct CallLeafDirectVector(method meth)
16342 %{
16343 match(CallLeafVector);
16344
16345 effect(USE meth);
16346
16347 ins_cost(CALL_COST);
16348
16349 format %{ "CALL, runtime leaf vector $meth" %}
16350
16351 ins_encode(aarch64_enc_java_to_runtime(meth));
16352
16353 ins_pipe(pipe_class_call);
16354 %}
16355
16356 // Call Runtime Instruction
16357
16358 instruct CallLeafNoFPDirect(method meth)
16359 %{
16360 match(CallLeafNoFP);
16361
16362 effect(USE meth);
16363
16364 ins_cost(CALL_COST);
16365
16366 format %{ "CALL, runtime leaf nofp $meth" %}
16367
16368 ins_encode( aarch64_enc_java_to_runtime(meth) );
16369
16370 ins_pipe(pipe_class_call);
16371 %}
16372
16373 // Tail Call; Jump from runtime stub to Java code.
16374 // Also known as an 'interprocedural jump'.
16375 // Target of jump will eventually return to caller.
16376 // TailJump below removes the return address.
16377 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16378 // emitted just above the TailCall which has reset rfp to the caller state.
16379 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16380 %{
16381 match(TailCall jump_target method_ptr);
16382
16383 ins_cost(CALL_COST);
16384
16385 format %{ "br $jump_target\t# $method_ptr holds method" %}
16386
16387 ins_encode(aarch64_enc_tail_call(jump_target));
16388
16389 ins_pipe(pipe_class_call);
16390 %}
16391
16392 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16393 %{
16394 match(TailJump jump_target ex_oop);
16395
16396 ins_cost(CALL_COST);
16397
16398 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16399
16400 ins_encode(aarch64_enc_tail_jmp(jump_target));
16401
16402 ins_pipe(pipe_class_call);
16403 %}
16404
16405 // Forward exception.
16406 instruct ForwardExceptionjmp()
16407 %{
16408 match(ForwardException);
16409 ins_cost(CALL_COST);
16410
16411 format %{ "b forward_exception_stub" %}
16412 ins_encode %{
16413 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16414 %}
16415 ins_pipe(pipe_class_call);
16416 %}
16417
16418 // Create exception oop: created by stack-crawling runtime code.
16419 // Created exception is now available to this handler, and is setup
16420 // just prior to jumping to this handler. No code emitted.
16421 // TODO check
16422 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16423 instruct CreateException(iRegP_R0 ex_oop)
16424 %{
16425 match(Set ex_oop (CreateEx));
16426
16427 format %{ " -- \t// exception oop; no code emitted" %}
16428
16429 size(0);
16430
16431 ins_encode( /*empty*/ );
16432
16433 ins_pipe(pipe_class_empty);
16434 %}
16435
16436 // Rethrow exception: The exception oop will come in the first
16437 // argument position. Then JUMP (not call) to the rethrow stub code.
16438 instruct RethrowException() %{
16439 match(Rethrow);
16440 ins_cost(CALL_COST);
16441
16442 format %{ "b rethrow_stub" %}
16443
16444 ins_encode( aarch64_enc_rethrow() );
16445
16446 ins_pipe(pipe_class_call);
16447 %}
16448
16449
16450 // Return Instruction
16451 // epilog node loads ret address into lr as part of frame pop
16452 instruct Ret()
16453 %{
16454 match(Return);
16455
16456 format %{ "ret\t// return register" %}
16457
16458 ins_encode( aarch64_enc_ret() );
16459
16460 ins_pipe(pipe_branch);
16461 %}
16462
16463 // Die now.
16464 instruct ShouldNotReachHere() %{
16465 match(Halt);
16466
16467 ins_cost(CALL_COST);
16468 format %{ "ShouldNotReachHere" %}
16469
16470 ins_encode %{
16471 if (is_reachable()) {
16472 const char* str = __ code_string(_halt_reason);
16473 __ stop(str);
16474 }
16475 %}
16476
16477 ins_pipe(pipe_class_default);
16478 %}
16479
16480 // ============================================================================
16481 // Partial Subtype Check
16482 //
16483 // superklass array for an instance of the superklass. Set a hidden
16484 // internal cache on a hit (cache is checked with exposed code in
16485 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16486 // encoding ALSO sets flags.
16487
16488 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16489 %{
16490 match(Set result (PartialSubtypeCheck sub super));
16491 predicate(!UseSecondarySupersTable);
16492 effect(KILL cr, KILL temp);
16493
16494 ins_cost(20 * INSN_COST); // slightly larger than the next version
16495 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16496
16497 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16498
16499 opcode(0x1); // Force zero of result reg on hit
16500
16501 ins_pipe(pipe_class_memory);
16502 %}
16503
16504 // Two versions of partialSubtypeCheck, both used when we need to
16505 // search for a super class in the secondary supers array. The first
16506 // is used when we don't know _a priori_ the class being searched
16507 // for. The second, far more common, is used when we do know: this is
16508 // used for instanceof, checkcast, and any case where C2 can determine
16509 // it by constant propagation.
16510
16511 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16512 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16513 rFlagsReg cr)
16514 %{
16515 match(Set result (PartialSubtypeCheck sub super));
16516 predicate(UseSecondarySupersTable);
16517 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16518
16519 ins_cost(10 * INSN_COST); // slightly larger than the next version
16520 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16521
16522 ins_encode %{
16523 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16524 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16525 $vtemp$$FloatRegister,
16526 $result$$Register, /*L_success*/nullptr);
16527 %}
16528
16529 ins_pipe(pipe_class_memory);
16530 %}
16531
16532 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16533 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16534 rFlagsReg cr)
16535 %{
16536 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16537 predicate(UseSecondarySupersTable);
16538 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16539
16540 ins_cost(5 * INSN_COST); // smaller than the next version
16541 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16542
16543 ins_encode %{
16544 bool success = false;
16545 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16546 if (InlineSecondarySupersTest) {
16547 success =
16548 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16549 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16550 $vtemp$$FloatRegister,
16551 $result$$Register,
16552 super_klass_slot);
16553 } else {
16554 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16555 success = (call != nullptr);
16556 }
16557 if (!success) {
16558 ciEnv::current()->record_failure("CodeCache is full");
16559 return;
16560 }
16561 %}
16562
16563 ins_pipe(pipe_class_memory);
16564 %}
16565
16566 // Intrisics for String.compareTo()
16567
16568 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16569 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16570 %{
16571 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16572 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16573 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16574
16575 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16576 ins_encode %{
16577 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16578 __ string_compare($str1$$Register, $str2$$Register,
16579 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16580 $tmp1$$Register, $tmp2$$Register,
16581 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16582 %}
16583 ins_pipe(pipe_class_memory);
16584 %}
16585
16586 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16587 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16588 %{
16589 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16590 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16591 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16592
16593 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16594 ins_encode %{
16595 __ string_compare($str1$$Register, $str2$$Register,
16596 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16597 $tmp1$$Register, $tmp2$$Register,
16598 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16599 %}
16600 ins_pipe(pipe_class_memory);
16601 %}
16602
16603 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16604 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16605 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16606 %{
16607 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16608 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16609 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16610 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16611
16612 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16613 ins_encode %{
16614 __ string_compare($str1$$Register, $str2$$Register,
16615 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16616 $tmp1$$Register, $tmp2$$Register,
16617 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16618 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16619 %}
16620 ins_pipe(pipe_class_memory);
16621 %}
16622
16623 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16624 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16625 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16626 %{
16627 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16628 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16629 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16630 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16631
16632 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16633 ins_encode %{
16634 __ string_compare($str1$$Register, $str2$$Register,
16635 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16636 $tmp1$$Register, $tmp2$$Register,
16637 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16638 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16639 %}
16640 ins_pipe(pipe_class_memory);
16641 %}
16642
16643 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16644 // these string_compare variants as NEON register type for convenience so that the prototype of
16645 // string_compare can be shared with all variants.
16646
16647 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16648 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16649 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16650 pRegGov_P1 pgtmp2, rFlagsReg cr)
16651 %{
16652 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16653 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16654 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16655 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16656
16657 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16658 ins_encode %{
16659 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16660 __ string_compare($str1$$Register, $str2$$Register,
16661 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16662 $tmp1$$Register, $tmp2$$Register,
16663 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16664 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16665 StrIntrinsicNode::LL);
16666 %}
16667 ins_pipe(pipe_class_memory);
16668 %}
16669
16670 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16671 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16672 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16673 pRegGov_P1 pgtmp2, rFlagsReg cr)
16674 %{
16675 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16676 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16677 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16678 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16679
16680 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16681 ins_encode %{
16682 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16683 __ string_compare($str1$$Register, $str2$$Register,
16684 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16685 $tmp1$$Register, $tmp2$$Register,
16686 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16687 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16688 StrIntrinsicNode::LU);
16689 %}
16690 ins_pipe(pipe_class_memory);
16691 %}
16692
16693 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16694 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16695 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16696 pRegGov_P1 pgtmp2, rFlagsReg cr)
16697 %{
16698 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16699 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16700 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16701 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16702
16703 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16704 ins_encode %{
16705 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16706 __ string_compare($str1$$Register, $str2$$Register,
16707 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16708 $tmp1$$Register, $tmp2$$Register,
16709 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16710 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16711 StrIntrinsicNode::UL);
16712 %}
16713 ins_pipe(pipe_class_memory);
16714 %}
16715
16716 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16717 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16718 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16719 pRegGov_P1 pgtmp2, rFlagsReg cr)
16720 %{
16721 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16722 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16723 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16724 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16725
16726 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16727 ins_encode %{
16728 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16729 __ string_compare($str1$$Register, $str2$$Register,
16730 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16731 $tmp1$$Register, $tmp2$$Register,
16732 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16733 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16734 StrIntrinsicNode::UU);
16735 %}
16736 ins_pipe(pipe_class_memory);
16737 %}
16738
16739 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16740 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16741 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16742 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16743 %{
16744 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16745 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16746 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16747 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16748 TEMP vtmp0, TEMP vtmp1, KILL cr);
16749 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16750 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16751
16752 ins_encode %{
16753 __ string_indexof($str1$$Register, $str2$$Register,
16754 $cnt1$$Register, $cnt2$$Register,
16755 $tmp1$$Register, $tmp2$$Register,
16756 $tmp3$$Register, $tmp4$$Register,
16757 $tmp5$$Register, $tmp6$$Register,
16758 -1, $result$$Register, StrIntrinsicNode::UU);
16759 %}
16760 ins_pipe(pipe_class_memory);
16761 %}
16762
16763 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16764 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16765 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16766 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16767 %{
16768 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16769 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16770 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16771 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16772 TEMP vtmp0, TEMP vtmp1, KILL cr);
16773 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16774 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16775
16776 ins_encode %{
16777 __ string_indexof($str1$$Register, $str2$$Register,
16778 $cnt1$$Register, $cnt2$$Register,
16779 $tmp1$$Register, $tmp2$$Register,
16780 $tmp3$$Register, $tmp4$$Register,
16781 $tmp5$$Register, $tmp6$$Register,
16782 -1, $result$$Register, StrIntrinsicNode::LL);
16783 %}
16784 ins_pipe(pipe_class_memory);
16785 %}
16786
16787 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16788 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16789 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16790 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16791 %{
16792 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16793 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16794 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16795 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16796 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16797 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16798 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16799
16800 ins_encode %{
16801 __ string_indexof($str1$$Register, $str2$$Register,
16802 $cnt1$$Register, $cnt2$$Register,
16803 $tmp1$$Register, $tmp2$$Register,
16804 $tmp3$$Register, $tmp4$$Register,
16805 $tmp5$$Register, $tmp6$$Register,
16806 -1, $result$$Register, StrIntrinsicNode::UL);
16807 %}
16808 ins_pipe(pipe_class_memory);
16809 %}
16810
16811 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16812 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16813 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16814 %{
16815 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16816 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16817 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16818 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16819 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16820 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16821
16822 ins_encode %{
16823 int icnt2 = (int)$int_cnt2$$constant;
16824 __ string_indexof($str1$$Register, $str2$$Register,
16825 $cnt1$$Register, zr,
16826 $tmp1$$Register, $tmp2$$Register,
16827 $tmp3$$Register, $tmp4$$Register, zr, zr,
16828 icnt2, $result$$Register, StrIntrinsicNode::UU);
16829 %}
16830 ins_pipe(pipe_class_memory);
16831 %}
16832
16833 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16834 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16835 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16836 %{
16837 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16838 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16839 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16840 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16841 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16842 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16843
16844 ins_encode %{
16845 int icnt2 = (int)$int_cnt2$$constant;
16846 __ string_indexof($str1$$Register, $str2$$Register,
16847 $cnt1$$Register, zr,
16848 $tmp1$$Register, $tmp2$$Register,
16849 $tmp3$$Register, $tmp4$$Register, zr, zr,
16850 icnt2, $result$$Register, StrIntrinsicNode::LL);
16851 %}
16852 ins_pipe(pipe_class_memory);
16853 %}
16854
16855 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16856 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16857 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16858 %{
16859 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16860 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16861 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16862 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16863 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16864 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16865
16866 ins_encode %{
16867 int icnt2 = (int)$int_cnt2$$constant;
16868 __ string_indexof($str1$$Register, $str2$$Register,
16869 $cnt1$$Register, zr,
16870 $tmp1$$Register, $tmp2$$Register,
16871 $tmp3$$Register, $tmp4$$Register, zr, zr,
16872 icnt2, $result$$Register, StrIntrinsicNode::UL);
16873 %}
16874 ins_pipe(pipe_class_memory);
16875 %}
16876
16877 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16878 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16879 iRegINoSp tmp3, rFlagsReg cr)
16880 %{
16881 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16882 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16883 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16884 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16885
16886 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16887
16888 ins_encode %{
16889 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16890 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16891 $tmp3$$Register);
16892 %}
16893 ins_pipe(pipe_class_memory);
16894 %}
16895
16896 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16897 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16898 iRegINoSp tmp3, rFlagsReg cr)
16899 %{
16900 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16901 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16902 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16903 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16904
16905 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16906
16907 ins_encode %{
16908 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16909 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16910 $tmp3$$Register);
16911 %}
16912 ins_pipe(pipe_class_memory);
16913 %}
16914
16915 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16916 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16917 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16918 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16919 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16920 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16921 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16922 ins_encode %{
16923 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16924 $result$$Register, $ztmp1$$FloatRegister,
16925 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16926 $ptmp$$PRegister, true /* isL */);
16927 %}
16928 ins_pipe(pipe_class_memory);
16929 %}
16930
16931 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16932 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16933 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16934 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16935 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16936 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16937 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16938 ins_encode %{
16939 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16940 $result$$Register, $ztmp1$$FloatRegister,
16941 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16942 $ptmp$$PRegister, false /* isL */);
16943 %}
16944 ins_pipe(pipe_class_memory);
16945 %}
16946
16947 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16948 iRegI_R0 result, rFlagsReg cr)
16949 %{
16950 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16951 match(Set result (StrEquals (Binary str1 str2) cnt));
16952 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16953
16954 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16955 ins_encode %{
16956 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16957 __ string_equals($str1$$Register, $str2$$Register,
16958 $result$$Register, $cnt$$Register);
16959 %}
16960 ins_pipe(pipe_class_memory);
16961 %}
16962
16963 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16964 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16965 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16966 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16967 iRegP_R10 tmp, rFlagsReg cr)
16968 %{
16969 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16970 match(Set result (AryEq ary1 ary2));
16971 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16972 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16973 TEMP vtmp6, TEMP vtmp7, KILL cr);
16974
16975 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16976 ins_encode %{
16977 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16978 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16979 $result$$Register, $tmp$$Register, 1);
16980 if (tpc == nullptr) {
16981 ciEnv::current()->record_failure("CodeCache is full");
16982 return;
16983 }
16984 %}
16985 ins_pipe(pipe_class_memory);
16986 %}
16987
16988 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16989 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16990 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16991 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16992 iRegP_R10 tmp, rFlagsReg cr)
16993 %{
16994 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16995 match(Set result (AryEq ary1 ary2));
16996 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16997 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16998 TEMP vtmp6, TEMP vtmp7, KILL cr);
16999
17000 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17001 ins_encode %{
17002 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17003 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17004 $result$$Register, $tmp$$Register, 2);
17005 if (tpc == nullptr) {
17006 ciEnv::current()->record_failure("CodeCache is full");
17007 return;
17008 }
17009 %}
17010 ins_pipe(pipe_class_memory);
17011 %}
17012
17013 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17014 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17015 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17016 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17017 %{
17018 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17019 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17020 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17021
17022 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17023 ins_encode %{
17024 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17025 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17026 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17027 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17028 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17029 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17030 (BasicType)$basic_type$$constant);
17031 if (tpc == nullptr) {
17032 ciEnv::current()->record_failure("CodeCache is full");
17033 return;
17034 }
17035 %}
17036 ins_pipe(pipe_class_memory);
17037 %}
17038
17039 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17040 %{
17041 match(Set result (CountPositives ary1 len));
17042 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17043 format %{ "count positives byte[] $ary1,$len -> $result" %}
17044 ins_encode %{
17045 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17046 if (tpc == nullptr) {
17047 ciEnv::current()->record_failure("CodeCache is full");
17048 return;
17049 }
17050 %}
17051 ins_pipe( pipe_slow );
17052 %}
17053
17054 // fast char[] to byte[] compression
17055 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17056 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17057 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17058 iRegI_R0 result, rFlagsReg cr)
17059 %{
17060 match(Set result (StrCompressedCopy src (Binary dst len)));
17061 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17062 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17063
17064 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17065 ins_encode %{
17066 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17067 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17068 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17069 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17070 %}
17071 ins_pipe(pipe_slow);
17072 %}
17073
17074 // fast byte[] to char[] inflation
17075 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17076 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17077 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17078 %{
17079 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17080 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17081 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17082 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17083
17084 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17085 ins_encode %{
17086 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17087 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17088 $vtmp2$$FloatRegister, $tmp$$Register);
17089 if (tpc == nullptr) {
17090 ciEnv::current()->record_failure("CodeCache is full");
17091 return;
17092 }
17093 %}
17094 ins_pipe(pipe_class_memory);
17095 %}
17096
17097 // encode char[] to byte[] in ISO_8859_1
17098 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17099 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17100 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17101 iRegI_R0 result, rFlagsReg cr)
17102 %{
17103 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17104 match(Set result (EncodeISOArray src (Binary dst len)));
17105 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17106 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17107
17108 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17109 ins_encode %{
17110 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17111 $result$$Register, false,
17112 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17113 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17114 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17115 %}
17116 ins_pipe(pipe_class_memory);
17117 %}
17118
17119 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17120 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17121 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17122 iRegI_R0 result, rFlagsReg cr)
17123 %{
17124 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17125 match(Set result (EncodeISOArray src (Binary dst len)));
17126 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17127 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17128
17129 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17130 ins_encode %{
17131 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17132 $result$$Register, true,
17133 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17134 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17135 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17136 %}
17137 ins_pipe(pipe_class_memory);
17138 %}
17139
17140 //----------------------------- CompressBits/ExpandBits ------------------------
17141
17142 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17143 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17144 match(Set dst (CompressBits src mask));
17145 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17146 format %{ "mov $tsrc, $src\n\t"
17147 "mov $tmask, $mask\n\t"
17148 "bext $tdst, $tsrc, $tmask\n\t"
17149 "mov $dst, $tdst"
17150 %}
17151 ins_encode %{
17152 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17153 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17154 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17155 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17156 %}
17157 ins_pipe(pipe_slow);
17158 %}
17159
17160 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17161 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17162 match(Set dst (CompressBits (LoadI mem) mask));
17163 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17164 format %{ "ldrs $tsrc, $mem\n\t"
17165 "ldrs $tmask, $mask\n\t"
17166 "bext $tdst, $tsrc, $tmask\n\t"
17167 "mov $dst, $tdst"
17168 %}
17169 ins_encode %{
17170 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17171 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17172 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17173 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17174 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17175 %}
17176 ins_pipe(pipe_slow);
17177 %}
17178
17179 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17180 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17181 match(Set dst (CompressBits src mask));
17182 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17183 format %{ "mov $tsrc, $src\n\t"
17184 "mov $tmask, $mask\n\t"
17185 "bext $tdst, $tsrc, $tmask\n\t"
17186 "mov $dst, $tdst"
17187 %}
17188 ins_encode %{
17189 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17190 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17191 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17192 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17193 %}
17194 ins_pipe(pipe_slow);
17195 %}
17196
17197 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17198 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17199 match(Set dst (CompressBits (LoadL mem) mask));
17200 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17201 format %{ "ldrd $tsrc, $mem\n\t"
17202 "ldrd $tmask, $mask\n\t"
17203 "bext $tdst, $tsrc, $tmask\n\t"
17204 "mov $dst, $tdst"
17205 %}
17206 ins_encode %{
17207 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17208 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17209 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17210 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17211 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17212 %}
17213 ins_pipe(pipe_slow);
17214 %}
17215
17216 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17217 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17218 match(Set dst (ExpandBits src mask));
17219 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17220 format %{ "mov $tsrc, $src\n\t"
17221 "mov $tmask, $mask\n\t"
17222 "bdep $tdst, $tsrc, $tmask\n\t"
17223 "mov $dst, $tdst"
17224 %}
17225 ins_encode %{
17226 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17227 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17228 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17229 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17230 %}
17231 ins_pipe(pipe_slow);
17232 %}
17233
17234 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17235 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17236 match(Set dst (ExpandBits (LoadI mem) mask));
17237 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17238 format %{ "ldrs $tsrc, $mem\n\t"
17239 "ldrs $tmask, $mask\n\t"
17240 "bdep $tdst, $tsrc, $tmask\n\t"
17241 "mov $dst, $tdst"
17242 %}
17243 ins_encode %{
17244 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17245 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17246 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17247 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17248 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17249 %}
17250 ins_pipe(pipe_slow);
17251 %}
17252
17253 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17254 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17255 match(Set dst (ExpandBits src mask));
17256 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17257 format %{ "mov $tsrc, $src\n\t"
17258 "mov $tmask, $mask\n\t"
17259 "bdep $tdst, $tsrc, $tmask\n\t"
17260 "mov $dst, $tdst"
17261 %}
17262 ins_encode %{
17263 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17264 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17265 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17266 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17267 %}
17268 ins_pipe(pipe_slow);
17269 %}
17270
17271
17272 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17273 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17274 match(Set dst (ExpandBits (LoadL mem) mask));
17275 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17276 format %{ "ldrd $tsrc, $mem\n\t"
17277 "ldrd $tmask, $mask\n\t"
17278 "bdep $tdst, $tsrc, $tmask\n\t"
17279 "mov $dst, $tdst"
17280 %}
17281 ins_encode %{
17282 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17283 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17284 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17285 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17286 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17287 %}
17288 ins_pipe(pipe_slow);
17289 %}
17290
17291 //----------------------------- Reinterpret ----------------------------------
17292 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17293 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17294 match(Set dst (ReinterpretHF2S src));
17295 format %{ "reinterpretHF2S $dst, $src" %}
17296 ins_encode %{
17297 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17298 %}
17299 ins_pipe(pipe_slow);
17300 %}
17301
17302 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17303 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17304 match(Set dst (ReinterpretS2HF src));
17305 format %{ "reinterpretS2HF $dst, $src" %}
17306 ins_encode %{
17307 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17308 %}
17309 ins_pipe(pipe_slow);
17310 %}
17311
17312 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17313 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17314 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17315 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17316 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17317 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17318 // can be omitted in this pattern, resulting in -
17319 // fcvt $dst, $src // Convert float to half-precision float
17320 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17321 %{
17322 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17323 format %{ "convF2HFAndS2HF $dst, $src" %}
17324 ins_encode %{
17325 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17326 %}
17327 ins_pipe(pipe_slow);
17328 %}
17329
17330 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17331 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17332 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17333 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17334 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17335 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17336 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17337 // resulting in -
17338 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17339 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17340 %{
17341 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17342 format %{ "convHF2SAndHF2F $dst, $src" %}
17343 ins_encode %{
17344 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17345 %}
17346 ins_pipe(pipe_slow);
17347 %}
17348
17349 // ============================================================================
17350 // This name is KNOWN by the ADLC and cannot be changed.
17351 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17352 // for this guy.
17353 instruct tlsLoadP(thread_RegP dst)
17354 %{
17355 match(Set dst (ThreadLocal));
17356
17357 ins_cost(0);
17358
17359 format %{ " -- \t// $dst=Thread::current(), empty" %}
17360
17361 size(0);
17362
17363 ins_encode( /*empty*/ );
17364
17365 ins_pipe(pipe_class_empty);
17366 %}
17367
17368 //----------PEEPHOLE RULES-----------------------------------------------------
17369 // These must follow all instruction definitions as they use the names
17370 // defined in the instructions definitions.
17371 //
17372 // peepmatch ( root_instr_name [preceding_instruction]* );
17373 //
17374 // peepconstraint %{
17375 // (instruction_number.operand_name relational_op instruction_number.operand_name
17376 // [, ...] );
17377 // // instruction numbers are zero-based using left to right order in peepmatch
17378 //
17379 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17380 // // provide an instruction_number.operand_name for each operand that appears
17381 // // in the replacement instruction's match rule
17382 //
17383 // ---------VM FLAGS---------------------------------------------------------
17384 //
17385 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17386 //
17387 // Each peephole rule is given an identifying number starting with zero and
17388 // increasing by one in the order seen by the parser. An individual peephole
17389 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17390 // on the command-line.
17391 //
17392 // ---------CURRENT LIMITATIONS----------------------------------------------
17393 //
17394 // Only match adjacent instructions in same basic block
17395 // Only equality constraints
17396 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17397 // Only one replacement instruction
17398 //
17399 // ---------EXAMPLE----------------------------------------------------------
17400 //
17401 // // pertinent parts of existing instructions in architecture description
17402 // instruct movI(iRegINoSp dst, iRegI src)
17403 // %{
17404 // match(Set dst (CopyI src));
17405 // %}
17406 //
17407 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17408 // %{
17409 // match(Set dst (AddI dst src));
17410 // effect(KILL cr);
17411 // %}
17412 //
17413 // // Change (inc mov) to lea
17414 // peephole %{
17415 // // increment preceded by register-register move
17416 // peepmatch ( incI_iReg movI );
17417 // // require that the destination register of the increment
17418 // // match the destination register of the move
17419 // peepconstraint ( 0.dst == 1.dst );
17420 // // construct a replacement instruction that sets
17421 // // the destination to ( move's source register + one )
17422 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17423 // %}
17424 //
17425
17426 // Implementation no longer uses movX instructions since
17427 // machine-independent system no longer uses CopyX nodes.
17428 //
17429 // peephole
17430 // %{
17431 // peepmatch (incI_iReg movI);
17432 // peepconstraint (0.dst == 1.dst);
17433 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17434 // %}
17435
17436 // peephole
17437 // %{
17438 // peepmatch (decI_iReg movI);
17439 // peepconstraint (0.dst == 1.dst);
17440 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17441 // %}
17442
17443 // peephole
17444 // %{
17445 // peepmatch (addI_iReg_imm movI);
17446 // peepconstraint (0.dst == 1.dst);
17447 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17448 // %}
17449
17450 // peephole
17451 // %{
17452 // peepmatch (incL_iReg movL);
17453 // peepconstraint (0.dst == 1.dst);
17454 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17455 // %}
17456
17457 // peephole
17458 // %{
17459 // peepmatch (decL_iReg movL);
17460 // peepconstraint (0.dst == 1.dst);
17461 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17462 // %}
17463
17464 // peephole
17465 // %{
17466 // peepmatch (addL_iReg_imm movL);
17467 // peepconstraint (0.dst == 1.dst);
17468 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17469 // %}
17470
17471 // peephole
17472 // %{
17473 // peepmatch (addP_iReg_imm movP);
17474 // peepconstraint (0.dst == 1.dst);
17475 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17476 // %}
17477
17478 // // Change load of spilled value to only a spill
17479 // instruct storeI(memory mem, iRegI src)
17480 // %{
17481 // match(Set mem (StoreI mem src));
17482 // %}
17483 //
17484 // instruct loadI(iRegINoSp dst, memory mem)
17485 // %{
17486 // match(Set dst (LoadI mem));
17487 // %}
17488 //
17489
17490 //----------SMARTSPILL RULES---------------------------------------------------
17491 // These must follow all instruction definitions as they use the names
17492 // defined in the instructions definitions.
17493
17494 // Local Variables:
17495 // mode: c++
17496 // End: