1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 // Dummy labels for just measuring the code size
1796 Label dummy_slow_path;
1797 Label dummy_continuation;
1798 Label dummy_guard;
1799 Label* slow_path = &dummy_slow_path;
1800 Label* continuation = &dummy_continuation;
1801 Label* guard = &dummy_guard;
1802 if (!Compile::current()->output()->in_scratch_emit_size()) {
1803 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1804 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1805 Compile::current()->output()->add_stub(stub);
1806 slow_path = &stub->entry();
1807 continuation = &stub->continuation();
1808 guard = &stub->guard();
1809 }
1810 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1811 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1812 }
1813
1814 if (VerifyStackAtCalls) {
1815 Unimplemented();
1816 }
1817
1818 C->output()->set_frame_complete(__ offset());
1819
1820 if (C->has_mach_constant_base_node()) {
1821 // NOTE: We set the table base offset here because users might be
1822 // emitted before MachConstantBaseNode.
1823 ConstantTable& constant_table = C->output()->constant_table();
1824 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1825 }
1826 }
1827
1828 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1829 {
1830 return MachNode::size(ra_); // too many variables; just compute it
1831 // the hard way
1832 }
1833
1834 int MachPrologNode::reloc() const
1835 {
1836 return 0;
1837 }
1838
1839 //=============================================================================
1840
1841 #ifndef PRODUCT
1842 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1843 Compile* C = ra_->C;
1844 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1845
1846 st->print("# pop frame %d\n\t",framesize);
1847
1848 if (framesize == 0) {
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1851 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1852 st->print("add sp, sp, #%d\n\t", framesize);
1853 } else {
1854 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1855 st->print("add sp, sp, rscratch1\n\t");
1856 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1857 }
1858 if (VM_Version::use_rop_protection()) {
1859 st->print("autiaz\n\t");
1860 st->print("ldr zr, [lr]\n\t");
1861 }
1862
1863 if (do_polling() && C->is_method_compilation()) {
1864 st->print("# test polling word\n\t");
1865 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1866 st->print("cmp sp, rscratch1\n\t");
1867 st->print("bhi #slow_path");
1868 }
1869 }
1870 #endif
1871
1872 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 __ remove_frame(framesize);
1877
1878 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1879 __ reserved_stack_check();
1880 }
1881
1882 if (do_polling() && C->is_method_compilation()) {
1883 Label dummy_label;
1884 Label* code_stub = &dummy_label;
1885 if (!C->output()->in_scratch_emit_size()) {
1886 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1887 C->output()->add_stub(stub);
1888 code_stub = &stub->entry();
1889 }
1890 __ relocate(relocInfo::poll_return_type);
1891 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1892 }
1893 }
1894
1895 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1896 // Variable size. Determine dynamically.
1897 return MachNode::size(ra_);
1898 }
1899
1900 int MachEpilogNode::reloc() const {
1901 // Return number of relocatable values contained in this instruction.
1902 return 1; // 1 for polling page.
1903 }
1904
1905 const Pipeline * MachEpilogNode::pipeline() const {
1906 return MachNode::pipeline_class();
1907 }
1908
1909 //=============================================================================
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && masm) {
1975 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1976 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1977 // stack->stack
1978 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1979 sve_vector_reg_size_in_bytes);
1980 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1981 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1982 sve_vector_reg_size_in_bytes);
1983 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1984 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1985 sve_vector_reg_size_in_bytes);
1986 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1987 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1988 as_FloatRegister(Matcher::_regEncode[src_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]));
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 } else if (masm) {
1994 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1995 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1996 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1997 // stack->stack
1998 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1999 if (ireg == Op_VecD) {
2000 __ unspill(rscratch1, true, src_offset);
2001 __ spill(rscratch1, true, dst_offset);
2002 } else {
2003 __ spill_copy128(src_offset, dst_offset);
2004 }
2005 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2006 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2007 ireg == Op_VecD ? __ T8B : __ T16B,
2008 as_FloatRegister(Matcher::_regEncode[src_lo]));
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2011 ireg == Op_VecD ? __ D : __ Q,
2012 ra_->reg2offset(dst_lo));
2013 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2014 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2015 ireg == Op_VecD ? __ D : __ Q,
2016 ra_->reg2offset(src_lo));
2017 } else {
2018 ShouldNotReachHere();
2019 }
2020 }
2021 } else if (masm) {
2022 switch (src_lo_rc) {
2023 case rc_int:
2024 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2025 if (is64) {
2026 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2027 as_Register(Matcher::_regEncode[src_lo]));
2028 } else {
2029 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 }
2032 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2033 if (is64) {
2034 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2035 as_Register(Matcher::_regEncode[src_lo]));
2036 } else {
2037 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2038 as_Register(Matcher::_regEncode[src_lo]));
2039 }
2040 } else { // gpr --> stack spill
2041 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2042 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2043 }
2044 break;
2045 case rc_float:
2046 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2047 if (is64) {
2048 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2049 as_FloatRegister(Matcher::_regEncode[src_lo]));
2050 } else {
2051 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2052 as_FloatRegister(Matcher::_regEncode[src_lo]));
2053 }
2054 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2055 if (is64) {
2056 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2057 as_FloatRegister(Matcher::_regEncode[src_lo]));
2058 } else {
2059 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 as_FloatRegister(Matcher::_regEncode[src_lo]));
2061 }
2062 } else { // fpr --> stack spill
2063 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2064 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2065 is64 ? __ D : __ S, dst_offset);
2066 }
2067 break;
2068 case rc_stack:
2069 if (dst_lo_rc == rc_int) { // stack --> gpr load
2070 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2071 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2072 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 is64 ? __ D : __ S, src_offset);
2074 } else if (dst_lo_rc == rc_predicate) {
2075 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2076 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2077 } else { // stack --> stack copy
2078 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2079 if (ideal_reg() == Op_RegVectMask) {
2080 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2081 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2082 } else {
2083 __ unspill(rscratch1, is64, src_offset);
2084 __ spill(rscratch1, is64, dst_offset);
2085 }
2086 }
2087 break;
2088 case rc_predicate:
2089 if (dst_lo_rc == rc_predicate) {
2090 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2091 } else if (dst_lo_rc == rc_stack) {
2092 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2093 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2094 } else {
2095 assert(false, "bad src and dst rc_class combination.");
2096 ShouldNotReachHere();
2097 }
2098 break;
2099 default:
2100 assert(false, "bad rc_class for spill");
2101 ShouldNotReachHere();
2102 }
2103 }
2104
2105 if (st) {
2106 st->print("spill ");
2107 if (src_lo_rc == rc_stack) {
2108 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2109 } else {
2110 st->print("%s -> ", Matcher::regName[src_lo]);
2111 }
2112 if (dst_lo_rc == rc_stack) {
2113 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2114 } else {
2115 st->print("%s", Matcher::regName[dst_lo]);
2116 }
2117 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2118 int vsize = 0;
2119 switch (ideal_reg()) {
2120 case Op_VecD:
2121 vsize = 64;
2122 break;
2123 case Op_VecX:
2124 vsize = 128;
2125 break;
2126 case Op_VecA:
2127 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2128 break;
2129 default:
2130 assert(false, "bad register type for spill");
2131 ShouldNotReachHere();
2132 }
2133 st->print("\t# vector spill size = %d", vsize);
2134 } else if (ideal_reg() == Op_RegVectMask) {
2135 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2136 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2137 st->print("\t# predicate spill size = %d", vsize);
2138 } else {
2139 st->print("\t# spill size = %d", is64 ? 64 : 32);
2140 }
2141 }
2142
2143 return 0;
2144
2145 }
2146
2147 #ifndef PRODUCT
2148 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2149 if (!ra_)
2150 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2151 else
2152 implementation(nullptr, ra_, false, st);
2153 }
2154 #endif
2155
2156 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2157 implementation(masm, ra_, false, nullptr);
2158 }
2159
2160 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2161 return MachNode::size(ra_);
2162 }
2163
2164 //=============================================================================
2165
2166 #ifndef PRODUCT
2167 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2168 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2169 int reg = ra_->get_reg_first(this);
2170 st->print("add %s, rsp, #%d]\t# box lock",
2171 Matcher::regName[reg], offset);
2172 }
2173 #endif
2174
2175 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177 int reg = ra_->get_encode(this);
2178
2179 // This add will handle any 24-bit signed offset. 24 bits allows an
2180 // 8 megabyte stack frame.
2181 __ add(as_Register(reg), sp, offset);
2182 }
2183
2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2185 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2186 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2187
2188 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2189 return NativeInstruction::instruction_size;
2190 } else {
2191 return 2 * NativeInstruction::instruction_size;
2192 }
2193 }
2194
2195 //=============================================================================
2196
2197 #ifndef PRODUCT
2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2199 {
2200 st->print_cr("# MachUEPNode");
2201 if (UseCompressedClassPointers) {
2202 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2203 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2204 st->print_cr("\tcmpw rscratch1, r10");
2205 } else {
2206 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2207 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2208 st->print_cr("\tcmp rscratch1, r10");
2209 }
2210 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2211 }
2212 #endif
2213
2214 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2215 {
2216 __ ic_check(InteriorEntryAlignment);
2217 }
2218
2219 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2220 {
2221 return MachNode::size(ra_);
2222 }
2223
2224 // REQUIRED EMIT CODE
2225
2226 //=============================================================================
2227
2228 // Emit exception handler code.
2229 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2230 {
2231 // mov rscratch1 #exception_blob_entry_point
2232 // br rscratch1
2233 // Note that the code buffer's insts_mark is always relative to insts.
2234 // That's why we must use the macroassembler to generate a handler.
2235 address base = __ start_a_stub(size_exception_handler());
2236 if (base == nullptr) {
2237 ciEnv::current()->record_failure("CodeCache is full");
2238 return 0; // CodeBuffer::expand failed
2239 }
2240 int offset = __ offset();
2241 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2242 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2243 __ end_a_stub();
2244 return offset;
2245 }
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258
2259 __ adr(lr, __ pc());
2260 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 __ end_a_stub();
2264 return offset;
2265 }
2266
2267 // REQUIRED MATCHER CODE
2268
2269 //=============================================================================
2270
2271 bool Matcher::match_rule_supported(int opcode) {
2272 if (!has_match_rule(opcode))
2273 return false;
2274
2275 switch (opcode) {
2276 case Op_OnSpinWait:
2277 return VM_Version::supports_on_spin_wait();
2278 case Op_CacheWB:
2279 case Op_CacheWBPreSync:
2280 case Op_CacheWBPostSync:
2281 if (!VM_Version::supports_data_cache_line_flush()) {
2282 return false;
2283 }
2284 break;
2285 case Op_ExpandBits:
2286 case Op_CompressBits:
2287 if (!VM_Version::supports_svebitperm()) {
2288 return false;
2289 }
2290 break;
2291 case Op_FmaF:
2292 case Op_FmaD:
2293 case Op_FmaVF:
2294 case Op_FmaVD:
2295 if (!UseFMA) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaHF:
2300 // UseFMA flag also needs to be checked along with FEAT_FP16
2301 if (!UseFMA || !is_feat_fp16_supported()) {
2302 return false;
2303 }
2304 break;
2305 case Op_AddHF:
2306 case Op_SubHF:
2307 case Op_MulHF:
2308 case Op_DivHF:
2309 case Op_MinHF:
2310 case Op_MaxHF:
2311 case Op_SqrtHF:
2312 // Half-precision floating point scalar operations require FEAT_FP16
2313 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2314 // features are supported.
2315 if (!is_feat_fp16_supported()) {
2316 return false;
2317 }
2318 break;
2319 }
2320
2321 return true; // Per default match rules are supported.
2322 }
2323
2324 const RegMask* Matcher::predicate_reg_mask(void) {
2325 return &_PR_REG_mask;
2326 }
2327
2328 bool Matcher::supports_vector_calling_convention(void) {
2329 return EnableVectorSupport;
2330 }
2331
2332 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2333 assert(EnableVectorSupport, "sanity");
2334 int lo = V0_num;
2335 int hi = V0_H_num;
2336 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2337 hi = V0_K_num;
2338 }
2339 return OptoRegPair(hi, lo);
2340 }
2341
2342 // Is this branch offset short enough that a short branch can be used?
2343 //
2344 // NOTE: If the platform does not provide any short branch variants, then
2345 // this method should return false for offset 0.
2346 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2347 // The passed offset is relative to address of the branch.
2348
2349 return (-32768 <= offset && offset < 32768);
2350 }
2351
2352 // Vector width in bytes.
2353 int Matcher::vector_width_in_bytes(BasicType bt) {
2354 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2355 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2356 // Minimum 2 values in vector
2357 if (size < 2*type2aelembytes(bt)) size = 0;
2358 // But never < 4
2359 if (size < 4) size = 0;
2360 return size;
2361 }
2362
2363 // Limits on vector size (number of elements) loaded into vector.
2364 int Matcher::max_vector_size(const BasicType bt) {
2365 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2366 }
2367
2368 int Matcher::min_vector_size(const BasicType bt) {
2369 int max_size = max_vector_size(bt);
2370 // Limit the min vector size to 8 bytes.
2371 int size = 8 / type2aelembytes(bt);
2372 if (bt == T_BYTE) {
2373 // To support vector api shuffle/rearrange.
2374 size = 4;
2375 } else if (bt == T_BOOLEAN) {
2376 // To support vector api load/store mask.
2377 size = 2;
2378 }
2379 if (size < 2) size = 2;
2380 return MIN2(size, max_size);
2381 }
2382
2383 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2384 return Matcher::max_vector_size(bt);
2385 }
2386
2387 // Actual max scalable vector register length.
2388 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2389 return Matcher::max_vector_size(bt);
2390 }
2391
2392 // Vector ideal reg.
2393 uint Matcher::vector_ideal_reg(int len) {
2394 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2395 return Op_VecA;
2396 }
2397 switch(len) {
2398 // For 16-bit/32-bit mask vector, reuse VecD.
2399 case 2:
2400 case 4:
2401 case 8: return Op_VecD;
2402 case 16: return Op_VecX;
2403 }
2404 ShouldNotReachHere();
2405 return 0;
2406 }
2407
2408 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2409 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2410 switch (ideal_reg) {
2411 case Op_VecA: return new vecAOper();
2412 case Op_VecD: return new vecDOper();
2413 case Op_VecX: return new vecXOper();
2414 }
2415 ShouldNotReachHere();
2416 return nullptr;
2417 }
2418
2419 bool Matcher::is_reg2reg_move(MachNode* m) {
2420 return false;
2421 }
2422
2423 bool Matcher::is_generic_vector(MachOper* opnd) {
2424 return opnd->opcode() == VREG;
2425 }
2426
2427 // Return whether or not this register is ever used as an argument.
2428 // This function is used on startup to build the trampoline stubs in
2429 // generateOptoStub. Registers not mentioned will be killed by the VM
2430 // call in the trampoline, and arguments in those registers not be
2431 // available to the callee.
2432 bool Matcher::can_be_java_arg(int reg)
2433 {
2434 return
2435 reg == R0_num || reg == R0_H_num ||
2436 reg == R1_num || reg == R1_H_num ||
2437 reg == R2_num || reg == R2_H_num ||
2438 reg == R3_num || reg == R3_H_num ||
2439 reg == R4_num || reg == R4_H_num ||
2440 reg == R5_num || reg == R5_H_num ||
2441 reg == R6_num || reg == R6_H_num ||
2442 reg == R7_num || reg == R7_H_num ||
2443 reg == V0_num || reg == V0_H_num ||
2444 reg == V1_num || reg == V1_H_num ||
2445 reg == V2_num || reg == V2_H_num ||
2446 reg == V3_num || reg == V3_H_num ||
2447 reg == V4_num || reg == V4_H_num ||
2448 reg == V5_num || reg == V5_H_num ||
2449 reg == V6_num || reg == V6_H_num ||
2450 reg == V7_num || reg == V7_H_num;
2451 }
2452
2453 bool Matcher::is_spillable_arg(int reg)
2454 {
2455 return can_be_java_arg(reg);
2456 }
2457
2458 uint Matcher::int_pressure_limit()
2459 {
2460 // JDK-8183543: When taking the number of available registers as int
2461 // register pressure threshold, the jtreg test:
2462 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2463 // failed due to C2 compilation failure with
2464 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2465 //
2466 // A derived pointer is live at CallNode and then is flagged by RA
2467 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2468 // derived pointers and lastly fail to spill after reaching maximum
2469 // number of iterations. Lowering the default pressure threshold to
2470 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2471 // a high register pressure area of the code so that split_DEF can
2472 // generate DefinitionSpillCopy for the derived pointer.
2473 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2474 if (!PreserveFramePointer) {
2475 // When PreserveFramePointer is off, frame pointer is allocatable,
2476 // but different from other SOC registers, it is excluded from
2477 // fatproj's mask because its save type is No-Save. Decrease 1 to
2478 // ensure high pressure at fatproj when PreserveFramePointer is off.
2479 // See check_pressure_at_fatproj().
2480 default_int_pressure_threshold--;
2481 }
2482 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2483 }
2484
2485 uint Matcher::float_pressure_limit()
2486 {
2487 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2488 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2489 }
2490
2491 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2492 return false;
2493 }
2494
2495 RegMask Matcher::divI_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 // Register for MODI projection of divmodI.
2501 RegMask Matcher::modI_proj_mask() {
2502 ShouldNotReachHere();
2503 return RegMask();
2504 }
2505
2506 // Register for DIVL projection of divmodL.
2507 RegMask Matcher::divL_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask();
2510 }
2511
2512 // Register for MODL projection of divmodL.
2513 RegMask Matcher::modL_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask();
2516 }
2517
2518 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2519 return FP_REG_mask();
2520 }
2521
2522 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2523 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2524 Node* u = addp->fast_out(i);
2525 if (u->is_LoadStore()) {
2526 // On AArch64, LoadStoreNodes (i.e. compare and swap
2527 // instructions) only take register indirect as an operand, so
2528 // any attempt to use an AddPNode as an input to a LoadStoreNode
2529 // must fail.
2530 return false;
2531 }
2532 if (u->is_Mem()) {
2533 int opsize = u->as_Mem()->memory_size();
2534 assert(opsize > 0, "unexpected memory operand size");
2535 if (u->as_Mem()->memory_size() != (1<<shift)) {
2536 return false;
2537 }
2538 }
2539 }
2540 return true;
2541 }
2542
2543 // Convert BootTest condition to Assembler condition.
2544 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2545 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2546 Assembler::Condition result;
2547 switch(cond) {
2548 case BoolTest::eq:
2549 result = Assembler::EQ; break;
2550 case BoolTest::ne:
2551 result = Assembler::NE; break;
2552 case BoolTest::le:
2553 result = Assembler::LE; break;
2554 case BoolTest::ge:
2555 result = Assembler::GE; break;
2556 case BoolTest::lt:
2557 result = Assembler::LT; break;
2558 case BoolTest::gt:
2559 result = Assembler::GT; break;
2560 case BoolTest::ule:
2561 result = Assembler::LS; break;
2562 case BoolTest::uge:
2563 result = Assembler::HS; break;
2564 case BoolTest::ult:
2565 result = Assembler::LO; break;
2566 case BoolTest::ugt:
2567 result = Assembler::HI; break;
2568 case BoolTest::overflow:
2569 result = Assembler::VS; break;
2570 case BoolTest::no_overflow:
2571 result = Assembler::VC; break;
2572 default:
2573 ShouldNotReachHere();
2574 return Assembler::Condition(-1);
2575 }
2576
2577 // Check conversion
2578 if (cond & BoolTest::unsigned_compare) {
2579 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2580 } else {
2581 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2582 }
2583
2584 return result;
2585 }
2586
2587 // Binary src (Replicate con)
2588 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2589 if (n == nullptr || m == nullptr) {
2590 return false;
2591 }
2592
2593 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2594 return false;
2595 }
2596
2597 Node* imm_node = m->in(1);
2598 if (!imm_node->is_Con()) {
2599 return false;
2600 }
2601
2602 const Type* t = imm_node->bottom_type();
2603 if (!(t->isa_int() || t->isa_long())) {
2604 return false;
2605 }
2606
2607 switch (n->Opcode()) {
2608 case Op_AndV:
2609 case Op_OrV:
2610 case Op_XorV: {
2611 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2612 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2613 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2614 }
2615 case Op_AddVB:
2616 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2617 case Op_AddVS:
2618 case Op_AddVI:
2619 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2620 case Op_AddVL:
2621 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2622 default:
2623 return false;
2624 }
2625 }
2626
2627 // (XorV src (Replicate m1))
2628 // (XorVMask src (MaskAll m1))
2629 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2630 if (n != nullptr && m != nullptr) {
2631 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2632 VectorNode::is_all_ones_vector(m);
2633 }
2634 return false;
2635 }
2636
2637 // Should the matcher clone input 'm' of node 'n'?
2638 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2639 if (is_vshift_con_pattern(n, m) ||
2640 is_vector_bitwise_not_pattern(n, m) ||
2641 is_valid_sve_arith_imm_pattern(n, m) ||
2642 is_encode_and_store_pattern(n, m)) {
2643 mstack.push(m, Visit);
2644 return true;
2645 }
2646 return false;
2647 }
2648
2649 // Should the Matcher clone shifts on addressing modes, expecting them
2650 // to be subsumed into complex addressing expressions or compute them
2651 // into registers?
2652 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2653
2654 // Loads and stores with indirect memory input (e.g., volatile loads and
2655 // stores) do not subsume the input into complex addressing expressions. If
2656 // the addressing expression is input to at least one such load or store, do
2657 // not clone the addressing expression. Query needs_acquiring_load and
2658 // needs_releasing_store as a proxy for indirect memory input, as it is not
2659 // possible to directly query for indirect memory input at this stage.
2660 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2661 Node* n = m->fast_out(i);
2662 if (n->is_Load() && needs_acquiring_load(n)) {
2663 return false;
2664 }
2665 if (n->is_Store() && needs_releasing_store(n)) {
2666 return false;
2667 }
2668 }
2669
2670 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2671 return true;
2672 }
2673
2674 Node *off = m->in(AddPNode::Offset);
2675 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2676 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.set(off->_idx); // Flag as address_visited
2680 mstack.push(off->in(2), Visit);
2681 Node *conv = off->in(1);
2682 if (conv->Opcode() == Op_ConvI2L &&
2683 // Are there other uses besides address expressions?
2684 !is_visited(conv)) {
2685 address_visited.set(conv->_idx); // Flag as address_visited
2686 mstack.push(conv->in(1), Pre_Visit);
2687 } else {
2688 mstack.push(conv, Pre_Visit);
2689 }
2690 address_visited.test_set(m->_idx); // Flag as address_visited
2691 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2692 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2693 return true;
2694 } else if (off->Opcode() == Op_ConvI2L &&
2695 // Are there other uses besides address expressions?
2696 !is_visited(off)) {
2697 address_visited.test_set(m->_idx); // Flag as address_visited
2698 address_visited.set(off->_idx); // Flag as address_visited
2699 mstack.push(off->in(1), Pre_Visit);
2700 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2701 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2702 return true;
2703 }
2704 return false;
2705 }
2706
2707 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2708 { \
2709 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2710 guarantee(DISP == 0, "mode not permitted for volatile"); \
2711 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2712 __ INSN(REG, as_Register(BASE)); \
2713 }
2714
2715
2716 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2717 {
2718 Address::extend scale;
2719
2720 // Hooboy, this is fugly. We need a way to communicate to the
2721 // encoder that the index needs to be sign extended, so we have to
2722 // enumerate all the cases.
2723 switch (opcode) {
2724 case INDINDEXSCALEDI2L:
2725 case INDINDEXSCALEDI2LN:
2726 case INDINDEXI2L:
2727 case INDINDEXI2LN:
2728 scale = Address::sxtw(size);
2729 break;
2730 default:
2731 scale = Address::lsl(size);
2732 }
2733
2734 if (index == -1) {
2735 return Address(base, disp);
2736 } else {
2737 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2738 return Address(base, as_Register(index), scale);
2739 }
2740 }
2741
2742
2743 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2744 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2745 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2746 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2747 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2748
2749 // Used for all non-volatile memory accesses. The use of
2750 // $mem->opcode() to discover whether this pattern uses sign-extended
2751 // offsets is something of a kludge.
2752 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2753 Register reg, int opcode,
2754 Register base, int index, int scale, int disp,
2755 int size_in_memory)
2756 {
2757 Address addr = mem2address(opcode, base, index, scale, disp);
2758 if (addr.getMode() == Address::base_plus_offset) {
2759 /* Fix up any out-of-range offsets. */
2760 assert_different_registers(rscratch1, base);
2761 assert_different_registers(rscratch1, reg);
2762 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2763 }
2764 (masm->*insn)(reg, addr);
2765 }
2766
2767 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2768 FloatRegister reg, int opcode,
2769 Register base, int index, int size, int disp,
2770 int size_in_memory)
2771 {
2772 Address::extend scale;
2773
2774 switch (opcode) {
2775 case INDINDEXSCALEDI2L:
2776 case INDINDEXSCALEDI2LN:
2777 scale = Address::sxtw(size);
2778 break;
2779 default:
2780 scale = Address::lsl(size);
2781 }
2782
2783 if (index == -1) {
2784 // Fix up any out-of-range offsets.
2785 assert_different_registers(rscratch1, base);
2786 Address addr = Address(base, disp);
2787 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2788 (masm->*insn)(reg, addr);
2789 } else {
2790 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2791 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2792 }
2793 }
2794
2795 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2796 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2797 int opcode, Register base, int index, int size, int disp)
2798 {
2799 if (index == -1) {
2800 (masm->*insn)(reg, T, Address(base, disp));
2801 } else {
2802 assert(disp == 0, "unsupported address mode");
2803 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2804 }
2805 }
2806
2807 %}
2808
2809
2810
2811 //----------ENCODING BLOCK-----------------------------------------------------
2812 // This block specifies the encoding classes used by the compiler to
2813 // output byte streams. Encoding classes are parameterized macros
2814 // used by Machine Instruction Nodes in order to generate the bit
2815 // encoding of the instruction. Operands specify their base encoding
2816 // interface with the interface keyword. There are currently
2817 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2818 // COND_INTER. REG_INTER causes an operand to generate a function
2819 // which returns its register number when queried. CONST_INTER causes
2820 // an operand to generate a function which returns the value of the
2821 // constant when queried. MEMORY_INTER causes an operand to generate
2822 // four functions which return the Base Register, the Index Register,
2823 // the Scale Value, and the Offset Value of the operand when queried.
2824 // COND_INTER causes an operand to generate six functions which return
2825 // the encoding code (ie - encoding bits for the instruction)
2826 // associated with each basic boolean condition for a conditional
2827 // instruction.
2828 //
2829 // Instructions specify two basic values for encoding. Again, a
2830 // function is available to check if the constant displacement is an
2831 // oop. They use the ins_encode keyword to specify their encoding
2832 // classes (which must be a sequence of enc_class names, and their
2833 // parameters, specified in the encoding block), and they use the
2834 // opcode keyword to specify, in order, their primary, secondary, and
2835 // tertiary opcode. Only the opcode sections which a particular
2836 // instruction needs for encoding need to be specified.
2837 encode %{
2838 // Build emit functions for each basic byte or larger field in the
2839 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2840 // from C++ code in the enc_class source block. Emit functions will
2841 // live in the main source block for now. In future, we can
2842 // generalize this by adding a syntax that specifies the sizes of
2843 // fields in an order, so that the adlc can build the emit functions
2844 // automagically
2845
2846 // catch all for unimplemented encodings
2847 enc_class enc_unimplemented %{
2848 __ unimplemented("C2 catch all");
2849 %}
2850
2851 // BEGIN Non-volatile memory access
2852
2853 // This encoding class is generated automatically from ad_encode.m4.
2854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2855 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2856 Register dst_reg = as_Register($dst$$reg);
2857 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2859 %}
2860
2861 // This encoding class is generated automatically from ad_encode.m4.
2862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2863 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2864 Register dst_reg = as_Register($dst$$reg);
2865 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2867 %}
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2952 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2960 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2968 Register src_reg = as_Register($src$$reg);
2969 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_strb0(memory1 mem) %{
2976 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2983 Register src_reg = as_Register($src$$reg);
2984 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_strh0(memory2 mem) %{
2991 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strw0(memory4 mem) %{
3006 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 // we sometimes get asked to store the stack pointer into the
3015 // current thread -- we cannot do that directly on AArch64
3016 if (src_reg == r31_sp) {
3017 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3018 __ mov(rscratch2, sp);
3019 src_reg = rscratch2;
3020 }
3021 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_str0(memory8 mem) %{
3028 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3035 FloatRegister src_reg = as_FloatRegister($src$$reg);
3036 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3043 FloatRegister src_reg = as_FloatRegister($src$$reg);
3044 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3051 __ membar(Assembler::StoreStore);
3052 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3054 %}
3055
3056 // END Non-volatile memory access
3057
3058 // Vector loads and stores
3059 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3078 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3079 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3102 FloatRegister src_reg = as_FloatRegister($src$$reg);
3103 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 // volatile loads and stores
3108
3109 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3110 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3111 rscratch1, stlrb);
3112 %}
3113
3114 enc_class aarch64_enc_stlrb0(memory mem) %{
3115 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, stlrb);
3117 %}
3118
3119 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3120 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3121 rscratch1, stlrh);
3122 %}
3123
3124 enc_class aarch64_enc_stlrh0(memory mem) %{
3125 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3126 rscratch1, stlrh);
3127 %}
3128
3129 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3130 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, stlrw);
3132 %}
3133
3134 enc_class aarch64_enc_stlrw0(memory mem) %{
3135 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, stlrw);
3137 %}
3138
3139 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3140 Register dst_reg = as_Register($dst$$reg);
3141 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 __ sxtbw(dst_reg, dst_reg);
3144 %}
3145
3146 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3147 Register dst_reg = as_Register($dst$$reg);
3148 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 __ sxtb(dst_reg, dst_reg);
3151 %}
3152
3153 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3154 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarb);
3156 %}
3157
3158 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3159 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarb);
3161 %}
3162
3163 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3164 Register dst_reg = as_Register($dst$$reg);
3165 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 __ sxthw(dst_reg, dst_reg);
3168 %}
3169
3170 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3171 Register dst_reg = as_Register($dst$$reg);
3172 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 __ sxth(dst_reg, dst_reg);
3175 %}
3176
3177 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3178 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarh);
3180 %}
3181
3182 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3183 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarh);
3185 %}
3186
3187 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3188 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarw);
3190 %}
3191
3192 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarw);
3195 %}
3196
3197 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3198 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldar);
3200 %}
3201
3202 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldarw);
3205 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3209 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldar);
3211 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3212 %}
3213
3214 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3215 Register src_reg = as_Register($src$$reg);
3216 // we sometimes get asked to store the stack pointer into the
3217 // current thread -- we cannot do that directly on AArch64
3218 if (src_reg == r31_sp) {
3219 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3220 __ mov(rscratch2, sp);
3221 src_reg = rscratch2;
3222 }
3223 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, stlr);
3225 %}
3226
3227 enc_class aarch64_enc_stlr0(memory mem) %{
3228 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3233 {
3234 FloatRegister src_reg = as_FloatRegister($src$$reg);
3235 __ fmovs(rscratch2, src_reg);
3236 }
3237 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, stlrw);
3239 %}
3240
3241 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3242 {
3243 FloatRegister src_reg = as_FloatRegister($src$$reg);
3244 __ fmovd(rscratch2, src_reg);
3245 }
3246 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, stlr);
3248 %}
3249
3250 // synchronized read/update encodings
3251
3252 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3253 Register dst_reg = as_Register($dst$$reg);
3254 Register base = as_Register($mem$$base);
3255 int index = $mem$$index;
3256 int scale = $mem$$scale;
3257 int disp = $mem$$disp;
3258 if (index == -1) {
3259 if (disp != 0) {
3260 __ lea(rscratch1, Address(base, disp));
3261 __ ldaxr(dst_reg, rscratch1);
3262 } else {
3263 // TODO
3264 // should we ever get anything other than this case?
3265 __ ldaxr(dst_reg, base);
3266 }
3267 } else {
3268 Register index_reg = as_Register(index);
3269 if (disp == 0) {
3270 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3271 __ ldaxr(dst_reg, rscratch1);
3272 } else {
3273 __ lea(rscratch1, Address(base, disp));
3274 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3275 __ ldaxr(dst_reg, rscratch1);
3276 }
3277 }
3278 %}
3279
3280 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3281 Register src_reg = as_Register($src$$reg);
3282 Register base = as_Register($mem$$base);
3283 int index = $mem$$index;
3284 int scale = $mem$$scale;
3285 int disp = $mem$$disp;
3286 if (index == -1) {
3287 if (disp != 0) {
3288 __ lea(rscratch2, Address(base, disp));
3289 __ stlxr(rscratch1, src_reg, rscratch2);
3290 } else {
3291 // TODO
3292 // should we ever get anything other than this case?
3293 __ stlxr(rscratch1, src_reg, base);
3294 }
3295 } else {
3296 Register index_reg = as_Register(index);
3297 if (disp == 0) {
3298 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3299 __ stlxr(rscratch1, src_reg, rscratch2);
3300 } else {
3301 __ lea(rscratch2, Address(base, disp));
3302 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3303 __ stlxr(rscratch1, src_reg, rscratch2);
3304 }
3305 }
3306 __ cmpw(rscratch1, zr);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ false, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::word, /*acquire*/ false, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::byte, /*acquire*/ false, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337
3338 // The only difference between aarch64_enc_cmpxchg and
3339 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3340 // CompareAndSwap sequence to serve as a barrier on acquiring a
3341 // lock.
3342 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3343 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3344 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3345 Assembler::xword, /*acquire*/ true, /*release*/ true,
3346 /*weak*/ false, noreg);
3347 %}
3348
3349 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3350 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3351 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3352 Assembler::word, /*acquire*/ true, /*release*/ true,
3353 /*weak*/ false, noreg);
3354 %}
3355
3356 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3357 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3358 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3359 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3360 /*weak*/ false, noreg);
3361 %}
3362
3363 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 Register res_reg = as_Register($res$$reg);
3373 __ cset(res_reg, Assembler::EQ);
3374 %}
3375
3376 // prefetch encodings
3377
3378 enc_class aarch64_enc_prefetchw(memory mem) %{
3379 Register base = as_Register($mem$$base);
3380 int index = $mem$$index;
3381 int scale = $mem$$scale;
3382 int disp = $mem$$disp;
3383 if (index == -1) {
3384 // Fix up any out-of-range offsets.
3385 assert_different_registers(rscratch1, base);
3386 Address addr = Address(base, disp);
3387 addr = __ legitimize_address(addr, 8, rscratch1);
3388 __ prfm(addr, PSTL1KEEP);
3389 } else {
3390 Register index_reg = as_Register(index);
3391 if (disp == 0) {
3392 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3393 } else {
3394 __ lea(rscratch1, Address(base, disp));
3395 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3396 }
3397 }
3398 %}
3399
3400 // mov encodings
3401
3402 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3403 uint32_t con = (uint32_t)$src$$constant;
3404 Register dst_reg = as_Register($dst$$reg);
3405 if (con == 0) {
3406 __ movw(dst_reg, zr);
3407 } else {
3408 __ movw(dst_reg, con);
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr || con == (address)1) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 if (rtype == relocInfo::oop_type) {
3430 __ movoop(dst_reg, (jobject)con);
3431 } else if (rtype == relocInfo::metadata_type) {
3432 __ mov_metadata(dst_reg, (Metadata*)con);
3433 } else {
3434 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3435 if (! __ is_valid_AArch64_address(con) ||
3436 con < (address)(uintptr_t)os::vm_page_size()) {
3437 __ mov(dst_reg, con);
3438 } else {
3439 uint64_t offset;
3440 __ adrp(dst_reg, con, offset);
3441 __ add(dst_reg, dst_reg, offset);
3442 }
3443 }
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 __ mov(dst_reg, (uint64_t)1);
3455 %}
3456
3457 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3458 Register dst_reg = as_Register($dst$$reg);
3459 address con = (address)$src$$constant;
3460 if (con == nullptr) {
3461 ShouldNotReachHere();
3462 } else {
3463 relocInfo::relocType rtype = $src->constant_reloc();
3464 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3465 __ set_narrow_oop(dst_reg, (jobject)con);
3466 }
3467 %}
3468
3469 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3470 Register dst_reg = as_Register($dst$$reg);
3471 __ mov(dst_reg, zr);
3472 %}
3473
3474 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3475 Register dst_reg = as_Register($dst$$reg);
3476 address con = (address)$src$$constant;
3477 if (con == nullptr) {
3478 ShouldNotReachHere();
3479 } else {
3480 relocInfo::relocType rtype = $src->constant_reloc();
3481 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3482 __ set_narrow_klass(dst_reg, (Klass *)con);
3483 }
3484 %}
3485
3486 // arithmetic encodings
3487
3488 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src_reg = as_Register($src1$$reg);
3491 int32_t con = (int32_t)$src2$$constant;
3492 // add has primary == 0, subtract has primary == 1
3493 if ($primary) { con = -con; }
3494 if (con < 0) {
3495 __ subw(dst_reg, src_reg, -con);
3496 } else {
3497 __ addw(dst_reg, src_reg, con);
3498 }
3499 %}
3500
3501 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 Register src_reg = as_Register($src1$$reg);
3504 int32_t con = (int32_t)$src2$$constant;
3505 // add has primary == 0, subtract has primary == 1
3506 if ($primary) { con = -con; }
3507 if (con < 0) {
3508 __ sub(dst_reg, src_reg, -con);
3509 } else {
3510 __ add(dst_reg, src_reg, con);
3511 }
3512 %}
3513
3514 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src1_reg = as_Register($src1$$reg);
3517 Register src2_reg = as_Register($src2$$reg);
3518 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3519 %}
3520
3521 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3522 Register dst_reg = as_Register($dst$$reg);
3523 Register src1_reg = as_Register($src1$$reg);
3524 Register src2_reg = as_Register($src2$$reg);
3525 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3526 %}
3527
3528 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3529 Register dst_reg = as_Register($dst$$reg);
3530 Register src1_reg = as_Register($src1$$reg);
3531 Register src2_reg = as_Register($src2$$reg);
3532 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3533 %}
3534
3535 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3536 Register dst_reg = as_Register($dst$$reg);
3537 Register src1_reg = as_Register($src1$$reg);
3538 Register src2_reg = as_Register($src2$$reg);
3539 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3540 %}
3541
3542 // compare instruction encodings
3543
3544 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 Register reg2 = as_Register($src2$$reg);
3547 __ cmpw(reg1, reg2);
3548 %}
3549
3550 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3551 Register reg = as_Register($src1$$reg);
3552 int32_t val = $src2$$constant;
3553 if (val >= 0) {
3554 __ subsw(zr, reg, val);
3555 } else {
3556 __ addsw(zr, reg, -val);
3557 }
3558 %}
3559
3560 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3561 Register reg1 = as_Register($src1$$reg);
3562 uint32_t val = (uint32_t)$src2$$constant;
3563 __ movw(rscratch1, val);
3564 __ cmpw(reg1, rscratch1);
3565 %}
3566
3567 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3568 Register reg1 = as_Register($src1$$reg);
3569 Register reg2 = as_Register($src2$$reg);
3570 __ cmp(reg1, reg2);
3571 %}
3572
3573 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3574 Register reg = as_Register($src1$$reg);
3575 int64_t val = $src2$$constant;
3576 if (val >= 0) {
3577 __ subs(zr, reg, val);
3578 } else if (val != -val) {
3579 __ adds(zr, reg, -val);
3580 } else {
3581 // aargh, Long.MIN_VALUE is a special case
3582 __ orr(rscratch1, zr, (uint64_t)val);
3583 __ subs(zr, reg, rscratch1);
3584 }
3585 %}
3586
3587 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3588 Register reg1 = as_Register($src1$$reg);
3589 uint64_t val = (uint64_t)$src2$$constant;
3590 __ mov(rscratch1, val);
3591 __ cmp(reg1, rscratch1);
3592 %}
3593
3594 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3595 Register reg1 = as_Register($src1$$reg);
3596 Register reg2 = as_Register($src2$$reg);
3597 __ cmp(reg1, reg2);
3598 %}
3599
3600 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3601 Register reg1 = as_Register($src1$$reg);
3602 Register reg2 = as_Register($src2$$reg);
3603 __ cmpw(reg1, reg2);
3604 %}
3605
3606 enc_class aarch64_enc_testp(iRegP src) %{
3607 Register reg = as_Register($src$$reg);
3608 __ cmp(reg, zr);
3609 %}
3610
3611 enc_class aarch64_enc_testn(iRegN src) %{
3612 Register reg = as_Register($src$$reg);
3613 __ cmpw(reg, zr);
3614 %}
3615
3616 enc_class aarch64_enc_b(label lbl) %{
3617 Label *L = $lbl$$label;
3618 __ b(*L);
3619 %}
3620
3621 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3622 Label *L = $lbl$$label;
3623 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3624 %}
3625
3626 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3627 Label *L = $lbl$$label;
3628 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3629 %}
3630
3631 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3632 %{
3633 Register sub_reg = as_Register($sub$$reg);
3634 Register super_reg = as_Register($super$$reg);
3635 Register temp_reg = as_Register($temp$$reg);
3636 Register result_reg = as_Register($result$$reg);
3637
3638 Label miss;
3639 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3640 nullptr, &miss,
3641 /*set_cond_codes:*/ true);
3642 if ($primary) {
3643 __ mov(result_reg, zr);
3644 }
3645 __ bind(miss);
3646 %}
3647
3648 enc_class aarch64_enc_java_static_call(method meth) %{
3649 address addr = (address)$meth$$method;
3650 address call;
3651 if (!_method) {
3652 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3653 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3659 // The NOP here is purely to ensure that eliding a call to
3660 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3661 __ nop();
3662 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3663 } else {
3664 int method_index = resolved_method_index(masm);
3665 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3666 : static_call_Relocation::spec(method_index);
3667 call = __ trampoline_call(Address(addr, rspec));
3668 if (call == nullptr) {
3669 ciEnv::current()->record_failure("CodeCache is full");
3670 return;
3671 }
3672 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3673 // Calls of the same statically bound method can share
3674 // a stub to the interpreter.
3675 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3676 } else {
3677 // Emit stub for static call
3678 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3679 if (stub == nullptr) {
3680 ciEnv::current()->record_failure("CodeCache is full");
3681 return;
3682 }
3683 }
3684 }
3685
3686 __ post_call_nop();
3687
3688 // Only non uncommon_trap calls need to reinitialize ptrue.
3689 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3690 __ reinitialize_ptrue();
3691 }
3692 %}
3693
3694 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3695 int method_index = resolved_method_index(masm);
3696 address call = __ ic_call((address)$meth$$method, method_index);
3697 if (call == nullptr) {
3698 ciEnv::current()->record_failure("CodeCache is full");
3699 return;
3700 }
3701 __ post_call_nop();
3702 if (Compile::current()->max_vector_size() > 0) {
3703 __ reinitialize_ptrue();
3704 }
3705 %}
3706
3707 enc_class aarch64_enc_call_epilog() %{
3708 if (VerifyStackAtCalls) {
3709 // Check that stack depth is unchanged: find majik cookie on stack
3710 __ call_Unimplemented();
3711 }
3712 %}
3713
3714 enc_class aarch64_enc_java_to_runtime(method meth) %{
3715 // some calls to generated routines (arraycopy code) are scheduled
3716 // by C2 as runtime calls. if so we can call them using a br (they
3717 // will be in a reachable segment) otherwise we have to use a blr
3718 // which loads the absolute address into a register.
3719 address entry = (address)$meth$$method;
3720 CodeBlob *cb = CodeCache::find_blob(entry);
3721 if (cb) {
3722 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3723 if (call == nullptr) {
3724 ciEnv::current()->record_failure("CodeCache is full");
3725 return;
3726 }
3727 __ post_call_nop();
3728 } else {
3729 Label retaddr;
3730 // Make the anchor frame walkable
3731 __ adr(rscratch2, retaddr);
3732 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3733 __ lea(rscratch1, RuntimeAddress(entry));
3734 __ blr(rscratch1);
3735 __ bind(retaddr);
3736 __ post_call_nop();
3737 }
3738 if (Compile::current()->max_vector_size() > 0) {
3739 __ reinitialize_ptrue();
3740 }
3741 %}
3742
3743 enc_class aarch64_enc_rethrow() %{
3744 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3745 %}
3746
3747 enc_class aarch64_enc_ret() %{
3748 #ifdef ASSERT
3749 if (Compile::current()->max_vector_size() > 0) {
3750 __ verify_ptrue();
3751 }
3752 #endif
3753 __ ret(lr);
3754 %}
3755
3756 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3757 Register target_reg = as_Register($jump_target$$reg);
3758 __ br(target_reg);
3759 %}
3760
3761 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3762 Register target_reg = as_Register($jump_target$$reg);
3763 // exception oop should be in r0
3764 // ret addr has been popped into lr
3765 // callee expects it in r3
3766 __ mov(r3, lr);
3767 __ br(target_reg);
3768 %}
3769
3770 %}
3771
3772 //----------FRAME--------------------------------------------------------------
3773 // Definition of frame structure and management information.
3774 //
3775 // S T A C K L A Y O U T Allocators stack-slot number
3776 // | (to get allocators register number
3777 // G Owned by | | v add OptoReg::stack0())
3778 // r CALLER | |
3779 // o | +--------+ pad to even-align allocators stack-slot
3780 // w V | pad0 | numbers; owned by CALLER
3781 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3782 // h ^ | in | 5
3783 // | | args | 4 Holes in incoming args owned by SELF
3784 // | | | | 3
3785 // | | +--------+
3786 // V | | old out| Empty on Intel, window on Sparc
3787 // | old |preserve| Must be even aligned.
3788 // | SP-+--------+----> Matcher::_old_SP, even aligned
3789 // | | in | 3 area for Intel ret address
3790 // Owned by |preserve| Empty on Sparc.
3791 // SELF +--------+
3792 // | | pad2 | 2 pad to align old SP
3793 // | +--------+ 1
3794 // | | locks | 0
3795 // | +--------+----> OptoReg::stack0(), even aligned
3796 // | | pad1 | 11 pad to align new SP
3797 // | +--------+
3798 // | | | 10
3799 // | | spills | 9 spills
3800 // V | | 8 (pad0 slot for callee)
3801 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3802 // ^ | out | 7
3803 // | | args | 6 Holes in outgoing args owned by CALLEE
3804 // Owned by +--------+
3805 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3806 // | new |preserve| Must be even-aligned.
3807 // | SP-+--------+----> Matcher::_new_SP, even aligned
3808 // | | |
3809 //
3810 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3811 // known from SELF's arguments and the Java calling convention.
3812 // Region 6-7 is determined per call site.
3813 // Note 2: If the calling convention leaves holes in the incoming argument
3814 // area, those holes are owned by SELF. Holes in the outgoing area
3815 // are owned by the CALLEE. Holes should not be necessary in the
3816 // incoming area, as the Java calling convention is completely under
3817 // the control of the AD file. Doubles can be sorted and packed to
3818 // avoid holes. Holes in the outgoing arguments may be necessary for
3819 // varargs C calling conventions.
3820 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3821 // even aligned with pad0 as needed.
3822 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3823 // (the latter is true on Intel but is it false on AArch64?)
3824 // region 6-11 is even aligned; it may be padded out more so that
3825 // the region from SP to FP meets the minimum stack alignment.
3826 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3827 // alignment. Region 11, pad1, may be dynamically extended so that
3828 // SP meets the minimum alignment.
3829
3830 frame %{
3831 // These three registers define part of the calling convention
3832 // between compiled code and the interpreter.
3833
3834 // Inline Cache Register or Method for I2C.
3835 inline_cache_reg(R12);
3836
3837 // Number of stack slots consumed by locking an object
3838 sync_stack_slots(2);
3839
3840 // Compiled code's Frame Pointer
3841 frame_pointer(R31);
3842
3843 // Interpreter stores its frame pointer in a register which is
3844 // stored to the stack by I2CAdaptors.
3845 // I2CAdaptors convert from interpreted java to compiled java.
3846 interpreter_frame_pointer(R29);
3847
3848 // Stack alignment requirement
3849 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3850
3851 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3852 // for calls to C. Supports the var-args backing area for register parms.
3853 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3854
3855 // The after-PROLOG location of the return address. Location of
3856 // return address specifies a type (REG or STACK) and a number
3857 // representing the register number (i.e. - use a register name) or
3858 // stack slot.
3859 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3860 // Otherwise, it is above the locks and verification slot and alignment word
3861 // TODO this may well be correct but need to check why that - 2 is there
3862 // ppc port uses 0 but we definitely need to allow for fixed_slots
3863 // which folds in the space used for monitors
3864 return_addr(STACK - 2 +
3865 align_up((Compile::current()->in_preserve_stack_slots() +
3866 Compile::current()->fixed_slots()),
3867 stack_alignment_in_slots()));
3868
3869 // Location of compiled Java return values. Same as C for now.
3870 return_value
3871 %{
3872 // TODO do we allow ideal_reg == Op_RegN???
3873 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3874 "only return normal values");
3875
3876 static const int lo[Op_RegL + 1] = { // enum name
3877 0, // Op_Node
3878 0, // Op_Set
3879 R0_num, // Op_RegN
3880 R0_num, // Op_RegI
3881 R0_num, // Op_RegP
3882 V0_num, // Op_RegF
3883 V0_num, // Op_RegD
3884 R0_num // Op_RegL
3885 };
3886
3887 static const int hi[Op_RegL + 1] = { // enum name
3888 0, // Op_Node
3889 0, // Op_Set
3890 OptoReg::Bad, // Op_RegN
3891 OptoReg::Bad, // Op_RegI
3892 R0_H_num, // Op_RegP
3893 OptoReg::Bad, // Op_RegF
3894 V0_H_num, // Op_RegD
3895 R0_H_num // Op_RegL
3896 };
3897
3898 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3899 %}
3900 %}
3901
3902 //----------ATTRIBUTES---------------------------------------------------------
3903 //----------Operand Attributes-------------------------------------------------
3904 op_attrib op_cost(1); // Required cost attribute
3905
3906 //----------Instruction Attributes---------------------------------------------
3907 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3908 ins_attrib ins_size(32); // Required size attribute (in bits)
3909 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3910 // a non-matching short branch variant
3911 // of some long branch?
3912 ins_attrib ins_alignment(4); // Required alignment attribute (must
3913 // be a power of 2) specifies the
3914 // alignment that some part of the
3915 // instruction (not necessarily the
3916 // start) requires. If > 1, a
3917 // compute_padding() function must be
3918 // provided for the instruction
3919
3920 //----------OPERANDS-----------------------------------------------------------
3921 // Operand definitions must precede instruction definitions for correct parsing
3922 // in the ADLC because operands constitute user defined types which are used in
3923 // instruction definitions.
3924
3925 //----------Simple Operands----------------------------------------------------
3926
3927 // Integer operands 32 bit
3928 // 32 bit immediate
3929 operand immI()
3930 %{
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit zero
3939 operand immI0()
3940 %{
3941 predicate(n->get_int() == 0);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // 32 bit unit increment
3950 operand immI_1()
3951 %{
3952 predicate(n->get_int() == 1);
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 // 32 bit unit decrement
3961 operand immI_M1()
3962 %{
3963 predicate(n->get_int() == -1);
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 // Shift values for add/sub extension shift
3972 operand immIExt()
3973 %{
3974 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3975 match(ConI);
3976
3977 op_cost(0);
3978 format %{ %}
3979 interface(CONST_INTER);
3980 %}
3981
3982 operand immI_gt_1()
3983 %{
3984 predicate(n->get_int() > 1);
3985 match(ConI);
3986
3987 op_cost(0);
3988 format %{ %}
3989 interface(CONST_INTER);
3990 %}
3991
3992 operand immI_le_4()
3993 %{
3994 predicate(n->get_int() <= 4);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 operand immI_16()
4003 %{
4004 predicate(n->get_int() == 16);
4005 match(ConI);
4006
4007 op_cost(0);
4008 format %{ %}
4009 interface(CONST_INTER);
4010 %}
4011
4012 operand immI_24()
4013 %{
4014 predicate(n->get_int() == 24);
4015 match(ConI);
4016
4017 op_cost(0);
4018 format %{ %}
4019 interface(CONST_INTER);
4020 %}
4021
4022 operand immI_32()
4023 %{
4024 predicate(n->get_int() == 32);
4025 match(ConI);
4026
4027 op_cost(0);
4028 format %{ %}
4029 interface(CONST_INTER);
4030 %}
4031
4032 operand immI_48()
4033 %{
4034 predicate(n->get_int() == 48);
4035 match(ConI);
4036
4037 op_cost(0);
4038 format %{ %}
4039 interface(CONST_INTER);
4040 %}
4041
4042 operand immI_56()
4043 %{
4044 predicate(n->get_int() == 56);
4045 match(ConI);
4046
4047 op_cost(0);
4048 format %{ %}
4049 interface(CONST_INTER);
4050 %}
4051
4052 operand immI_255()
4053 %{
4054 predicate(n->get_int() == 255);
4055 match(ConI);
4056
4057 op_cost(0);
4058 format %{ %}
4059 interface(CONST_INTER);
4060 %}
4061
4062 operand immI_65535()
4063 %{
4064 predicate(n->get_int() == 65535);
4065 match(ConI);
4066
4067 op_cost(0);
4068 format %{ %}
4069 interface(CONST_INTER);
4070 %}
4071
4072 operand immI_positive()
4073 %{
4074 predicate(n->get_int() > 0);
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 // BoolTest condition for signed compare
4083 operand immI_cmp_cond()
4084 %{
4085 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4086 match(ConI);
4087
4088 op_cost(0);
4089 format %{ %}
4090 interface(CONST_INTER);
4091 %}
4092
4093 // BoolTest condition for unsigned compare
4094 operand immI_cmpU_cond()
4095 %{
4096 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4097 match(ConI);
4098
4099 op_cost(0);
4100 format %{ %}
4101 interface(CONST_INTER);
4102 %}
4103
4104 operand immL_255()
4105 %{
4106 predicate(n->get_long() == 255L);
4107 match(ConL);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immL_65535()
4115 %{
4116 predicate(n->get_long() == 65535L);
4117 match(ConL);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_4294967295()
4125 %{
4126 predicate(n->get_long() == 4294967295L);
4127 match(ConL);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immL_bitmask()
4135 %{
4136 predicate((n->get_long() != 0)
4137 && ((n->get_long() & 0xc000000000000000l) == 0)
4138 && is_power_of_2(n->get_long() + 1));
4139 match(ConL);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 operand immI_bitmask()
4147 %{
4148 predicate((n->get_int() != 0)
4149 && ((n->get_int() & 0xc0000000) == 0)
4150 && is_power_of_2(n->get_int() + 1));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 operand immL_positive_bitmaskI()
4159 %{
4160 predicate((n->get_long() != 0)
4161 && ((julong)n->get_long() < 0x80000000ULL)
4162 && is_power_of_2(n->get_long() + 1));
4163 match(ConL);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 // Scale values for scaled offset addressing modes (up to long but not quad)
4171 operand immIScale()
4172 %{
4173 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4174 match(ConI);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 // 5 bit signed integer
4182 operand immI5()
4183 %{
4184 predicate(Assembler::is_simm(n->get_int(), 5));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 // 7 bit unsigned integer
4193 operand immIU7()
4194 %{
4195 predicate(Assembler::is_uimm(n->get_int(), 7));
4196 match(ConI);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 // Offset for scaled or unscaled immediate loads and stores
4204 operand immIOffset()
4205 %{
4206 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immIOffset1()
4215 %{
4216 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immIOffset2()
4225 %{
4226 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immIOffset4()
4235 %{
4236 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immIOffset8()
4245 %{
4246 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immIOffset16()
4255 %{
4256 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4257 match(ConI);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immLOffset()
4265 %{
4266 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4267 match(ConL);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immLoffset1()
4275 %{
4276 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4277 match(ConL);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immLoffset2()
4285 %{
4286 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4287 match(ConL);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immLoffset4()
4295 %{
4296 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4297 match(ConL);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 operand immLoffset8()
4305 %{
4306 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4307 match(ConL);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immLoffset16()
4315 %{
4316 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 5 bit signed long integer
4325 operand immL5()
4326 %{
4327 predicate(Assembler::is_simm(n->get_long(), 5));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 7 bit unsigned long integer
4336 operand immLU7()
4337 %{
4338 predicate(Assembler::is_uimm(n->get_long(), 7));
4339 match(ConL);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit signed value.
4347 operand immI8()
4348 %{
4349 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4350 match(ConI);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 // 8 bit signed value (simm8), or #simm8 LSL 8.
4358 operand immI8_shift8()
4359 %{
4360 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4361 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4362 match(ConI);
4363
4364 op_cost(0);
4365 format %{ %}
4366 interface(CONST_INTER);
4367 %}
4368
4369 // 8 bit signed value (simm8), or #simm8 LSL 8.
4370 operand immL8_shift8()
4371 %{
4372 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4373 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4374 match(ConL);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 8 bit integer valid for vector add sub immediate
4382 operand immBAddSubV()
4383 %{
4384 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4385 match(ConI);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 32 bit integer valid for add sub immediate
4393 operand immIAddSub()
4394 %{
4395 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4396 match(ConI);
4397 op_cost(0);
4398 format %{ %}
4399 interface(CONST_INTER);
4400 %}
4401
4402 // 32 bit integer valid for vector add sub immediate
4403 operand immIAddSubV()
4404 %{
4405 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4406 match(ConI);
4407
4408 op_cost(0);
4409 format %{ %}
4410 interface(CONST_INTER);
4411 %}
4412
4413 // 32 bit unsigned integer valid for logical immediate
4414
4415 operand immBLog()
4416 %{
4417 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4418 match(ConI);
4419
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 operand immSLog()
4426 %{
4427 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4428 match(ConI);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 operand immILog()
4436 %{
4437 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4438 match(ConI);
4439
4440 op_cost(0);
4441 format %{ %}
4442 interface(CONST_INTER);
4443 %}
4444
4445 // Integer operands 64 bit
4446 // 64 bit immediate
4447 operand immL()
4448 %{
4449 match(ConL);
4450
4451 op_cost(0);
4452 format %{ %}
4453 interface(CONST_INTER);
4454 %}
4455
4456 // 64 bit zero
4457 operand immL0()
4458 %{
4459 predicate(n->get_long() == 0);
4460 match(ConL);
4461
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // 64 bit unit decrement
4468 operand immL_M1()
4469 %{
4470 predicate(n->get_long() == -1);
4471 match(ConL);
4472
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // 64 bit integer valid for add sub immediate
4479 operand immLAddSub()
4480 %{
4481 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4482 match(ConL);
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // 64 bit integer valid for addv subv immediate
4489 operand immLAddSubV()
4490 %{
4491 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4492 match(ConL);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // 64 bit integer valid for logical immediate
4500 operand immLLog()
4501 %{
4502 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4503 match(ConL);
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 // Long Immediate: low 32-bit mask
4510 operand immL_32bits()
4511 %{
4512 predicate(n->get_long() == 0xFFFFFFFFL);
4513 match(ConL);
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 // Pointer operands
4520 // Pointer Immediate
4521 operand immP()
4522 %{
4523 match(ConP);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 // nullptr Pointer Immediate
4531 operand immP0()
4532 %{
4533 predicate(n->get_ptr() == 0);
4534 match(ConP);
4535
4536 op_cost(0);
4537 format %{ %}
4538 interface(CONST_INTER);
4539 %}
4540
4541 // Pointer Immediate One
4542 // this is used in object initialization (initial object header)
4543 operand immP_1()
4544 %{
4545 predicate(n->get_ptr() == 1);
4546 match(ConP);
4547
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // Card Table Byte Map Base
4554 operand immByteMapBase()
4555 %{
4556 // Get base of card map
4557 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4558 is_card_table_address((address)(n->get_ptr())));
4559 match(ConP);
4560
4561 op_cost(0);
4562 format %{ %}
4563 interface(CONST_INTER);
4564 %}
4565
4566 // AOT Runtime Constants Address
4567 operand immAOTRuntimeConstantsAddress()
4568 %{
4569 // Check if the address is in the range of AOT Runtime Constants
4570 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4571 match(ConP);
4572
4573 op_cost(0);
4574 format %{ %}
4575 interface(CONST_INTER);
4576 %}
4577
4578 // Float and Double operands
4579 // Double Immediate
4580 operand immD()
4581 %{
4582 match(ConD);
4583 op_cost(0);
4584 format %{ %}
4585 interface(CONST_INTER);
4586 %}
4587
4588 // Double Immediate: +0.0d
4589 operand immD0()
4590 %{
4591 predicate(jlong_cast(n->getd()) == 0);
4592 match(ConD);
4593
4594 op_cost(0);
4595 format %{ %}
4596 interface(CONST_INTER);
4597 %}
4598
4599 // constant 'double +0.0'.
4600 operand immDPacked()
4601 %{
4602 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4603 match(ConD);
4604 op_cost(0);
4605 format %{ %}
4606 interface(CONST_INTER);
4607 %}
4608
4609 // Float Immediate
4610 operand immF()
4611 %{
4612 match(ConF);
4613 op_cost(0);
4614 format %{ %}
4615 interface(CONST_INTER);
4616 %}
4617
4618 // Float Immediate: +0.0f.
4619 operand immF0()
4620 %{
4621 predicate(jint_cast(n->getf()) == 0);
4622 match(ConF);
4623
4624 op_cost(0);
4625 format %{ %}
4626 interface(CONST_INTER);
4627 %}
4628
4629 // Half Float (FP16) Immediate
4630 operand immH()
4631 %{
4632 match(ConH);
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 //
4639 operand immFPacked()
4640 %{
4641 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4642 match(ConF);
4643 op_cost(0);
4644 format %{ %}
4645 interface(CONST_INTER);
4646 %}
4647
4648 // Narrow pointer operands
4649 // Narrow Pointer Immediate
4650 operand immN()
4651 %{
4652 match(ConN);
4653
4654 op_cost(0);
4655 format %{ %}
4656 interface(CONST_INTER);
4657 %}
4658
4659 // Narrow nullptr Pointer Immediate
4660 operand immN0()
4661 %{
4662 predicate(n->get_narrowcon() == 0);
4663 match(ConN);
4664
4665 op_cost(0);
4666 format %{ %}
4667 interface(CONST_INTER);
4668 %}
4669
4670 operand immNKlass()
4671 %{
4672 match(ConNKlass);
4673
4674 op_cost(0);
4675 format %{ %}
4676 interface(CONST_INTER);
4677 %}
4678
4679 // Integer 32 bit Register Operands
4680 // Integer 32 bitRegister (excludes SP)
4681 operand iRegI()
4682 %{
4683 constraint(ALLOC_IN_RC(any_reg32));
4684 match(RegI);
4685 match(iRegINoSp);
4686 op_cost(0);
4687 format %{ %}
4688 interface(REG_INTER);
4689 %}
4690
4691 // Integer 32 bit Register not Special
4692 operand iRegINoSp()
4693 %{
4694 constraint(ALLOC_IN_RC(no_special_reg32));
4695 match(RegI);
4696 op_cost(0);
4697 format %{ %}
4698 interface(REG_INTER);
4699 %}
4700
4701 // Integer 64 bit Register Operands
4702 // Integer 64 bit Register (includes SP)
4703 operand iRegL()
4704 %{
4705 constraint(ALLOC_IN_RC(any_reg));
4706 match(RegL);
4707 match(iRegLNoSp);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Integer 64 bit Register not Special
4714 operand iRegLNoSp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_reg));
4717 match(RegL);
4718 match(iRegL_R0);
4719 format %{ %}
4720 interface(REG_INTER);
4721 %}
4722
4723 // Pointer Register Operands
4724 // Pointer Register
4725 operand iRegP()
4726 %{
4727 constraint(ALLOC_IN_RC(ptr_reg));
4728 match(RegP);
4729 match(iRegPNoSp);
4730 match(iRegP_R0);
4731 //match(iRegP_R2);
4732 //match(iRegP_R4);
4733 match(iRegP_R5);
4734 match(thread_RegP);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Pointer 64 bit Register not Special
4741 operand iRegPNoSp()
4742 %{
4743 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4744 match(RegP);
4745 // match(iRegP);
4746 // match(iRegP_R0);
4747 // match(iRegP_R2);
4748 // match(iRegP_R4);
4749 // match(iRegP_R5);
4750 // match(thread_RegP);
4751 op_cost(0);
4752 format %{ %}
4753 interface(REG_INTER);
4754 %}
4755
4756 // This operand is not allowed to use rfp even if
4757 // rfp is not used to hold the frame pointer.
4758 operand iRegPNoSpNoRfp()
4759 %{
4760 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4761 match(RegP);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Pointer 64 bit Register R0 only
4769 operand iRegP_R0()
4770 %{
4771 constraint(ALLOC_IN_RC(r0_reg));
4772 match(RegP);
4773 // match(iRegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R1 only
4781 operand iRegP_R1()
4782 %{
4783 constraint(ALLOC_IN_RC(r1_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R2 only
4793 operand iRegP_R2()
4794 %{
4795 constraint(ALLOC_IN_RC(r2_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Pointer 64 bit Register R3 only
4805 operand iRegP_R3()
4806 %{
4807 constraint(ALLOC_IN_RC(r3_reg));
4808 match(RegP);
4809 // match(iRegP);
4810 match(iRegPNoSp);
4811 op_cost(0);
4812 format %{ %}
4813 interface(REG_INTER);
4814 %}
4815
4816 // Pointer 64 bit Register R4 only
4817 operand iRegP_R4()
4818 %{
4819 constraint(ALLOC_IN_RC(r4_reg));
4820 match(RegP);
4821 // match(iRegP);
4822 match(iRegPNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Pointer 64 bit Register R5 only
4829 operand iRegP_R5()
4830 %{
4831 constraint(ALLOC_IN_RC(r5_reg));
4832 match(RegP);
4833 // match(iRegP);
4834 match(iRegPNoSp);
4835 op_cost(0);
4836 format %{ %}
4837 interface(REG_INTER);
4838 %}
4839
4840 // Pointer 64 bit Register R10 only
4841 operand iRegP_R10()
4842 %{
4843 constraint(ALLOC_IN_RC(r10_reg));
4844 match(RegP);
4845 // match(iRegP);
4846 match(iRegPNoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852 // Long 64 bit Register R0 only
4853 operand iRegL_R0()
4854 %{
4855 constraint(ALLOC_IN_RC(r0_reg));
4856 match(RegL);
4857 match(iRegLNoSp);
4858 op_cost(0);
4859 format %{ %}
4860 interface(REG_INTER);
4861 %}
4862
4863 // Long 64 bit Register R11 only
4864 operand iRegL_R11()
4865 %{
4866 constraint(ALLOC_IN_RC(r11_reg));
4867 match(RegL);
4868 match(iRegLNoSp);
4869 op_cost(0);
4870 format %{ %}
4871 interface(REG_INTER);
4872 %}
4873
4874 // Register R0 only
4875 operand iRegI_R0()
4876 %{
4877 constraint(ALLOC_IN_RC(int_r0_reg));
4878 match(RegI);
4879 match(iRegINoSp);
4880 op_cost(0);
4881 format %{ %}
4882 interface(REG_INTER);
4883 %}
4884
4885 // Register R2 only
4886 operand iRegI_R2()
4887 %{
4888 constraint(ALLOC_IN_RC(int_r2_reg));
4889 match(RegI);
4890 match(iRegINoSp);
4891 op_cost(0);
4892 format %{ %}
4893 interface(REG_INTER);
4894 %}
4895
4896 // Register R3 only
4897 operand iRegI_R3()
4898 %{
4899 constraint(ALLOC_IN_RC(int_r3_reg));
4900 match(RegI);
4901 match(iRegINoSp);
4902 op_cost(0);
4903 format %{ %}
4904 interface(REG_INTER);
4905 %}
4906
4907
4908 // Register R4 only
4909 operand iRegI_R4()
4910 %{
4911 constraint(ALLOC_IN_RC(int_r4_reg));
4912 match(RegI);
4913 match(iRegINoSp);
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919
4920 // Pointer Register Operands
4921 // Narrow Pointer Register
4922 operand iRegN()
4923 %{
4924 constraint(ALLOC_IN_RC(any_reg32));
4925 match(RegN);
4926 match(iRegNNoSp);
4927 op_cost(0);
4928 format %{ %}
4929 interface(REG_INTER);
4930 %}
4931
4932 // Integer 64 bit Register not Special
4933 operand iRegNNoSp()
4934 %{
4935 constraint(ALLOC_IN_RC(no_special_reg32));
4936 match(RegN);
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 // Float Register
4943 // Float register operands
4944 operand vRegF()
4945 %{
4946 constraint(ALLOC_IN_RC(float_reg));
4947 match(RegF);
4948
4949 op_cost(0);
4950 format %{ %}
4951 interface(REG_INTER);
4952 %}
4953
4954 // Double Register
4955 // Double register operands
4956 operand vRegD()
4957 %{
4958 constraint(ALLOC_IN_RC(double_reg));
4959 match(RegD);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 // Generic vector class. This will be used for
4967 // all vector operands, including NEON and SVE.
4968 operand vReg()
4969 %{
4970 constraint(ALLOC_IN_RC(dynamic));
4971 match(VecA);
4972 match(VecD);
4973 match(VecX);
4974
4975 op_cost(0);
4976 format %{ %}
4977 interface(REG_INTER);
4978 %}
4979
4980 operand vecA()
4981 %{
4982 constraint(ALLOC_IN_RC(vectora_reg));
4983 match(VecA);
4984
4985 op_cost(0);
4986 format %{ %}
4987 interface(REG_INTER);
4988 %}
4989
4990 operand vecD()
4991 %{
4992 constraint(ALLOC_IN_RC(vectord_reg));
4993 match(VecD);
4994
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vecX()
5001 %{
5002 constraint(ALLOC_IN_RC(vectorx_reg));
5003 match(VecX);
5004
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 operand vRegD_V0()
5011 %{
5012 constraint(ALLOC_IN_RC(v0_reg));
5013 match(RegD);
5014 op_cost(0);
5015 format %{ %}
5016 interface(REG_INTER);
5017 %}
5018
5019 operand vRegD_V1()
5020 %{
5021 constraint(ALLOC_IN_RC(v1_reg));
5022 match(RegD);
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vRegD_V2()
5029 %{
5030 constraint(ALLOC_IN_RC(v2_reg));
5031 match(RegD);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vRegD_V3()
5038 %{
5039 constraint(ALLOC_IN_RC(v3_reg));
5040 match(RegD);
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V4()
5047 %{
5048 constraint(ALLOC_IN_RC(v4_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V5()
5056 %{
5057 constraint(ALLOC_IN_RC(v5_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V6()
5065 %{
5066 constraint(ALLOC_IN_RC(v6_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand vRegD_V7()
5074 %{
5075 constraint(ALLOC_IN_RC(v7_reg));
5076 match(RegD);
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V12()
5083 %{
5084 constraint(ALLOC_IN_RC(v12_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V13()
5092 %{
5093 constraint(ALLOC_IN_RC(v13_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand pReg()
5101 %{
5102 constraint(ALLOC_IN_RC(pr_reg));
5103 match(RegVectMask);
5104 match(pRegGov);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 operand pRegGov()
5111 %{
5112 constraint(ALLOC_IN_RC(gov_pr));
5113 match(RegVectMask);
5114 match(pReg);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand pRegGov_P0()
5121 %{
5122 constraint(ALLOC_IN_RC(p0_reg));
5123 match(RegVectMask);
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand pRegGov_P1()
5130 %{
5131 constraint(ALLOC_IN_RC(p1_reg));
5132 match(RegVectMask);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 // Flags register, used as output of signed compare instructions
5139
5140 // note that on AArch64 we also use this register as the output for
5141 // for floating point compare instructions (CmpF CmpD). this ensures
5142 // that ordered inequality tests use GT, GE, LT or LE none of which
5143 // pass through cases where the result is unordered i.e. one or both
5144 // inputs to the compare is a NaN. this means that the ideal code can
5145 // replace e.g. a GT with an LE and not end up capturing the NaN case
5146 // (where the comparison should always fail). EQ and NE tests are
5147 // always generated in ideal code so that unordered folds into the NE
5148 // case, matching the behaviour of AArch64 NE.
5149 //
5150 // This differs from x86 where the outputs of FP compares use a
5151 // special FP flags registers and where compares based on this
5152 // register are distinguished into ordered inequalities (cmpOpUCF) and
5153 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5154 // to explicitly handle the unordered case in branches. x86 also has
5155 // to include extra CMoveX rules to accept a cmpOpUCF input.
5156
5157 operand rFlagsReg()
5158 %{
5159 constraint(ALLOC_IN_RC(int_flags));
5160 match(RegFlags);
5161
5162 op_cost(0);
5163 format %{ "RFLAGS" %}
5164 interface(REG_INTER);
5165 %}
5166
5167 // Flags register, used as output of unsigned compare instructions
5168 operand rFlagsRegU()
5169 %{
5170 constraint(ALLOC_IN_RC(int_flags));
5171 match(RegFlags);
5172
5173 op_cost(0);
5174 format %{ "RFLAGSU" %}
5175 interface(REG_INTER);
5176 %}
5177
5178 // Special Registers
5179
5180 // Method Register
5181 operand inline_cache_RegP(iRegP reg)
5182 %{
5183 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5184 match(reg);
5185 match(iRegPNoSp);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 // Thread Register
5192 operand thread_RegP(iRegP reg)
5193 %{
5194 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5195 match(reg);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 //----------Memory Operands----------------------------------------------------
5202
5203 operand indirect(iRegP reg)
5204 %{
5205 constraint(ALLOC_IN_RC(ptr_reg));
5206 match(reg);
5207 op_cost(0);
5208 format %{ "[$reg]" %}
5209 interface(MEMORY_INTER) %{
5210 base($reg);
5211 index(0xffffffff);
5212 scale(0x0);
5213 disp(0x0);
5214 %}
5215 %}
5216
5217 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5218 %{
5219 constraint(ALLOC_IN_RC(ptr_reg));
5220 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5221 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5222 op_cost(0);
5223 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5224 interface(MEMORY_INTER) %{
5225 base($reg);
5226 index($ireg);
5227 scale($scale);
5228 disp(0x0);
5229 %}
5230 %}
5231
5232 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5233 %{
5234 constraint(ALLOC_IN_RC(ptr_reg));
5235 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5236 match(AddP reg (LShiftL lreg scale));
5237 op_cost(0);
5238 format %{ "$reg, $lreg lsl($scale)" %}
5239 interface(MEMORY_INTER) %{
5240 base($reg);
5241 index($lreg);
5242 scale($scale);
5243 disp(0x0);
5244 %}
5245 %}
5246
5247 operand indIndexI2L(iRegP reg, iRegI ireg)
5248 %{
5249 constraint(ALLOC_IN_RC(ptr_reg));
5250 match(AddP reg (ConvI2L ireg));
5251 op_cost(0);
5252 format %{ "$reg, $ireg, 0, I2L" %}
5253 interface(MEMORY_INTER) %{
5254 base($reg);
5255 index($ireg);
5256 scale(0x0);
5257 disp(0x0);
5258 %}
5259 %}
5260
5261 operand indIndex(iRegP reg, iRegL lreg)
5262 %{
5263 constraint(ALLOC_IN_RC(ptr_reg));
5264 match(AddP reg lreg);
5265 op_cost(0);
5266 format %{ "$reg, $lreg" %}
5267 interface(MEMORY_INTER) %{
5268 base($reg);
5269 index($lreg);
5270 scale(0x0);
5271 disp(0x0);
5272 %}
5273 %}
5274
5275 operand indOffI1(iRegP reg, immIOffset1 off)
5276 %{
5277 constraint(ALLOC_IN_RC(ptr_reg));
5278 match(AddP reg off);
5279 op_cost(0);
5280 format %{ "[$reg, $off]" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index(0xffffffff);
5284 scale(0x0);
5285 disp($off);
5286 %}
5287 %}
5288
5289 operand indOffI2(iRegP reg, immIOffset2 off)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 match(AddP reg off);
5293 op_cost(0);
5294 format %{ "[$reg, $off]" %}
5295 interface(MEMORY_INTER) %{
5296 base($reg);
5297 index(0xffffffff);
5298 scale(0x0);
5299 disp($off);
5300 %}
5301 %}
5302
5303 operand indOffI4(iRegP reg, immIOffset4 off)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(AddP reg off);
5307 op_cost(0);
5308 format %{ "[$reg, $off]" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index(0xffffffff);
5312 scale(0x0);
5313 disp($off);
5314 %}
5315 %}
5316
5317 operand indOffI8(iRegP reg, immIOffset8 off)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 match(AddP reg off);
5321 op_cost(0);
5322 format %{ "[$reg, $off]" %}
5323 interface(MEMORY_INTER) %{
5324 base($reg);
5325 index(0xffffffff);
5326 scale(0x0);
5327 disp($off);
5328 %}
5329 %}
5330
5331 operand indOffI16(iRegP reg, immIOffset16 off)
5332 %{
5333 constraint(ALLOC_IN_RC(ptr_reg));
5334 match(AddP reg off);
5335 op_cost(0);
5336 format %{ "[$reg, $off]" %}
5337 interface(MEMORY_INTER) %{
5338 base($reg);
5339 index(0xffffffff);
5340 scale(0x0);
5341 disp($off);
5342 %}
5343 %}
5344
5345 operand indOffL1(iRegP reg, immLoffset1 off)
5346 %{
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(AddP reg off);
5349 op_cost(0);
5350 format %{ "[$reg, $off]" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index(0xffffffff);
5354 scale(0x0);
5355 disp($off);
5356 %}
5357 %}
5358
5359 operand indOffL2(iRegP reg, immLoffset2 off)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg off);
5363 op_cost(0);
5364 format %{ "[$reg, $off]" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index(0xffffffff);
5368 scale(0x0);
5369 disp($off);
5370 %}
5371 %}
5372
5373 operand indOffL4(iRegP reg, immLoffset4 off)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg off);
5377 op_cost(0);
5378 format %{ "[$reg, $off]" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index(0xffffffff);
5382 scale(0x0);
5383 disp($off);
5384 %}
5385 %}
5386
5387 operand indOffL8(iRegP reg, immLoffset8 off)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(AddP reg off);
5391 op_cost(0);
5392 format %{ "[$reg, $off]" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp($off);
5398 %}
5399 %}
5400
5401 operand indOffL16(iRegP reg, immLoffset16 off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP reg off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indirectX2P(iRegL reg)
5416 %{
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(CastX2P reg);
5419 op_cost(0);
5420 format %{ "[$reg]\t# long -> ptr" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp(0x0);
5426 %}
5427 %}
5428
5429 operand indOffX2P(iRegL reg, immLOffset off)
5430 %{
5431 constraint(ALLOC_IN_RC(ptr_reg));
5432 match(AddP (CastX2P reg) off);
5433 op_cost(0);
5434 format %{ "[$reg, $off]\t# long -> ptr" %}
5435 interface(MEMORY_INTER) %{
5436 base($reg);
5437 index(0xffffffff);
5438 scale(0x0);
5439 disp($off);
5440 %}
5441 %}
5442
5443 operand indirectN(iRegN reg)
5444 %{
5445 predicate(CompressedOops::shift() == 0);
5446 constraint(ALLOC_IN_RC(ptr_reg));
5447 match(DecodeN reg);
5448 op_cost(0);
5449 format %{ "[$reg]\t# narrow" %}
5450 interface(MEMORY_INTER) %{
5451 base($reg);
5452 index(0xffffffff);
5453 scale(0x0);
5454 disp(0x0);
5455 %}
5456 %}
5457
5458 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5459 %{
5460 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5463 op_cost(0);
5464 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index($ireg);
5468 scale($scale);
5469 disp(0x0);
5470 %}
5471 %}
5472
5473 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5474 %{
5475 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5476 constraint(ALLOC_IN_RC(ptr_reg));
5477 match(AddP (DecodeN reg) (LShiftL lreg scale));
5478 op_cost(0);
5479 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5480 interface(MEMORY_INTER) %{
5481 base($reg);
5482 index($lreg);
5483 scale($scale);
5484 disp(0x0);
5485 %}
5486 %}
5487
5488 operand indIndexI2LN(iRegN reg, iRegI ireg)
5489 %{
5490 predicate(CompressedOops::shift() == 0);
5491 constraint(ALLOC_IN_RC(ptr_reg));
5492 match(AddP (DecodeN reg) (ConvI2L ireg));
5493 op_cost(0);
5494 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5495 interface(MEMORY_INTER) %{
5496 base($reg);
5497 index($ireg);
5498 scale(0x0);
5499 disp(0x0);
5500 %}
5501 %}
5502
5503 operand indIndexN(iRegN reg, iRegL lreg)
5504 %{
5505 predicate(CompressedOops::shift() == 0);
5506 constraint(ALLOC_IN_RC(ptr_reg));
5507 match(AddP (DecodeN reg) lreg);
5508 op_cost(0);
5509 format %{ "$reg, $lreg\t# narrow" %}
5510 interface(MEMORY_INTER) %{
5511 base($reg);
5512 index($lreg);
5513 scale(0x0);
5514 disp(0x0);
5515 %}
5516 %}
5517
5518 operand indOffIN(iRegN reg, immIOffset off)
5519 %{
5520 predicate(CompressedOops::shift() == 0);
5521 constraint(ALLOC_IN_RC(ptr_reg));
5522 match(AddP (DecodeN reg) off);
5523 op_cost(0);
5524 format %{ "[$reg, $off]\t# narrow" %}
5525 interface(MEMORY_INTER) %{
5526 base($reg);
5527 index(0xffffffff);
5528 scale(0x0);
5529 disp($off);
5530 %}
5531 %}
5532
5533 operand indOffLN(iRegN reg, immLOffset off)
5534 %{
5535 predicate(CompressedOops::shift() == 0);
5536 constraint(ALLOC_IN_RC(ptr_reg));
5537 match(AddP (DecodeN reg) off);
5538 op_cost(0);
5539 format %{ "[$reg, $off]\t# narrow" %}
5540 interface(MEMORY_INTER) %{
5541 base($reg);
5542 index(0xffffffff);
5543 scale(0x0);
5544 disp($off);
5545 %}
5546 %}
5547
5548
5549 //----------Special Memory Operands--------------------------------------------
5550 // Stack Slot Operand - This operand is used for loading and storing temporary
5551 // values on the stack where a match requires a value to
5552 // flow through memory.
5553 operand stackSlotP(sRegP reg)
5554 %{
5555 constraint(ALLOC_IN_RC(stack_slots));
5556 op_cost(100);
5557 // No match rule because this operand is only generated in matching
5558 // match(RegP);
5559 format %{ "[$reg]" %}
5560 interface(MEMORY_INTER) %{
5561 base(0x1e); // RSP
5562 index(0x0); // No Index
5563 scale(0x0); // No Scale
5564 disp($reg); // Stack Offset
5565 %}
5566 %}
5567
5568 operand stackSlotI(sRegI reg)
5569 %{
5570 constraint(ALLOC_IN_RC(stack_slots));
5571 // No match rule because this operand is only generated in matching
5572 // match(RegI);
5573 format %{ "[$reg]" %}
5574 interface(MEMORY_INTER) %{
5575 base(0x1e); // RSP
5576 index(0x0); // No Index
5577 scale(0x0); // No Scale
5578 disp($reg); // Stack Offset
5579 %}
5580 %}
5581
5582 operand stackSlotF(sRegF reg)
5583 %{
5584 constraint(ALLOC_IN_RC(stack_slots));
5585 // No match rule because this operand is only generated in matching
5586 // match(RegF);
5587 format %{ "[$reg]" %}
5588 interface(MEMORY_INTER) %{
5589 base(0x1e); // RSP
5590 index(0x0); // No Index
5591 scale(0x0); // No Scale
5592 disp($reg); // Stack Offset
5593 %}
5594 %}
5595
5596 operand stackSlotD(sRegD reg)
5597 %{
5598 constraint(ALLOC_IN_RC(stack_slots));
5599 // No match rule because this operand is only generated in matching
5600 // match(RegD);
5601 format %{ "[$reg]" %}
5602 interface(MEMORY_INTER) %{
5603 base(0x1e); // RSP
5604 index(0x0); // No Index
5605 scale(0x0); // No Scale
5606 disp($reg); // Stack Offset
5607 %}
5608 %}
5609
5610 operand stackSlotL(sRegL reg)
5611 %{
5612 constraint(ALLOC_IN_RC(stack_slots));
5613 // No match rule because this operand is only generated in matching
5614 // match(RegL);
5615 format %{ "[$reg]" %}
5616 interface(MEMORY_INTER) %{
5617 base(0x1e); // RSP
5618 index(0x0); // No Index
5619 scale(0x0); // No Scale
5620 disp($reg); // Stack Offset
5621 %}
5622 %}
5623
5624 // Operands for expressing Control Flow
5625 // NOTE: Label is a predefined operand which should not be redefined in
5626 // the AD file. It is generically handled within the ADLC.
5627
5628 //----------Conditional Branch Operands----------------------------------------
5629 // Comparison Op - This is the operation of the comparison, and is limited to
5630 // the following set of codes:
5631 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5632 //
5633 // Other attributes of the comparison, such as unsignedness, are specified
5634 // by the comparison instruction that sets a condition code flags register.
5635 // That result is represented by a flags operand whose subtype is appropriate
5636 // to the unsignedness (etc.) of the comparison.
5637 //
5638 // Later, the instruction which matches both the Comparison Op (a Bool) and
5639 // the flags (produced by the Cmp) specifies the coding of the comparison op
5640 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5641
5642 // used for signed integral comparisons and fp comparisons
5643
5644 operand cmpOp()
5645 %{
5646 match(Bool);
5647
5648 format %{ "" %}
5649 interface(COND_INTER) %{
5650 equal(0x0, "eq");
5651 not_equal(0x1, "ne");
5652 less(0xb, "lt");
5653 greater_equal(0xa, "ge");
5654 less_equal(0xd, "le");
5655 greater(0xc, "gt");
5656 overflow(0x6, "vs");
5657 no_overflow(0x7, "vc");
5658 %}
5659 %}
5660
5661 // used for unsigned integral comparisons
5662
5663 operand cmpOpU()
5664 %{
5665 match(Bool);
5666
5667 format %{ "" %}
5668 interface(COND_INTER) %{
5669 equal(0x0, "eq");
5670 not_equal(0x1, "ne");
5671 less(0x3, "lo");
5672 greater_equal(0x2, "hs");
5673 less_equal(0x9, "ls");
5674 greater(0x8, "hi");
5675 overflow(0x6, "vs");
5676 no_overflow(0x7, "vc");
5677 %}
5678 %}
5679
5680 // used for certain integral comparisons which can be
5681 // converted to cbxx or tbxx instructions
5682
5683 operand cmpOpEqNe()
5684 %{
5685 match(Bool);
5686 op_cost(0);
5687 predicate(n->as_Bool()->_test._test == BoolTest::ne
5688 || n->as_Bool()->_test._test == BoolTest::eq);
5689
5690 format %{ "" %}
5691 interface(COND_INTER) %{
5692 equal(0x0, "eq");
5693 not_equal(0x1, "ne");
5694 less(0xb, "lt");
5695 greater_equal(0xa, "ge");
5696 less_equal(0xd, "le");
5697 greater(0xc, "gt");
5698 overflow(0x6, "vs");
5699 no_overflow(0x7, "vc");
5700 %}
5701 %}
5702
5703 // used for certain integral comparisons which can be
5704 // converted to cbxx or tbxx instructions
5705
5706 operand cmpOpLtGe()
5707 %{
5708 match(Bool);
5709 op_cost(0);
5710
5711 predicate(n->as_Bool()->_test._test == BoolTest::lt
5712 || n->as_Bool()->_test._test == BoolTest::ge);
5713
5714 format %{ "" %}
5715 interface(COND_INTER) %{
5716 equal(0x0, "eq");
5717 not_equal(0x1, "ne");
5718 less(0xb, "lt");
5719 greater_equal(0xa, "ge");
5720 less_equal(0xd, "le");
5721 greater(0xc, "gt");
5722 overflow(0x6, "vs");
5723 no_overflow(0x7, "vc");
5724 %}
5725 %}
5726
5727 // used for certain unsigned integral comparisons which can be
5728 // converted to cbxx or tbxx instructions
5729
5730 operand cmpOpUEqNeLeGt()
5731 %{
5732 match(Bool);
5733 op_cost(0);
5734
5735 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5736 n->as_Bool()->_test._test == BoolTest::ne ||
5737 n->as_Bool()->_test._test == BoolTest::le ||
5738 n->as_Bool()->_test._test == BoolTest::gt);
5739
5740 format %{ "" %}
5741 interface(COND_INTER) %{
5742 equal(0x0, "eq");
5743 not_equal(0x1, "ne");
5744 less(0x3, "lo");
5745 greater_equal(0x2, "hs");
5746 less_equal(0x9, "ls");
5747 greater(0x8, "hi");
5748 overflow(0x6, "vs");
5749 no_overflow(0x7, "vc");
5750 %}
5751 %}
5752
5753 // Special operand allowing long args to int ops to be truncated for free
5754
5755 operand iRegL2I(iRegL reg) %{
5756
5757 op_cost(0);
5758
5759 match(ConvL2I reg);
5760
5761 format %{ "l2i($reg)" %}
5762
5763 interface(REG_INTER)
5764 %}
5765
5766 operand iRegL2P(iRegL reg) %{
5767
5768 op_cost(0);
5769
5770 match(CastX2P reg);
5771
5772 format %{ "l2p($reg)" %}
5773
5774 interface(REG_INTER)
5775 %}
5776
5777 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5778 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5779 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5780 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5781
5782 //----------OPERAND CLASSES----------------------------------------------------
5783 // Operand Classes are groups of operands that are used as to simplify
5784 // instruction definitions by not requiring the AD writer to specify
5785 // separate instructions for every form of operand when the
5786 // instruction accepts multiple operand types with the same basic
5787 // encoding and format. The classic case of this is memory operands.
5788
5789 // memory is used to define read/write location for load/store
5790 // instruction defs. we can turn a memory op into an Address
5791
5792 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5793 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5794
5795 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5796 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5797
5798 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5799 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5800
5801 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5802 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5803
5804 // All of the memory operands. For the pipeline description.
5805 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5806 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5807 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5808
5809
5810 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5811 // operations. it allows the src to be either an iRegI or a (ConvL2I
5812 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5813 // can be elided because the 32-bit instruction will just employ the
5814 // lower 32 bits anyway.
5815 //
5816 // n.b. this does not elide all L2I conversions. if the truncated
5817 // value is consumed by more than one operation then the ConvL2I
5818 // cannot be bundled into the consuming nodes so an l2i gets planted
5819 // (actually a movw $dst $src) and the downstream instructions consume
5820 // the result of the l2i as an iRegI input. That's a shame since the
5821 // movw is actually redundant but its not too costly.
5822
5823 opclass iRegIorL2I(iRegI, iRegL2I);
5824 opclass iRegPorL2P(iRegP, iRegL2P);
5825
5826 //----------PIPELINE-----------------------------------------------------------
5827 // Rules which define the behavior of the target architectures pipeline.
5828
5829 // For specific pipelines, eg A53, define the stages of that pipeline
5830 //pipe_desc(ISS, EX1, EX2, WR);
5831 #define ISS S0
5832 #define EX1 S1
5833 #define EX2 S2
5834 #define WR S3
5835
5836 // Integer ALU reg operation
5837 pipeline %{
5838
5839 attributes %{
5840 // ARM instructions are of fixed length
5841 fixed_size_instructions; // Fixed size instructions TODO does
5842 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5843 // ARM instructions come in 32-bit word units
5844 instruction_unit_size = 4; // An instruction is 4 bytes long
5845 instruction_fetch_unit_size = 64; // The processor fetches one line
5846 instruction_fetch_units = 1; // of 64 bytes
5847
5848 // List of nop instructions
5849 nops( MachNop );
5850 %}
5851
5852 // We don't use an actual pipeline model so don't care about resources
5853 // or description. we do use pipeline classes to introduce fixed
5854 // latencies
5855
5856 //----------RESOURCES----------------------------------------------------------
5857 // Resources are the functional units available to the machine
5858
5859 resources( INS0, INS1, INS01 = INS0 | INS1,
5860 ALU0, ALU1, ALU = ALU0 | ALU1,
5861 MAC,
5862 DIV,
5863 BRANCH,
5864 LDST,
5865 NEON_FP);
5866
5867 //----------PIPELINE DESCRIPTION-----------------------------------------------
5868 // Pipeline Description specifies the stages in the machine's pipeline
5869
5870 // Define the pipeline as a generic 6 stage pipeline
5871 pipe_desc(S0, S1, S2, S3, S4, S5);
5872
5873 //----------PIPELINE CLASSES---------------------------------------------------
5874 // Pipeline Classes describe the stages in which input and output are
5875 // referenced by the hardware pipeline.
5876
5877 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5878 %{
5879 single_instruction;
5880 src1 : S1(read);
5881 src2 : S2(read);
5882 dst : S5(write);
5883 INS01 : ISS;
5884 NEON_FP : S5;
5885 %}
5886
5887 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5888 %{
5889 single_instruction;
5890 src1 : S1(read);
5891 src2 : S2(read);
5892 dst : S5(write);
5893 INS01 : ISS;
5894 NEON_FP : S5;
5895 %}
5896
5897 pipe_class fp_uop_s(vRegF dst, vRegF src)
5898 %{
5899 single_instruction;
5900 src : S1(read);
5901 dst : S5(write);
5902 INS01 : ISS;
5903 NEON_FP : S5;
5904 %}
5905
5906 pipe_class fp_uop_d(vRegD dst, vRegD src)
5907 %{
5908 single_instruction;
5909 src : S1(read);
5910 dst : S5(write);
5911 INS01 : ISS;
5912 NEON_FP : S5;
5913 %}
5914
5915 pipe_class fp_d2f(vRegF dst, vRegD src)
5916 %{
5917 single_instruction;
5918 src : S1(read);
5919 dst : S5(write);
5920 INS01 : ISS;
5921 NEON_FP : S5;
5922 %}
5923
5924 pipe_class fp_f2d(vRegD dst, vRegF src)
5925 %{
5926 single_instruction;
5927 src : S1(read);
5928 dst : S5(write);
5929 INS01 : ISS;
5930 NEON_FP : S5;
5931 %}
5932
5933 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5934 %{
5935 single_instruction;
5936 src : S1(read);
5937 dst : S5(write);
5938 INS01 : ISS;
5939 NEON_FP : S5;
5940 %}
5941
5942 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5943 %{
5944 single_instruction;
5945 src : S1(read);
5946 dst : S5(write);
5947 INS01 : ISS;
5948 NEON_FP : S5;
5949 %}
5950
5951 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5952 %{
5953 single_instruction;
5954 src : S1(read);
5955 dst : S5(write);
5956 INS01 : ISS;
5957 NEON_FP : S5;
5958 %}
5959
5960 pipe_class fp_l2f(vRegF dst, iRegL src)
5961 %{
5962 single_instruction;
5963 src : S1(read);
5964 dst : S5(write);
5965 INS01 : ISS;
5966 NEON_FP : S5;
5967 %}
5968
5969 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5970 %{
5971 single_instruction;
5972 src : S1(read);
5973 dst : S5(write);
5974 INS01 : ISS;
5975 NEON_FP : S5;
5976 %}
5977
5978 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5979 %{
5980 single_instruction;
5981 src : S1(read);
5982 dst : S5(write);
5983 INS01 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5988 %{
5989 single_instruction;
5990 src : S1(read);
5991 dst : S5(write);
5992 INS01 : ISS;
5993 NEON_FP : S5;
5994 %}
5995
5996 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5997 %{
5998 single_instruction;
5999 src : S1(read);
6000 dst : S5(write);
6001 INS01 : ISS;
6002 NEON_FP : S5;
6003 %}
6004
6005 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6006 %{
6007 single_instruction;
6008 src1 : S1(read);
6009 src2 : S2(read);
6010 dst : S5(write);
6011 INS0 : ISS;
6012 NEON_FP : S5;
6013 %}
6014
6015 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6016 %{
6017 single_instruction;
6018 src1 : S1(read);
6019 src2 : S2(read);
6020 dst : S5(write);
6021 INS0 : ISS;
6022 NEON_FP : S5;
6023 %}
6024
6025 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6026 %{
6027 single_instruction;
6028 cr : S1(read);
6029 src1 : S1(read);
6030 src2 : S1(read);
6031 dst : S3(write);
6032 INS01 : ISS;
6033 NEON_FP : S3;
6034 %}
6035
6036 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6037 %{
6038 single_instruction;
6039 cr : S1(read);
6040 src1 : S1(read);
6041 src2 : S1(read);
6042 dst : S3(write);
6043 INS01 : ISS;
6044 NEON_FP : S3;
6045 %}
6046
6047 pipe_class fp_imm_s(vRegF dst)
6048 %{
6049 single_instruction;
6050 dst : S3(write);
6051 INS01 : ISS;
6052 NEON_FP : S3;
6053 %}
6054
6055 pipe_class fp_imm_d(vRegD dst)
6056 %{
6057 single_instruction;
6058 dst : S3(write);
6059 INS01 : ISS;
6060 NEON_FP : S3;
6061 %}
6062
6063 pipe_class fp_load_constant_s(vRegF dst)
6064 %{
6065 single_instruction;
6066 dst : S4(write);
6067 INS01 : ISS;
6068 NEON_FP : S4;
6069 %}
6070
6071 pipe_class fp_load_constant_d(vRegD dst)
6072 %{
6073 single_instruction;
6074 dst : S4(write);
6075 INS01 : ISS;
6076 NEON_FP : S4;
6077 %}
6078
6079 //------- Integer ALU operations --------------------------
6080
6081 // Integer ALU reg-reg operation
6082 // Operands needed in EX1, result generated in EX2
6083 // Eg. ADD x0, x1, x2
6084 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6085 %{
6086 single_instruction;
6087 dst : EX2(write);
6088 src1 : EX1(read);
6089 src2 : EX1(read);
6090 INS01 : ISS; // Dual issue as instruction 0 or 1
6091 ALU : EX2;
6092 %}
6093
6094 // Integer ALU reg-reg operation with constant shift
6095 // Shifted register must be available in LATE_ISS instead of EX1
6096 // Eg. ADD x0, x1, x2, LSL #2
6097 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6098 %{
6099 single_instruction;
6100 dst : EX2(write);
6101 src1 : EX1(read);
6102 src2 : ISS(read);
6103 INS01 : ISS;
6104 ALU : EX2;
6105 %}
6106
6107 // Integer ALU reg operation with constant shift
6108 // Eg. LSL x0, x1, #shift
6109 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6110 %{
6111 single_instruction;
6112 dst : EX2(write);
6113 src1 : ISS(read);
6114 INS01 : ISS;
6115 ALU : EX2;
6116 %}
6117
6118 // Integer ALU reg-reg operation with variable shift
6119 // Both operands must be available in LATE_ISS instead of EX1
6120 // Result is available in EX1 instead of EX2
6121 // Eg. LSLV x0, x1, x2
6122 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6123 %{
6124 single_instruction;
6125 dst : EX1(write);
6126 src1 : ISS(read);
6127 src2 : ISS(read);
6128 INS01 : ISS;
6129 ALU : EX1;
6130 %}
6131
6132 // Integer ALU reg-reg operation with extract
6133 // As for _vshift above, but result generated in EX2
6134 // Eg. EXTR x0, x1, x2, #N
6135 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6136 %{
6137 single_instruction;
6138 dst : EX2(write);
6139 src1 : ISS(read);
6140 src2 : ISS(read);
6141 INS1 : ISS; // Can only dual issue as Instruction 1
6142 ALU : EX1;
6143 %}
6144
6145 // Integer ALU reg operation
6146 // Eg. NEG x0, x1
6147 pipe_class ialu_reg(iRegI dst, iRegI src)
6148 %{
6149 single_instruction;
6150 dst : EX2(write);
6151 src : EX1(read);
6152 INS01 : ISS;
6153 ALU : EX2;
6154 %}
6155
6156 // Integer ALU reg mmediate operation
6157 // Eg. ADD x0, x1, #N
6158 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6159 %{
6160 single_instruction;
6161 dst : EX2(write);
6162 src1 : EX1(read);
6163 INS01 : ISS;
6164 ALU : EX2;
6165 %}
6166
6167 // Integer ALU immediate operation (no source operands)
6168 // Eg. MOV x0, #N
6169 pipe_class ialu_imm(iRegI dst)
6170 %{
6171 single_instruction;
6172 dst : EX1(write);
6173 INS01 : ISS;
6174 ALU : EX1;
6175 %}
6176
6177 //------- Compare operation -------------------------------
6178
6179 // Compare reg-reg
6180 // Eg. CMP x0, x1
6181 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6182 %{
6183 single_instruction;
6184 // fixed_latency(16);
6185 cr : EX2(write);
6186 op1 : EX1(read);
6187 op2 : EX1(read);
6188 INS01 : ISS;
6189 ALU : EX2;
6190 %}
6191
6192 // Compare reg-reg
6193 // Eg. CMP x0, #N
6194 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6195 %{
6196 single_instruction;
6197 // fixed_latency(16);
6198 cr : EX2(write);
6199 op1 : EX1(read);
6200 INS01 : ISS;
6201 ALU : EX2;
6202 %}
6203
6204 //------- Conditional instructions ------------------------
6205
6206 // Conditional no operands
6207 // Eg. CSINC x0, zr, zr, <cond>
6208 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6209 %{
6210 single_instruction;
6211 cr : EX1(read);
6212 dst : EX2(write);
6213 INS01 : ISS;
6214 ALU : EX2;
6215 %}
6216
6217 // Conditional 2 operand
6218 // EG. CSEL X0, X1, X2, <cond>
6219 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6220 %{
6221 single_instruction;
6222 cr : EX1(read);
6223 src1 : EX1(read);
6224 src2 : EX1(read);
6225 dst : EX2(write);
6226 INS01 : ISS;
6227 ALU : EX2;
6228 %}
6229
6230 // Conditional 2 operand
6231 // EG. CSEL X0, X1, X2, <cond>
6232 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6233 %{
6234 single_instruction;
6235 cr : EX1(read);
6236 src : EX1(read);
6237 dst : EX2(write);
6238 INS01 : ISS;
6239 ALU : EX2;
6240 %}
6241
6242 //------- Multiply pipeline operations --------------------
6243
6244 // Multiply reg-reg
6245 // Eg. MUL w0, w1, w2
6246 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6247 %{
6248 single_instruction;
6249 dst : WR(write);
6250 src1 : ISS(read);
6251 src2 : ISS(read);
6252 INS01 : ISS;
6253 MAC : WR;
6254 %}
6255
6256 // Multiply accumulate
6257 // Eg. MADD w0, w1, w2, w3
6258 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6259 %{
6260 single_instruction;
6261 dst : WR(write);
6262 src1 : ISS(read);
6263 src2 : ISS(read);
6264 src3 : ISS(read);
6265 INS01 : ISS;
6266 MAC : WR;
6267 %}
6268
6269 // Eg. MUL w0, w1, w2
6270 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6271 %{
6272 single_instruction;
6273 fixed_latency(3); // Maximum latency for 64 bit mul
6274 dst : WR(write);
6275 src1 : ISS(read);
6276 src2 : ISS(read);
6277 INS01 : ISS;
6278 MAC : WR;
6279 %}
6280
6281 // Multiply accumulate
6282 // Eg. MADD w0, w1, w2, w3
6283 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6284 %{
6285 single_instruction;
6286 fixed_latency(3); // Maximum latency for 64 bit mul
6287 dst : WR(write);
6288 src1 : ISS(read);
6289 src2 : ISS(read);
6290 src3 : ISS(read);
6291 INS01 : ISS;
6292 MAC : WR;
6293 %}
6294
6295 //------- Divide pipeline operations --------------------
6296
6297 // Eg. SDIV w0, w1, w2
6298 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6299 %{
6300 single_instruction;
6301 fixed_latency(8); // Maximum latency for 32 bit divide
6302 dst : WR(write);
6303 src1 : ISS(read);
6304 src2 : ISS(read);
6305 INS0 : ISS; // Can only dual issue as instruction 0
6306 DIV : WR;
6307 %}
6308
6309 // Eg. SDIV x0, x1, x2
6310 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6311 %{
6312 single_instruction;
6313 fixed_latency(16); // Maximum latency for 64 bit divide
6314 dst : WR(write);
6315 src1 : ISS(read);
6316 src2 : ISS(read);
6317 INS0 : ISS; // Can only dual issue as instruction 0
6318 DIV : WR;
6319 %}
6320
6321 //------- Load pipeline operations ------------------------
6322
6323 // Load - prefetch
6324 // Eg. PFRM <mem>
6325 pipe_class iload_prefetch(memory mem)
6326 %{
6327 single_instruction;
6328 mem : ISS(read);
6329 INS01 : ISS;
6330 LDST : WR;
6331 %}
6332
6333 // Load - reg, mem
6334 // Eg. LDR x0, <mem>
6335 pipe_class iload_reg_mem(iRegI dst, memory mem)
6336 %{
6337 single_instruction;
6338 dst : WR(write);
6339 mem : ISS(read);
6340 INS01 : ISS;
6341 LDST : WR;
6342 %}
6343
6344 // Load - reg, reg
6345 // Eg. LDR x0, [sp, x1]
6346 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6347 %{
6348 single_instruction;
6349 dst : WR(write);
6350 src : ISS(read);
6351 INS01 : ISS;
6352 LDST : WR;
6353 %}
6354
6355 //------- Store pipeline operations -----------------------
6356
6357 // Store - zr, mem
6358 // Eg. STR zr, <mem>
6359 pipe_class istore_mem(memory mem)
6360 %{
6361 single_instruction;
6362 mem : ISS(read);
6363 INS01 : ISS;
6364 LDST : WR;
6365 %}
6366
6367 // Store - reg, mem
6368 // Eg. STR x0, <mem>
6369 pipe_class istore_reg_mem(iRegI src, memory mem)
6370 %{
6371 single_instruction;
6372 mem : ISS(read);
6373 src : EX2(read);
6374 INS01 : ISS;
6375 LDST : WR;
6376 %}
6377
6378 // Store - reg, reg
6379 // Eg. STR x0, [sp, x1]
6380 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6381 %{
6382 single_instruction;
6383 dst : ISS(read);
6384 src : EX2(read);
6385 INS01 : ISS;
6386 LDST : WR;
6387 %}
6388
6389 //------- Store pipeline operations -----------------------
6390
6391 // Branch
6392 pipe_class pipe_branch()
6393 %{
6394 single_instruction;
6395 INS01 : ISS;
6396 BRANCH : EX1;
6397 %}
6398
6399 // Conditional branch
6400 pipe_class pipe_branch_cond(rFlagsReg cr)
6401 %{
6402 single_instruction;
6403 cr : EX1(read);
6404 INS01 : ISS;
6405 BRANCH : EX1;
6406 %}
6407
6408 // Compare & Branch
6409 // EG. CBZ/CBNZ
6410 pipe_class pipe_cmp_branch(iRegI op1)
6411 %{
6412 single_instruction;
6413 op1 : EX1(read);
6414 INS01 : ISS;
6415 BRANCH : EX1;
6416 %}
6417
6418 //------- Synchronisation operations ----------------------
6419
6420 // Any operation requiring serialization.
6421 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6422 pipe_class pipe_serial()
6423 %{
6424 single_instruction;
6425 force_serialization;
6426 fixed_latency(16);
6427 INS01 : ISS(2); // Cannot dual issue with any other instruction
6428 LDST : WR;
6429 %}
6430
6431 // Generic big/slow expanded idiom - also serialized
6432 pipe_class pipe_slow()
6433 %{
6434 instruction_count(10);
6435 multiple_bundles;
6436 force_serialization;
6437 fixed_latency(16);
6438 INS01 : ISS(2); // Cannot dual issue with any other instruction
6439 LDST : WR;
6440 %}
6441
6442 // Empty pipeline class
6443 pipe_class pipe_class_empty()
6444 %{
6445 single_instruction;
6446 fixed_latency(0);
6447 %}
6448
6449 // Default pipeline class.
6450 pipe_class pipe_class_default()
6451 %{
6452 single_instruction;
6453 fixed_latency(2);
6454 %}
6455
6456 // Pipeline class for compares.
6457 pipe_class pipe_class_compare()
6458 %{
6459 single_instruction;
6460 fixed_latency(16);
6461 %}
6462
6463 // Pipeline class for memory operations.
6464 pipe_class pipe_class_memory()
6465 %{
6466 single_instruction;
6467 fixed_latency(16);
6468 %}
6469
6470 // Pipeline class for call.
6471 pipe_class pipe_class_call()
6472 %{
6473 single_instruction;
6474 fixed_latency(100);
6475 %}
6476
6477 // Define the class for the Nop node.
6478 define %{
6479 MachNop = pipe_class_empty;
6480 %}
6481
6482 %}
6483 //----------INSTRUCTIONS-------------------------------------------------------
6484 //
6485 // match -- States which machine-independent subtree may be replaced
6486 // by this instruction.
6487 // ins_cost -- The estimated cost of this instruction is used by instruction
6488 // selection to identify a minimum cost tree of machine
6489 // instructions that matches a tree of machine-independent
6490 // instructions.
6491 // format -- A string providing the disassembly for this instruction.
6492 // The value of an instruction's operand may be inserted
6493 // by referring to it with a '$' prefix.
6494 // opcode -- Three instruction opcodes may be provided. These are referred
6495 // to within an encode class as $primary, $secondary, and $tertiary
6496 // rrspectively. The primary opcode is commonly used to
6497 // indicate the type of machine instruction, while secondary
6498 // and tertiary are often used for prefix options or addressing
6499 // modes.
6500 // ins_encode -- A list of encode classes with parameters. The encode class
6501 // name must have been defined in an 'enc_class' specification
6502 // in the encode section of the architecture description.
6503
6504 // ============================================================================
6505 // Memory (Load/Store) Instructions
6506
6507 // Load Instructions
6508
6509 // Load Byte (8 bit signed)
6510 instruct loadB(iRegINoSp dst, memory1 mem)
6511 %{
6512 match(Set dst (LoadB mem));
6513 predicate(!needs_acquiring_load(n));
6514
6515 ins_cost(4 * INSN_COST);
6516 format %{ "ldrsbw $dst, $mem\t# byte" %}
6517
6518 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6519
6520 ins_pipe(iload_reg_mem);
6521 %}
6522
6523 // Load Byte (8 bit signed) into long
6524 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6525 %{
6526 match(Set dst (ConvI2L (LoadB mem)));
6527 predicate(!needs_acquiring_load(n->in(1)));
6528
6529 ins_cost(4 * INSN_COST);
6530 format %{ "ldrsb $dst, $mem\t# byte" %}
6531
6532 ins_encode(aarch64_enc_ldrsb(dst, mem));
6533
6534 ins_pipe(iload_reg_mem);
6535 %}
6536
6537 // Load Byte (8 bit unsigned)
6538 instruct loadUB(iRegINoSp dst, memory1 mem)
6539 %{
6540 match(Set dst (LoadUB mem));
6541 predicate(!needs_acquiring_load(n));
6542
6543 ins_cost(4 * INSN_COST);
6544 format %{ "ldrbw $dst, $mem\t# byte" %}
6545
6546 ins_encode(aarch64_enc_ldrb(dst, mem));
6547
6548 ins_pipe(iload_reg_mem);
6549 %}
6550
6551 // Load Byte (8 bit unsigned) into long
6552 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6553 %{
6554 match(Set dst (ConvI2L (LoadUB mem)));
6555 predicate(!needs_acquiring_load(n->in(1)));
6556
6557 ins_cost(4 * INSN_COST);
6558 format %{ "ldrb $dst, $mem\t# byte" %}
6559
6560 ins_encode(aarch64_enc_ldrb(dst, mem));
6561
6562 ins_pipe(iload_reg_mem);
6563 %}
6564
6565 // Load Short (16 bit signed)
6566 instruct loadS(iRegINoSp dst, memory2 mem)
6567 %{
6568 match(Set dst (LoadS mem));
6569 predicate(!needs_acquiring_load(n));
6570
6571 ins_cost(4 * INSN_COST);
6572 format %{ "ldrshw $dst, $mem\t# short" %}
6573
6574 ins_encode(aarch64_enc_ldrshw(dst, mem));
6575
6576 ins_pipe(iload_reg_mem);
6577 %}
6578
6579 // Load Short (16 bit signed) into long
6580 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6581 %{
6582 match(Set dst (ConvI2L (LoadS mem)));
6583 predicate(!needs_acquiring_load(n->in(1)));
6584
6585 ins_cost(4 * INSN_COST);
6586 format %{ "ldrsh $dst, $mem\t# short" %}
6587
6588 ins_encode(aarch64_enc_ldrsh(dst, mem));
6589
6590 ins_pipe(iload_reg_mem);
6591 %}
6592
6593 // Load Char (16 bit unsigned)
6594 instruct loadUS(iRegINoSp dst, memory2 mem)
6595 %{
6596 match(Set dst (LoadUS mem));
6597 predicate(!needs_acquiring_load(n));
6598
6599 ins_cost(4 * INSN_COST);
6600 format %{ "ldrh $dst, $mem\t# short" %}
6601
6602 ins_encode(aarch64_enc_ldrh(dst, mem));
6603
6604 ins_pipe(iload_reg_mem);
6605 %}
6606
6607 // Load Short/Char (16 bit unsigned) into long
6608 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6609 %{
6610 match(Set dst (ConvI2L (LoadUS mem)));
6611 predicate(!needs_acquiring_load(n->in(1)));
6612
6613 ins_cost(4 * INSN_COST);
6614 format %{ "ldrh $dst, $mem\t# short" %}
6615
6616 ins_encode(aarch64_enc_ldrh(dst, mem));
6617
6618 ins_pipe(iload_reg_mem);
6619 %}
6620
6621 // Load Integer (32 bit signed)
6622 instruct loadI(iRegINoSp dst, memory4 mem)
6623 %{
6624 match(Set dst (LoadI mem));
6625 predicate(!needs_acquiring_load(n));
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrw $dst, $mem\t# int" %}
6629
6630 ins_encode(aarch64_enc_ldrw(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 // Load Integer (32 bit signed) into long
6636 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6637 %{
6638 match(Set dst (ConvI2L (LoadI mem)));
6639 predicate(!needs_acquiring_load(n->in(1)));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrsw $dst, $mem\t# int" %}
6643
6644 ins_encode(aarch64_enc_ldrsw(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Integer (32 bit unsigned) into long
6650 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6651 %{
6652 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6653 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrw $dst, $mem\t# int" %}
6657
6658 ins_encode(aarch64_enc_ldrw(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Long (64 bit signed)
6664 instruct loadL(iRegLNoSp dst, memory8 mem)
6665 %{
6666 match(Set dst (LoadL mem));
6667 predicate(!needs_acquiring_load(n));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldr $dst, $mem\t# int" %}
6671
6672 ins_encode(aarch64_enc_ldr(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Range
6678 instruct loadRange(iRegINoSp dst, memory4 mem)
6679 %{
6680 match(Set dst (LoadRange mem));
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrw $dst, $mem\t# range" %}
6684
6685 ins_encode(aarch64_enc_ldrw(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Pointer
6691 instruct loadP(iRegPNoSp dst, memory8 mem)
6692 %{
6693 match(Set dst (LoadP mem));
6694 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldr $dst, $mem\t# ptr" %}
6698
6699 ins_encode(aarch64_enc_ldr(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Compressed Pointer
6705 instruct loadN(iRegNNoSp dst, memory4 mem)
6706 %{
6707 match(Set dst (LoadN mem));
6708 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6712
6713 ins_encode(aarch64_enc_ldrw(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Klass Pointer
6719 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6720 %{
6721 match(Set dst (LoadKlass mem));
6722 predicate(!needs_acquiring_load(n));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldr $dst, $mem\t# class" %}
6726
6727 ins_encode(aarch64_enc_ldr(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Narrow Klass Pointer
6733 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6734 %{
6735 match(Set dst (LoadNKlass mem));
6736 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6740
6741 ins_encode(aarch64_enc_ldrw(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6747 %{
6748 match(Set dst (LoadNKlass mem));
6749 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6750
6751 ins_cost(4 * INSN_COST);
6752 format %{
6753 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6754 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6755 %}
6756 ins_encode %{
6757 // inlined aarch64_enc_ldrw
6758 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6759 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6760 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6761 %}
6762 ins_pipe(iload_reg_mem);
6763 %}
6764
6765 // Load Float
6766 instruct loadF(vRegF dst, memory4 mem)
6767 %{
6768 match(Set dst (LoadF mem));
6769 predicate(!needs_acquiring_load(n));
6770
6771 ins_cost(4 * INSN_COST);
6772 format %{ "ldrs $dst, $mem\t# float" %}
6773
6774 ins_encode( aarch64_enc_ldrs(dst, mem) );
6775
6776 ins_pipe(pipe_class_memory);
6777 %}
6778
6779 // Load Double
6780 instruct loadD(vRegD dst, memory8 mem)
6781 %{
6782 match(Set dst (LoadD mem));
6783 predicate(!needs_acquiring_load(n));
6784
6785 ins_cost(4 * INSN_COST);
6786 format %{ "ldrd $dst, $mem\t# double" %}
6787
6788 ins_encode( aarch64_enc_ldrd(dst, mem) );
6789
6790 ins_pipe(pipe_class_memory);
6791 %}
6792
6793
6794 // Load Int Constant
6795 instruct loadConI(iRegINoSp dst, immI src)
6796 %{
6797 match(Set dst src);
6798
6799 ins_cost(INSN_COST);
6800 format %{ "mov $dst, $src\t# int" %}
6801
6802 ins_encode( aarch64_enc_movw_imm(dst, src) );
6803
6804 ins_pipe(ialu_imm);
6805 %}
6806
6807 // Load Long Constant
6808 instruct loadConL(iRegLNoSp dst, immL src)
6809 %{
6810 match(Set dst src);
6811
6812 ins_cost(INSN_COST);
6813 format %{ "mov $dst, $src\t# long" %}
6814
6815 ins_encode( aarch64_enc_mov_imm(dst, src) );
6816
6817 ins_pipe(ialu_imm);
6818 %}
6819
6820 // Load Pointer Constant
6821
6822 instruct loadConP(iRegPNoSp dst, immP con)
6823 %{
6824 match(Set dst con);
6825
6826 ins_cost(INSN_COST * 4);
6827 format %{
6828 "mov $dst, $con\t# ptr\n\t"
6829 %}
6830
6831 ins_encode(aarch64_enc_mov_p(dst, con));
6832
6833 ins_pipe(ialu_imm);
6834 %}
6835
6836 // Load Null Pointer Constant
6837
6838 instruct loadConP0(iRegPNoSp dst, immP0 con)
6839 %{
6840 match(Set dst con);
6841
6842 ins_cost(INSN_COST);
6843 format %{ "mov $dst, $con\t# nullptr ptr" %}
6844
6845 ins_encode(aarch64_enc_mov_p0(dst, con));
6846
6847 ins_pipe(ialu_imm);
6848 %}
6849
6850 // Load Pointer Constant One
6851
6852 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6853 %{
6854 match(Set dst con);
6855
6856 ins_cost(INSN_COST);
6857 format %{ "mov $dst, $con\t# nullptr ptr" %}
6858
6859 ins_encode(aarch64_enc_mov_p1(dst, con));
6860
6861 ins_pipe(ialu_imm);
6862 %}
6863
6864 // Load Byte Map Base Constant
6865
6866 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6867 %{
6868 match(Set dst con);
6869
6870 ins_cost(INSN_COST);
6871 format %{ "adr $dst, $con\t# Byte Map Base" %}
6872
6873 ins_encode %{
6874 __ load_byte_map_base($dst$$Register);
6875 %}
6876
6877 ins_pipe(ialu_imm);
6878 %}
6879
6880 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6881 %{
6882 match(Set dst con);
6883
6884 ins_cost(INSN_COST);
6885 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6886
6887 ins_encode %{
6888 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6889 %}
6890
6891 ins_pipe(ialu_imm);
6892 %}
6893
6894 // Load Narrow Pointer Constant
6895
6896 instruct loadConN(iRegNNoSp dst, immN con)
6897 %{
6898 match(Set dst con);
6899
6900 ins_cost(INSN_COST * 4);
6901 format %{ "mov $dst, $con\t# compressed ptr" %}
6902
6903 ins_encode(aarch64_enc_mov_n(dst, con));
6904
6905 ins_pipe(ialu_imm);
6906 %}
6907
6908 // Load Narrow Null Pointer Constant
6909
6910 instruct loadConN0(iRegNNoSp dst, immN0 con)
6911 %{
6912 match(Set dst con);
6913
6914 ins_cost(INSN_COST);
6915 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6916
6917 ins_encode(aarch64_enc_mov_n0(dst, con));
6918
6919 ins_pipe(ialu_imm);
6920 %}
6921
6922 // Load Narrow Klass Constant
6923
6924 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6925 %{
6926 match(Set dst con);
6927
6928 ins_cost(INSN_COST);
6929 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6930
6931 ins_encode(aarch64_enc_mov_nk(dst, con));
6932
6933 ins_pipe(ialu_imm);
6934 %}
6935
6936 // Load Packed Float Constant
6937
6938 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6939 match(Set dst con);
6940 ins_cost(INSN_COST * 4);
6941 format %{ "fmovs $dst, $con"%}
6942 ins_encode %{
6943 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6944 %}
6945
6946 ins_pipe(fp_imm_s);
6947 %}
6948
6949 // Load Float Constant
6950
6951 instruct loadConF(vRegF dst, immF con) %{
6952 match(Set dst con);
6953
6954 ins_cost(INSN_COST * 4);
6955
6956 format %{
6957 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6958 %}
6959
6960 ins_encode %{
6961 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6962 %}
6963
6964 ins_pipe(fp_load_constant_s);
6965 %}
6966
6967 // Load Packed Double Constant
6968
6969 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6970 match(Set dst con);
6971 ins_cost(INSN_COST);
6972 format %{ "fmovd $dst, $con"%}
6973 ins_encode %{
6974 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6975 %}
6976
6977 ins_pipe(fp_imm_d);
6978 %}
6979
6980 // Load Double Constant
6981
6982 instruct loadConD(vRegD dst, immD con) %{
6983 match(Set dst con);
6984
6985 ins_cost(INSN_COST * 5);
6986 format %{
6987 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6988 %}
6989
6990 ins_encode %{
6991 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6992 %}
6993
6994 ins_pipe(fp_load_constant_d);
6995 %}
6996
6997 // Load Half Float Constant
6998 // The "ldr" instruction loads a 32-bit word from the constant pool into a
6999 // 32-bit register but only the bottom half will be populated and the top
7000 // 16 bits are zero.
7001 instruct loadConH(vRegF dst, immH con) %{
7002 match(Set dst con);
7003 format %{
7004 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7005 %}
7006 ins_encode %{
7007 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7008 %}
7009 ins_pipe(fp_load_constant_s);
7010 %}
7011
7012 // Store Instructions
7013
7014 // Store Byte
7015 instruct storeB(iRegIorL2I src, memory1 mem)
7016 %{
7017 match(Set mem (StoreB mem src));
7018 predicate(!needs_releasing_store(n));
7019
7020 ins_cost(INSN_COST);
7021 format %{ "strb $src, $mem\t# byte" %}
7022
7023 ins_encode(aarch64_enc_strb(src, mem));
7024
7025 ins_pipe(istore_reg_mem);
7026 %}
7027
7028
7029 instruct storeimmB0(immI0 zero, memory1 mem)
7030 %{
7031 match(Set mem (StoreB mem zero));
7032 predicate(!needs_releasing_store(n));
7033
7034 ins_cost(INSN_COST);
7035 format %{ "strb rscractch2, $mem\t# byte" %}
7036
7037 ins_encode(aarch64_enc_strb0(mem));
7038
7039 ins_pipe(istore_mem);
7040 %}
7041
7042 // Store Char/Short
7043 instruct storeC(iRegIorL2I src, memory2 mem)
7044 %{
7045 match(Set mem (StoreC mem src));
7046 predicate(!needs_releasing_store(n));
7047
7048 ins_cost(INSN_COST);
7049 format %{ "strh $src, $mem\t# short" %}
7050
7051 ins_encode(aarch64_enc_strh(src, mem));
7052
7053 ins_pipe(istore_reg_mem);
7054 %}
7055
7056 instruct storeimmC0(immI0 zero, memory2 mem)
7057 %{
7058 match(Set mem (StoreC mem zero));
7059 predicate(!needs_releasing_store(n));
7060
7061 ins_cost(INSN_COST);
7062 format %{ "strh zr, $mem\t# short" %}
7063
7064 ins_encode(aarch64_enc_strh0(mem));
7065
7066 ins_pipe(istore_mem);
7067 %}
7068
7069 // Store Integer
7070
7071 instruct storeI(iRegIorL2I src, memory4 mem)
7072 %{
7073 match(Set mem(StoreI mem src));
7074 predicate(!needs_releasing_store(n));
7075
7076 ins_cost(INSN_COST);
7077 format %{ "strw $src, $mem\t# int" %}
7078
7079 ins_encode(aarch64_enc_strw(src, mem));
7080
7081 ins_pipe(istore_reg_mem);
7082 %}
7083
7084 instruct storeimmI0(immI0 zero, memory4 mem)
7085 %{
7086 match(Set mem(StoreI mem zero));
7087 predicate(!needs_releasing_store(n));
7088
7089 ins_cost(INSN_COST);
7090 format %{ "strw zr, $mem\t# int" %}
7091
7092 ins_encode(aarch64_enc_strw0(mem));
7093
7094 ins_pipe(istore_mem);
7095 %}
7096
7097 // Store Long (64 bit signed)
7098 instruct storeL(iRegL src, memory8 mem)
7099 %{
7100 match(Set mem (StoreL mem src));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "str $src, $mem\t# int" %}
7105
7106 ins_encode(aarch64_enc_str(src, mem));
7107
7108 ins_pipe(istore_reg_mem);
7109 %}
7110
7111 // Store Long (64 bit signed)
7112 instruct storeimmL0(immL0 zero, memory8 mem)
7113 %{
7114 match(Set mem (StoreL mem zero));
7115 predicate(!needs_releasing_store(n));
7116
7117 ins_cost(INSN_COST);
7118 format %{ "str zr, $mem\t# int" %}
7119
7120 ins_encode(aarch64_enc_str0(mem));
7121
7122 ins_pipe(istore_mem);
7123 %}
7124
7125 // Store Pointer
7126 instruct storeP(iRegP src, memory8 mem)
7127 %{
7128 match(Set mem (StoreP mem src));
7129 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7130
7131 ins_cost(INSN_COST);
7132 format %{ "str $src, $mem\t# ptr" %}
7133
7134 ins_encode(aarch64_enc_str(src, mem));
7135
7136 ins_pipe(istore_reg_mem);
7137 %}
7138
7139 // Store Pointer
7140 instruct storeimmP0(immP0 zero, memory8 mem)
7141 %{
7142 match(Set mem (StoreP mem zero));
7143 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7144
7145 ins_cost(INSN_COST);
7146 format %{ "str zr, $mem\t# ptr" %}
7147
7148 ins_encode(aarch64_enc_str0(mem));
7149
7150 ins_pipe(istore_mem);
7151 %}
7152
7153 // Store Compressed Pointer
7154 instruct storeN(iRegN src, memory4 mem)
7155 %{
7156 match(Set mem (StoreN mem src));
7157 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7158
7159 ins_cost(INSN_COST);
7160 format %{ "strw $src, $mem\t# compressed ptr" %}
7161
7162 ins_encode(aarch64_enc_strw(src, mem));
7163
7164 ins_pipe(istore_reg_mem);
7165 %}
7166
7167 instruct storeImmN0(immN0 zero, memory4 mem)
7168 %{
7169 match(Set mem (StoreN mem zero));
7170 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7171
7172 ins_cost(INSN_COST);
7173 format %{ "strw zr, $mem\t# compressed ptr" %}
7174
7175 ins_encode(aarch64_enc_strw0(mem));
7176
7177 ins_pipe(istore_mem);
7178 %}
7179
7180 // Store Float
7181 instruct storeF(vRegF src, memory4 mem)
7182 %{
7183 match(Set mem (StoreF mem src));
7184 predicate(!needs_releasing_store(n));
7185
7186 ins_cost(INSN_COST);
7187 format %{ "strs $src, $mem\t# float" %}
7188
7189 ins_encode( aarch64_enc_strs(src, mem) );
7190
7191 ins_pipe(pipe_class_memory);
7192 %}
7193
7194 // TODO
7195 // implement storeImmF0 and storeFImmPacked
7196
7197 // Store Double
7198 instruct storeD(vRegD src, memory8 mem)
7199 %{
7200 match(Set mem (StoreD mem src));
7201 predicate(!needs_releasing_store(n));
7202
7203 ins_cost(INSN_COST);
7204 format %{ "strd $src, $mem\t# double" %}
7205
7206 ins_encode( aarch64_enc_strd(src, mem) );
7207
7208 ins_pipe(pipe_class_memory);
7209 %}
7210
7211 // Store Compressed Klass Pointer
7212 instruct storeNKlass(iRegN src, memory4 mem)
7213 %{
7214 predicate(!needs_releasing_store(n));
7215 match(Set mem (StoreNKlass mem src));
7216
7217 ins_cost(INSN_COST);
7218 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7219
7220 ins_encode(aarch64_enc_strw(src, mem));
7221
7222 ins_pipe(istore_reg_mem);
7223 %}
7224
7225 // TODO
7226 // implement storeImmD0 and storeDImmPacked
7227
7228 // prefetch instructions
7229 // Must be safe to execute with invalid address (cannot fault).
7230
7231 instruct prefetchalloc( memory8 mem ) %{
7232 match(PrefetchAllocation mem);
7233
7234 ins_cost(INSN_COST);
7235 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7236
7237 ins_encode( aarch64_enc_prefetchw(mem) );
7238
7239 ins_pipe(iload_prefetch);
7240 %}
7241
7242 // ---------------- volatile loads and stores ----------------
7243
7244 // Load Byte (8 bit signed)
7245 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7246 %{
7247 match(Set dst (LoadB mem));
7248
7249 ins_cost(VOLATILE_REF_COST);
7250 format %{ "ldarsb $dst, $mem\t# byte" %}
7251
7252 ins_encode(aarch64_enc_ldarsb(dst, mem));
7253
7254 ins_pipe(pipe_serial);
7255 %}
7256
7257 // Load Byte (8 bit signed) into long
7258 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7259 %{
7260 match(Set dst (ConvI2L (LoadB mem)));
7261
7262 ins_cost(VOLATILE_REF_COST);
7263 format %{ "ldarsb $dst, $mem\t# byte" %}
7264
7265 ins_encode(aarch64_enc_ldarsb(dst, mem));
7266
7267 ins_pipe(pipe_serial);
7268 %}
7269
7270 // Load Byte (8 bit unsigned)
7271 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7272 %{
7273 match(Set dst (LoadUB mem));
7274
7275 ins_cost(VOLATILE_REF_COST);
7276 format %{ "ldarb $dst, $mem\t# byte" %}
7277
7278 ins_encode(aarch64_enc_ldarb(dst, mem));
7279
7280 ins_pipe(pipe_serial);
7281 %}
7282
7283 // Load Byte (8 bit unsigned) into long
7284 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7285 %{
7286 match(Set dst (ConvI2L (LoadUB mem)));
7287
7288 ins_cost(VOLATILE_REF_COST);
7289 format %{ "ldarb $dst, $mem\t# byte" %}
7290
7291 ins_encode(aarch64_enc_ldarb(dst, mem));
7292
7293 ins_pipe(pipe_serial);
7294 %}
7295
7296 // Load Short (16 bit signed)
7297 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7298 %{
7299 match(Set dst (LoadS mem));
7300
7301 ins_cost(VOLATILE_REF_COST);
7302 format %{ "ldarshw $dst, $mem\t# short" %}
7303
7304 ins_encode(aarch64_enc_ldarshw(dst, mem));
7305
7306 ins_pipe(pipe_serial);
7307 %}
7308
7309 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (LoadUS mem));
7312
7313 ins_cost(VOLATILE_REF_COST);
7314 format %{ "ldarhw $dst, $mem\t# short" %}
7315
7316 ins_encode(aarch64_enc_ldarhw(dst, mem));
7317
7318 ins_pipe(pipe_serial);
7319 %}
7320
7321 // Load Short/Char (16 bit unsigned) into long
7322 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7323 %{
7324 match(Set dst (ConvI2L (LoadUS mem)));
7325
7326 ins_cost(VOLATILE_REF_COST);
7327 format %{ "ldarh $dst, $mem\t# short" %}
7328
7329 ins_encode(aarch64_enc_ldarh(dst, mem));
7330
7331 ins_pipe(pipe_serial);
7332 %}
7333
7334 // Load Short/Char (16 bit signed) into long
7335 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7336 %{
7337 match(Set dst (ConvI2L (LoadS mem)));
7338
7339 ins_cost(VOLATILE_REF_COST);
7340 format %{ "ldarh $dst, $mem\t# short" %}
7341
7342 ins_encode(aarch64_enc_ldarsh(dst, mem));
7343
7344 ins_pipe(pipe_serial);
7345 %}
7346
7347 // Load Integer (32 bit signed)
7348 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7349 %{
7350 match(Set dst (LoadI mem));
7351
7352 ins_cost(VOLATILE_REF_COST);
7353 format %{ "ldarw $dst, $mem\t# int" %}
7354
7355 ins_encode(aarch64_enc_ldarw(dst, mem));
7356
7357 ins_pipe(pipe_serial);
7358 %}
7359
7360 // Load Integer (32 bit unsigned) into long
7361 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7362 %{
7363 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7364
7365 ins_cost(VOLATILE_REF_COST);
7366 format %{ "ldarw $dst, $mem\t# int" %}
7367
7368 ins_encode(aarch64_enc_ldarw(dst, mem));
7369
7370 ins_pipe(pipe_serial);
7371 %}
7372
7373 // Load Long (64 bit signed)
7374 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7375 %{
7376 match(Set dst (LoadL mem));
7377
7378 ins_cost(VOLATILE_REF_COST);
7379 format %{ "ldar $dst, $mem\t# int" %}
7380
7381 ins_encode(aarch64_enc_ldar(dst, mem));
7382
7383 ins_pipe(pipe_serial);
7384 %}
7385
7386 // Load Pointer
7387 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set dst (LoadP mem));
7390 predicate(n->as_Load()->barrier_data() == 0);
7391
7392 ins_cost(VOLATILE_REF_COST);
7393 format %{ "ldar $dst, $mem\t# ptr" %}
7394
7395 ins_encode(aarch64_enc_ldar(dst, mem));
7396
7397 ins_pipe(pipe_serial);
7398 %}
7399
7400 // Load Compressed Pointer
7401 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7402 %{
7403 match(Set dst (LoadN mem));
7404 predicate(n->as_Load()->barrier_data() == 0);
7405
7406 ins_cost(VOLATILE_REF_COST);
7407 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7408
7409 ins_encode(aarch64_enc_ldarw(dst, mem));
7410
7411 ins_pipe(pipe_serial);
7412 %}
7413
7414 // Load Float
7415 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7416 %{
7417 match(Set dst (LoadF mem));
7418
7419 ins_cost(VOLATILE_REF_COST);
7420 format %{ "ldars $dst, $mem\t# float" %}
7421
7422 ins_encode( aarch64_enc_fldars(dst, mem) );
7423
7424 ins_pipe(pipe_serial);
7425 %}
7426
7427 // Load Double
7428 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7429 %{
7430 match(Set dst (LoadD mem));
7431
7432 ins_cost(VOLATILE_REF_COST);
7433 format %{ "ldard $dst, $mem\t# double" %}
7434
7435 ins_encode( aarch64_enc_fldard(dst, mem) );
7436
7437 ins_pipe(pipe_serial);
7438 %}
7439
7440 // Store Byte
7441 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7442 %{
7443 match(Set mem (StoreB mem src));
7444
7445 ins_cost(VOLATILE_REF_COST);
7446 format %{ "stlrb $src, $mem\t# byte" %}
7447
7448 ins_encode(aarch64_enc_stlrb(src, mem));
7449
7450 ins_pipe(pipe_class_memory);
7451 %}
7452
7453 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7454 %{
7455 match(Set mem (StoreB mem zero));
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "stlrb zr, $mem\t# byte" %}
7459
7460 ins_encode(aarch64_enc_stlrb0(mem));
7461
7462 ins_pipe(pipe_class_memory);
7463 %}
7464
7465 // Store Char/Short
7466 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set mem (StoreC mem src));
7469
7470 ins_cost(VOLATILE_REF_COST);
7471 format %{ "stlrh $src, $mem\t# short" %}
7472
7473 ins_encode(aarch64_enc_stlrh(src, mem));
7474
7475 ins_pipe(pipe_class_memory);
7476 %}
7477
7478 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7479 %{
7480 match(Set mem (StoreC mem zero));
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlrh zr, $mem\t# short" %}
7484
7485 ins_encode(aarch64_enc_stlrh0(mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 // Store Integer
7491
7492 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7493 %{
7494 match(Set mem(StoreI mem src));
7495
7496 ins_cost(VOLATILE_REF_COST);
7497 format %{ "stlrw $src, $mem\t# int" %}
7498
7499 ins_encode(aarch64_enc_stlrw(src, mem));
7500
7501 ins_pipe(pipe_class_memory);
7502 %}
7503
7504 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem(StoreI mem zero));
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "stlrw zr, $mem\t# int" %}
7510
7511 ins_encode(aarch64_enc_stlrw0(mem));
7512
7513 ins_pipe(pipe_class_memory);
7514 %}
7515
7516 // Store Long (64 bit signed)
7517 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7518 %{
7519 match(Set mem (StoreL mem src));
7520
7521 ins_cost(VOLATILE_REF_COST);
7522 format %{ "stlr $src, $mem\t# int" %}
7523
7524 ins_encode(aarch64_enc_stlr(src, mem));
7525
7526 ins_pipe(pipe_class_memory);
7527 %}
7528
7529 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem (StoreL mem zero));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlr zr, $mem\t# int" %}
7535
7536 ins_encode(aarch64_enc_stlr0(mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 // Store Pointer
7542 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7543 %{
7544 match(Set mem (StoreP mem src));
7545 predicate(n->as_Store()->barrier_data() == 0);
7546
7547 ins_cost(VOLATILE_REF_COST);
7548 format %{ "stlr $src, $mem\t# ptr" %}
7549
7550 ins_encode(aarch64_enc_stlr(src, mem));
7551
7552 ins_pipe(pipe_class_memory);
7553 %}
7554
7555 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7556 %{
7557 match(Set mem (StoreP mem zero));
7558 predicate(n->as_Store()->barrier_data() == 0);
7559
7560 ins_cost(VOLATILE_REF_COST);
7561 format %{ "stlr zr, $mem\t# ptr" %}
7562
7563 ins_encode(aarch64_enc_stlr0(mem));
7564
7565 ins_pipe(pipe_class_memory);
7566 %}
7567
7568 // Store Compressed Pointer
7569 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem (StoreN mem src));
7572 predicate(n->as_Store()->barrier_data() == 0);
7573
7574 ins_cost(VOLATILE_REF_COST);
7575 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7576
7577 ins_encode(aarch64_enc_stlrw(src, mem));
7578
7579 ins_pipe(pipe_class_memory);
7580 %}
7581
7582 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreN mem zero));
7585 predicate(n->as_Store()->barrier_data() == 0);
7586
7587 ins_cost(VOLATILE_REF_COST);
7588 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7589
7590 ins_encode(aarch64_enc_stlrw0(mem));
7591
7592 ins_pipe(pipe_class_memory);
7593 %}
7594
7595 // Store Float
7596 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7597 %{
7598 match(Set mem (StoreF mem src));
7599
7600 ins_cost(VOLATILE_REF_COST);
7601 format %{ "stlrs $src, $mem\t# float" %}
7602
7603 ins_encode( aarch64_enc_fstlrs(src, mem) );
7604
7605 ins_pipe(pipe_class_memory);
7606 %}
7607
7608 // TODO
7609 // implement storeImmF0 and storeFImmPacked
7610
7611 // Store Double
7612 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7613 %{
7614 match(Set mem (StoreD mem src));
7615
7616 ins_cost(VOLATILE_REF_COST);
7617 format %{ "stlrd $src, $mem\t# double" %}
7618
7619 ins_encode( aarch64_enc_fstlrd(src, mem) );
7620
7621 ins_pipe(pipe_class_memory);
7622 %}
7623
7624 // ---------------- end of volatile loads and stores ----------------
7625
7626 instruct cacheWB(indirect addr)
7627 %{
7628 predicate(VM_Version::supports_data_cache_line_flush());
7629 match(CacheWB addr);
7630
7631 ins_cost(100);
7632 format %{"cache wb $addr" %}
7633 ins_encode %{
7634 assert($addr->index_position() < 0, "should be");
7635 assert($addr$$disp == 0, "should be");
7636 __ cache_wb(Address($addr$$base$$Register, 0));
7637 %}
7638 ins_pipe(pipe_slow); // XXX
7639 %}
7640
7641 instruct cacheWBPreSync()
7642 %{
7643 predicate(VM_Version::supports_data_cache_line_flush());
7644 match(CacheWBPreSync);
7645
7646 ins_cost(100);
7647 format %{"cache wb presync" %}
7648 ins_encode %{
7649 __ cache_wbsync(true);
7650 %}
7651 ins_pipe(pipe_slow); // XXX
7652 %}
7653
7654 instruct cacheWBPostSync()
7655 %{
7656 predicate(VM_Version::supports_data_cache_line_flush());
7657 match(CacheWBPostSync);
7658
7659 ins_cost(100);
7660 format %{"cache wb postsync" %}
7661 ins_encode %{
7662 __ cache_wbsync(false);
7663 %}
7664 ins_pipe(pipe_slow); // XXX
7665 %}
7666
7667 // ============================================================================
7668 // BSWAP Instructions
7669
7670 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7671 match(Set dst (ReverseBytesI src));
7672
7673 ins_cost(INSN_COST);
7674 format %{ "revw $dst, $src" %}
7675
7676 ins_encode %{
7677 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7678 %}
7679
7680 ins_pipe(ialu_reg);
7681 %}
7682
7683 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7684 match(Set dst (ReverseBytesL src));
7685
7686 ins_cost(INSN_COST);
7687 format %{ "rev $dst, $src" %}
7688
7689 ins_encode %{
7690 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7691 %}
7692
7693 ins_pipe(ialu_reg);
7694 %}
7695
7696 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7697 match(Set dst (ReverseBytesUS src));
7698
7699 ins_cost(INSN_COST);
7700 format %{ "rev16w $dst, $src" %}
7701
7702 ins_encode %{
7703 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7704 %}
7705
7706 ins_pipe(ialu_reg);
7707 %}
7708
7709 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7710 match(Set dst (ReverseBytesS src));
7711
7712 ins_cost(INSN_COST);
7713 format %{ "rev16w $dst, $src\n\t"
7714 "sbfmw $dst, $dst, #0, #15" %}
7715
7716 ins_encode %{
7717 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7718 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7719 %}
7720
7721 ins_pipe(ialu_reg);
7722 %}
7723
7724 // ============================================================================
7725 // Zero Count Instructions
7726
7727 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7728 match(Set dst (CountLeadingZerosI src));
7729
7730 ins_cost(INSN_COST);
7731 format %{ "clzw $dst, $src" %}
7732 ins_encode %{
7733 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7734 %}
7735
7736 ins_pipe(ialu_reg);
7737 %}
7738
7739 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7740 match(Set dst (CountLeadingZerosL src));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "clz $dst, $src" %}
7744 ins_encode %{
7745 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7746 %}
7747
7748 ins_pipe(ialu_reg);
7749 %}
7750
7751 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7752 match(Set dst (CountTrailingZerosI src));
7753
7754 ins_cost(INSN_COST * 2);
7755 format %{ "rbitw $dst, $src\n\t"
7756 "clzw $dst, $dst" %}
7757 ins_encode %{
7758 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7759 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7760 %}
7761
7762 ins_pipe(ialu_reg);
7763 %}
7764
7765 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7766 match(Set dst (CountTrailingZerosL src));
7767
7768 ins_cost(INSN_COST * 2);
7769 format %{ "rbit $dst, $src\n\t"
7770 "clz $dst, $dst" %}
7771 ins_encode %{
7772 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7773 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7774 %}
7775
7776 ins_pipe(ialu_reg);
7777 %}
7778
7779 //---------- Population Count Instructions -------------------------------------
7780 //
7781
7782 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7783 match(Set dst (PopCountI src));
7784 effect(TEMP tmp);
7785 ins_cost(INSN_COST * 13);
7786
7787 format %{ "movw $src, $src\n\t"
7788 "mov $tmp, $src\t# vector (1D)\n\t"
7789 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7790 "addv $tmp, $tmp\t# vector (8B)\n\t"
7791 "mov $dst, $tmp\t# vector (1D)" %}
7792 ins_encode %{
7793 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7794 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7795 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7796 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7797 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7798 %}
7799
7800 ins_pipe(pipe_class_default);
7801 %}
7802
7803 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7804 match(Set dst (PopCountI (LoadI mem)));
7805 effect(TEMP tmp);
7806 ins_cost(INSN_COST * 13);
7807
7808 format %{ "ldrs $tmp, $mem\n\t"
7809 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7810 "addv $tmp, $tmp\t# vector (8B)\n\t"
7811 "mov $dst, $tmp\t# vector (1D)" %}
7812 ins_encode %{
7813 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7814 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7815 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7816 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7817 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7818 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7819 %}
7820
7821 ins_pipe(pipe_class_default);
7822 %}
7823
7824 // Note: Long.bitCount(long) returns an int.
7825 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7826 match(Set dst (PopCountL src));
7827 effect(TEMP tmp);
7828 ins_cost(INSN_COST * 13);
7829
7830 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7831 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7832 "addv $tmp, $tmp\t# vector (8B)\n\t"
7833 "mov $dst, $tmp\t# vector (1D)" %}
7834 ins_encode %{
7835 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7836 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7837 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7838 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7839 %}
7840
7841 ins_pipe(pipe_class_default);
7842 %}
7843
7844 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7845 match(Set dst (PopCountL (LoadL mem)));
7846 effect(TEMP tmp);
7847 ins_cost(INSN_COST * 13);
7848
7849 format %{ "ldrd $tmp, $mem\n\t"
7850 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7851 "addv $tmp, $tmp\t# vector (8B)\n\t"
7852 "mov $dst, $tmp\t# vector (1D)" %}
7853 ins_encode %{
7854 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7855 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7857 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7858 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7859 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7860 %}
7861
7862 ins_pipe(pipe_class_default);
7863 %}
7864
7865 // ============================================================================
7866 // VerifyVectorAlignment Instruction
7867
7868 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7869 match(Set addr (VerifyVectorAlignment addr mask));
7870 effect(KILL cr);
7871 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7872 ins_encode %{
7873 Label Lskip;
7874 // check if masked bits of addr are zero
7875 __ tst($addr$$Register, $mask$$constant);
7876 __ br(Assembler::EQ, Lskip);
7877 __ stop("verify_vector_alignment found a misaligned vector memory access");
7878 __ bind(Lskip);
7879 %}
7880 ins_pipe(pipe_slow);
7881 %}
7882
7883 // ============================================================================
7884 // MemBar Instruction
7885
7886 instruct load_fence() %{
7887 match(LoadFence);
7888 ins_cost(VOLATILE_REF_COST);
7889
7890 format %{ "load_fence" %}
7891
7892 ins_encode %{
7893 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7894 %}
7895 ins_pipe(pipe_serial);
7896 %}
7897
7898 instruct unnecessary_membar_acquire() %{
7899 predicate(unnecessary_acquire(n));
7900 match(MemBarAcquire);
7901 ins_cost(0);
7902
7903 format %{ "membar_acquire (elided)" %}
7904
7905 ins_encode %{
7906 __ block_comment("membar_acquire (elided)");
7907 %}
7908
7909 ins_pipe(pipe_class_empty);
7910 %}
7911
7912 instruct membar_acquire() %{
7913 match(MemBarAcquire);
7914 ins_cost(VOLATILE_REF_COST);
7915
7916 format %{ "membar_acquire\n\t"
7917 "dmb ishld" %}
7918
7919 ins_encode %{
7920 __ block_comment("membar_acquire");
7921 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7922 %}
7923
7924 ins_pipe(pipe_serial);
7925 %}
7926
7927
7928 instruct membar_acquire_lock() %{
7929 match(MemBarAcquireLock);
7930 ins_cost(VOLATILE_REF_COST);
7931
7932 format %{ "membar_acquire_lock (elided)" %}
7933
7934 ins_encode %{
7935 __ block_comment("membar_acquire_lock (elided)");
7936 %}
7937
7938 ins_pipe(pipe_serial);
7939 %}
7940
7941 instruct store_fence() %{
7942 match(StoreFence);
7943 ins_cost(VOLATILE_REF_COST);
7944
7945 format %{ "store_fence" %}
7946
7947 ins_encode %{
7948 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7949 %}
7950 ins_pipe(pipe_serial);
7951 %}
7952
7953 instruct unnecessary_membar_release() %{
7954 predicate(unnecessary_release(n));
7955 match(MemBarRelease);
7956 ins_cost(0);
7957
7958 format %{ "membar_release (elided)" %}
7959
7960 ins_encode %{
7961 __ block_comment("membar_release (elided)");
7962 %}
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct membar_release() %{
7967 match(MemBarRelease);
7968 ins_cost(VOLATILE_REF_COST);
7969
7970 format %{ "membar_release\n\t"
7971 "dmb ishst\n\tdmb ishld" %}
7972
7973 ins_encode %{
7974 __ block_comment("membar_release");
7975 // These will be merged if AlwaysMergeDMB is enabled.
7976 __ membar(Assembler::StoreStore);
7977 __ membar(Assembler::LoadStore);
7978 %}
7979 ins_pipe(pipe_serial);
7980 %}
7981
7982 instruct membar_storestore() %{
7983 match(MemBarStoreStore);
7984 match(StoreStoreFence);
7985 ins_cost(VOLATILE_REF_COST);
7986
7987 format %{ "MEMBAR-store-store" %}
7988
7989 ins_encode %{
7990 __ membar(Assembler::StoreStore);
7991 %}
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 instruct membar_release_lock() %{
7996 match(MemBarReleaseLock);
7997 ins_cost(VOLATILE_REF_COST);
7998
7999 format %{ "membar_release_lock (elided)" %}
8000
8001 ins_encode %{
8002 __ block_comment("membar_release_lock (elided)");
8003 %}
8004
8005 ins_pipe(pipe_serial);
8006 %}
8007
8008 instruct unnecessary_membar_volatile() %{
8009 predicate(unnecessary_volatile(n));
8010 match(MemBarVolatile);
8011 ins_cost(0);
8012
8013 format %{ "membar_volatile (elided)" %}
8014
8015 ins_encode %{
8016 __ block_comment("membar_volatile (elided)");
8017 %}
8018
8019 ins_pipe(pipe_serial);
8020 %}
8021
8022 instruct membar_volatile() %{
8023 match(MemBarVolatile);
8024 ins_cost(VOLATILE_REF_COST*100);
8025
8026 format %{ "membar_volatile\n\t"
8027 "dmb ish"%}
8028
8029 ins_encode %{
8030 __ block_comment("membar_volatile");
8031 __ membar(Assembler::StoreLoad);
8032 %}
8033
8034 ins_pipe(pipe_serial);
8035 %}
8036
8037 // ============================================================================
8038 // Cast/Convert Instructions
8039
8040 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8041 match(Set dst (CastX2P src));
8042
8043 ins_cost(INSN_COST);
8044 format %{ "mov $dst, $src\t# long -> ptr" %}
8045
8046 ins_encode %{
8047 if ($dst$$reg != $src$$reg) {
8048 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8049 }
8050 %}
8051
8052 ins_pipe(ialu_reg);
8053 %}
8054
8055 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8056 match(Set dst (CastP2X src));
8057
8058 ins_cost(INSN_COST);
8059 format %{ "mov $dst, $src\t# ptr -> long" %}
8060
8061 ins_encode %{
8062 if ($dst$$reg != $src$$reg) {
8063 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8064 }
8065 %}
8066
8067 ins_pipe(ialu_reg);
8068 %}
8069
8070 // Convert oop into int for vectors alignment masking
8071 instruct convP2I(iRegINoSp dst, iRegP src) %{
8072 match(Set dst (ConvL2I (CastP2X src)));
8073
8074 ins_cost(INSN_COST);
8075 format %{ "movw $dst, $src\t# ptr -> int" %}
8076 ins_encode %{
8077 __ movw($dst$$Register, $src$$Register);
8078 %}
8079
8080 ins_pipe(ialu_reg);
8081 %}
8082
8083 // Convert compressed oop into int for vectors alignment masking
8084 // in case of 32bit oops (heap < 4Gb).
8085 instruct convN2I(iRegINoSp dst, iRegN src)
8086 %{
8087 predicate(CompressedOops::shift() == 0);
8088 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8089
8090 ins_cost(INSN_COST);
8091 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8092 ins_encode %{
8093 __ movw($dst$$Register, $src$$Register);
8094 %}
8095
8096 ins_pipe(ialu_reg);
8097 %}
8098
8099
8100 // Convert oop pointer into compressed form
8101 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8102 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8103 match(Set dst (EncodeP src));
8104 effect(KILL cr);
8105 ins_cost(INSN_COST * 3);
8106 format %{ "encode_heap_oop $dst, $src" %}
8107 ins_encode %{
8108 Register s = $src$$Register;
8109 Register d = $dst$$Register;
8110 __ encode_heap_oop(d, s);
8111 %}
8112 ins_pipe(ialu_reg);
8113 %}
8114
8115 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8116 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8117 match(Set dst (EncodeP src));
8118 ins_cost(INSN_COST * 3);
8119 format %{ "encode_heap_oop_not_null $dst, $src" %}
8120 ins_encode %{
8121 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8122 %}
8123 ins_pipe(ialu_reg);
8124 %}
8125
8126 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8127 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8128 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8129 match(Set dst (DecodeN src));
8130 ins_cost(INSN_COST * 3);
8131 format %{ "decode_heap_oop $dst, $src" %}
8132 ins_encode %{
8133 Register s = $src$$Register;
8134 Register d = $dst$$Register;
8135 __ decode_heap_oop(d, s);
8136 %}
8137 ins_pipe(ialu_reg);
8138 %}
8139
8140 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8141 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8142 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8143 match(Set dst (DecodeN src));
8144 ins_cost(INSN_COST * 3);
8145 format %{ "decode_heap_oop_not_null $dst, $src" %}
8146 ins_encode %{
8147 Register s = $src$$Register;
8148 Register d = $dst$$Register;
8149 __ decode_heap_oop_not_null(d, s);
8150 %}
8151 ins_pipe(ialu_reg);
8152 %}
8153
8154 // n.b. AArch64 implementations of encode_klass_not_null and
8155 // decode_klass_not_null do not modify the flags register so, unlike
8156 // Intel, we don't kill CR as a side effect here
8157
8158 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8159 match(Set dst (EncodePKlass src));
8160
8161 ins_cost(INSN_COST * 3);
8162 format %{ "encode_klass_not_null $dst,$src" %}
8163
8164 ins_encode %{
8165 Register src_reg = as_Register($src$$reg);
8166 Register dst_reg = as_Register($dst$$reg);
8167 __ encode_klass_not_null(dst_reg, src_reg);
8168 %}
8169
8170 ins_pipe(ialu_reg);
8171 %}
8172
8173 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8174 match(Set dst (DecodeNKlass src));
8175
8176 ins_cost(INSN_COST * 3);
8177 format %{ "decode_klass_not_null $dst,$src" %}
8178
8179 ins_encode %{
8180 Register src_reg = as_Register($src$$reg);
8181 Register dst_reg = as_Register($dst$$reg);
8182 if (dst_reg != src_reg) {
8183 __ decode_klass_not_null(dst_reg, src_reg);
8184 } else {
8185 __ decode_klass_not_null(dst_reg);
8186 }
8187 %}
8188
8189 ins_pipe(ialu_reg);
8190 %}
8191
8192 instruct checkCastPP(iRegPNoSp dst)
8193 %{
8194 match(Set dst (CheckCastPP dst));
8195
8196 size(0);
8197 format %{ "# checkcastPP of $dst" %}
8198 ins_encode(/* empty encoding */);
8199 ins_pipe(pipe_class_empty);
8200 %}
8201
8202 instruct castPP(iRegPNoSp dst)
8203 %{
8204 match(Set dst (CastPP dst));
8205
8206 size(0);
8207 format %{ "# castPP of $dst" %}
8208 ins_encode(/* empty encoding */);
8209 ins_pipe(pipe_class_empty);
8210 %}
8211
8212 instruct castII(iRegI dst)
8213 %{
8214 predicate(VerifyConstraintCasts == 0);
8215 match(Set dst (CastII dst));
8216
8217 size(0);
8218 format %{ "# castII of $dst" %}
8219 ins_encode(/* empty encoding */);
8220 ins_cost(0);
8221 ins_pipe(pipe_class_empty);
8222 %}
8223
8224 instruct castII_checked(iRegI dst, rFlagsReg cr)
8225 %{
8226 predicate(VerifyConstraintCasts > 0);
8227 match(Set dst (CastII dst));
8228 effect(KILL cr);
8229
8230 format %{ "# castII_checked of $dst" %}
8231 ins_encode %{
8232 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8233 %}
8234 ins_pipe(pipe_slow);
8235 %}
8236
8237 instruct castLL(iRegL dst)
8238 %{
8239 predicate(VerifyConstraintCasts == 0);
8240 match(Set dst (CastLL dst));
8241
8242 size(0);
8243 format %{ "# castLL of $dst" %}
8244 ins_encode(/* empty encoding */);
8245 ins_cost(0);
8246 ins_pipe(pipe_class_empty);
8247 %}
8248
8249 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8250 %{
8251 predicate(VerifyConstraintCasts > 0);
8252 match(Set dst (CastLL dst));
8253 effect(KILL cr);
8254
8255 format %{ "# castLL_checked of $dst" %}
8256 ins_encode %{
8257 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8258 %}
8259 ins_pipe(pipe_slow);
8260 %}
8261
8262 instruct castFF(vRegF dst)
8263 %{
8264 match(Set dst (CastFF dst));
8265
8266 size(0);
8267 format %{ "# castFF of $dst" %}
8268 ins_encode(/* empty encoding */);
8269 ins_cost(0);
8270 ins_pipe(pipe_class_empty);
8271 %}
8272
8273 instruct castDD(vRegD dst)
8274 %{
8275 match(Set dst (CastDD dst));
8276
8277 size(0);
8278 format %{ "# castDD of $dst" %}
8279 ins_encode(/* empty encoding */);
8280 ins_cost(0);
8281 ins_pipe(pipe_class_empty);
8282 %}
8283
8284 instruct castVV(vReg dst)
8285 %{
8286 match(Set dst (CastVV dst));
8287
8288 size(0);
8289 format %{ "# castVV of $dst" %}
8290 ins_encode(/* empty encoding */);
8291 ins_cost(0);
8292 ins_pipe(pipe_class_empty);
8293 %}
8294
8295 instruct castVVMask(pRegGov dst)
8296 %{
8297 match(Set dst (CastVV dst));
8298
8299 size(0);
8300 format %{ "# castVV of $dst" %}
8301 ins_encode(/* empty encoding */);
8302 ins_cost(0);
8303 ins_pipe(pipe_class_empty);
8304 %}
8305
8306 // ============================================================================
8307 // Atomic operation instructions
8308 //
8309
8310 // standard CompareAndSwapX when we are using barriers
8311 // these have higher priority than the rules selected by a predicate
8312
8313 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8314 // can't match them
8315
8316 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8317
8318 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8319 ins_cost(2 * VOLATILE_REF_COST);
8320
8321 effect(KILL cr);
8322
8323 format %{
8324 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8325 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8326 %}
8327
8328 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8329 aarch64_enc_cset_eq(res));
8330
8331 ins_pipe(pipe_slow);
8332 %}
8333
8334 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8335
8336 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8337 ins_cost(2 * VOLATILE_REF_COST);
8338
8339 effect(KILL cr);
8340
8341 format %{
8342 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8343 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8344 %}
8345
8346 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8347 aarch64_enc_cset_eq(res));
8348
8349 ins_pipe(pipe_slow);
8350 %}
8351
8352 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8353
8354 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8355 ins_cost(2 * VOLATILE_REF_COST);
8356
8357 effect(KILL cr);
8358
8359 format %{
8360 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8361 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8362 %}
8363
8364 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8365 aarch64_enc_cset_eq(res));
8366
8367 ins_pipe(pipe_slow);
8368 %}
8369
8370 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8371
8372 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8373 ins_cost(2 * VOLATILE_REF_COST);
8374
8375 effect(KILL cr);
8376
8377 format %{
8378 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8379 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8380 %}
8381
8382 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8383 aarch64_enc_cset_eq(res));
8384
8385 ins_pipe(pipe_slow);
8386 %}
8387
8388 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8389
8390 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8391 predicate(n->as_LoadStore()->barrier_data() == 0);
8392 ins_cost(2 * VOLATILE_REF_COST);
8393
8394 effect(KILL cr);
8395
8396 format %{
8397 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8398 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8399 %}
8400
8401 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8402 aarch64_enc_cset_eq(res));
8403
8404 ins_pipe(pipe_slow);
8405 %}
8406
8407 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8408
8409 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8410 predicate(n->as_LoadStore()->barrier_data() == 0);
8411 ins_cost(2 * VOLATILE_REF_COST);
8412
8413 effect(KILL cr);
8414
8415 format %{
8416 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8417 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8418 %}
8419
8420 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8421 aarch64_enc_cset_eq(res));
8422
8423 ins_pipe(pipe_slow);
8424 %}
8425
8426 // alternative CompareAndSwapX when we are eliding barriers
8427
8428 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8429
8430 predicate(needs_acquiring_load_exclusive(n));
8431 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8432 ins_cost(VOLATILE_REF_COST);
8433
8434 effect(KILL cr);
8435
8436 format %{
8437 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8438 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8439 %}
8440
8441 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8442 aarch64_enc_cset_eq(res));
8443
8444 ins_pipe(pipe_slow);
8445 %}
8446
8447 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8448
8449 predicate(needs_acquiring_load_exclusive(n));
8450 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8451 ins_cost(VOLATILE_REF_COST);
8452
8453 effect(KILL cr);
8454
8455 format %{
8456 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8457 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8458 %}
8459
8460 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8461 aarch64_enc_cset_eq(res));
8462
8463 ins_pipe(pipe_slow);
8464 %}
8465
8466 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8467
8468 predicate(needs_acquiring_load_exclusive(n));
8469 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8470 ins_cost(VOLATILE_REF_COST);
8471
8472 effect(KILL cr);
8473
8474 format %{
8475 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8476 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8477 %}
8478
8479 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8480 aarch64_enc_cset_eq(res));
8481
8482 ins_pipe(pipe_slow);
8483 %}
8484
8485 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8486
8487 predicate(needs_acquiring_load_exclusive(n));
8488 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8489 ins_cost(VOLATILE_REF_COST);
8490
8491 effect(KILL cr);
8492
8493 format %{
8494 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8495 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8496 %}
8497
8498 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8499 aarch64_enc_cset_eq(res));
8500
8501 ins_pipe(pipe_slow);
8502 %}
8503
8504 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8505
8506 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8507 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8508 ins_cost(VOLATILE_REF_COST);
8509
8510 effect(KILL cr);
8511
8512 format %{
8513 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8514 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8515 %}
8516
8517 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8518 aarch64_enc_cset_eq(res));
8519
8520 ins_pipe(pipe_slow);
8521 %}
8522
8523 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8524
8525 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8526 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8527 ins_cost(VOLATILE_REF_COST);
8528
8529 effect(KILL cr);
8530
8531 format %{
8532 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8533 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8534 %}
8535
8536 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8537 aarch64_enc_cset_eq(res));
8538
8539 ins_pipe(pipe_slow);
8540 %}
8541
8542
8543 // ---------------------------------------------------------------------
8544
8545 // BEGIN This section of the file is automatically generated. Do not edit --------------
8546
8547 // Sundry CAS operations. Note that release is always true,
8548 // regardless of the memory ordering of the CAS. This is because we
8549 // need the volatile case to be sequentially consistent but there is
8550 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8551 // can't check the type of memory ordering here, so we always emit a
8552 // STLXR.
8553
8554 // This section is generated from cas.m4
8555
8556
8557 // This pattern is generated automatically from cas.m4.
8558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8559 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8560 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8561 ins_cost(2 * VOLATILE_REF_COST);
8562 effect(TEMP_DEF res, KILL cr);
8563 format %{
8564 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8565 %}
8566 ins_encode %{
8567 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8568 Assembler::byte, /*acquire*/ false, /*release*/ true,
8569 /*weak*/ false, $res$$Register);
8570 __ sxtbw($res$$Register, $res$$Register);
8571 %}
8572 ins_pipe(pipe_slow);
8573 %}
8574
8575 // This pattern is generated automatically from cas.m4.
8576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8577 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8578 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8579 ins_cost(2 * VOLATILE_REF_COST);
8580 effect(TEMP_DEF res, KILL cr);
8581 format %{
8582 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8583 %}
8584 ins_encode %{
8585 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8586 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8587 /*weak*/ false, $res$$Register);
8588 __ sxthw($res$$Register, $res$$Register);
8589 %}
8590 ins_pipe(pipe_slow);
8591 %}
8592
8593 // This pattern is generated automatically from cas.m4.
8594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8595 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8596 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8597 ins_cost(2 * VOLATILE_REF_COST);
8598 effect(TEMP_DEF res, KILL cr);
8599 format %{
8600 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8601 %}
8602 ins_encode %{
8603 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8604 Assembler::word, /*acquire*/ false, /*release*/ true,
8605 /*weak*/ false, $res$$Register);
8606 %}
8607 ins_pipe(pipe_slow);
8608 %}
8609
8610 // This pattern is generated automatically from cas.m4.
8611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8612 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8613 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8614 ins_cost(2 * VOLATILE_REF_COST);
8615 effect(TEMP_DEF res, KILL cr);
8616 format %{
8617 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8618 %}
8619 ins_encode %{
8620 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8621 Assembler::xword, /*acquire*/ false, /*release*/ true,
8622 /*weak*/ false, $res$$Register);
8623 %}
8624 ins_pipe(pipe_slow);
8625 %}
8626
8627 // This pattern is generated automatically from cas.m4.
8628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8629 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8630 predicate(n->as_LoadStore()->barrier_data() == 0);
8631 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8632 ins_cost(2 * VOLATILE_REF_COST);
8633 effect(TEMP_DEF res, KILL cr);
8634 format %{
8635 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8636 %}
8637 ins_encode %{
8638 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8639 Assembler::word, /*acquire*/ false, /*release*/ true,
8640 /*weak*/ false, $res$$Register);
8641 %}
8642 ins_pipe(pipe_slow);
8643 %}
8644
8645 // This pattern is generated automatically from cas.m4.
8646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8647 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8648 predicate(n->as_LoadStore()->barrier_data() == 0);
8649 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8650 ins_cost(2 * VOLATILE_REF_COST);
8651 effect(TEMP_DEF res, KILL cr);
8652 format %{
8653 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8654 %}
8655 ins_encode %{
8656 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8657 Assembler::xword, /*acquire*/ false, /*release*/ true,
8658 /*weak*/ false, $res$$Register);
8659 %}
8660 ins_pipe(pipe_slow);
8661 %}
8662
8663 // This pattern is generated automatically from cas.m4.
8664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8665 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8666 predicate(needs_acquiring_load_exclusive(n));
8667 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8668 ins_cost(VOLATILE_REF_COST);
8669 effect(TEMP_DEF res, KILL cr);
8670 format %{
8671 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8672 %}
8673 ins_encode %{
8674 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8675 Assembler::byte, /*acquire*/ true, /*release*/ true,
8676 /*weak*/ false, $res$$Register);
8677 __ sxtbw($res$$Register, $res$$Register);
8678 %}
8679 ins_pipe(pipe_slow);
8680 %}
8681
8682 // This pattern is generated automatically from cas.m4.
8683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8684 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8685 predicate(needs_acquiring_load_exclusive(n));
8686 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8687 ins_cost(VOLATILE_REF_COST);
8688 effect(TEMP_DEF res, KILL cr);
8689 format %{
8690 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8691 %}
8692 ins_encode %{
8693 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8694 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8695 /*weak*/ false, $res$$Register);
8696 __ sxthw($res$$Register, $res$$Register);
8697 %}
8698 ins_pipe(pipe_slow);
8699 %}
8700
8701 // This pattern is generated automatically from cas.m4.
8702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8703 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8704 predicate(needs_acquiring_load_exclusive(n));
8705 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8706 ins_cost(VOLATILE_REF_COST);
8707 effect(TEMP_DEF res, KILL cr);
8708 format %{
8709 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8710 %}
8711 ins_encode %{
8712 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8713 Assembler::word, /*acquire*/ true, /*release*/ true,
8714 /*weak*/ false, $res$$Register);
8715 %}
8716 ins_pipe(pipe_slow);
8717 %}
8718
8719 // This pattern is generated automatically from cas.m4.
8720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8721 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8722 predicate(needs_acquiring_load_exclusive(n));
8723 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8724 ins_cost(VOLATILE_REF_COST);
8725 effect(TEMP_DEF res, KILL cr);
8726 format %{
8727 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8728 %}
8729 ins_encode %{
8730 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731 Assembler::xword, /*acquire*/ true, /*release*/ true,
8732 /*weak*/ false, $res$$Register);
8733 %}
8734 ins_pipe(pipe_slow);
8735 %}
8736
8737 // This pattern is generated automatically from cas.m4.
8738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8739 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8740 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8741 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8742 ins_cost(VOLATILE_REF_COST);
8743 effect(TEMP_DEF res, KILL cr);
8744 format %{
8745 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8746 %}
8747 ins_encode %{
8748 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8749 Assembler::word, /*acquire*/ true, /*release*/ true,
8750 /*weak*/ false, $res$$Register);
8751 %}
8752 ins_pipe(pipe_slow);
8753 %}
8754
8755 // This pattern is generated automatically from cas.m4.
8756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8757 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8758 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8759 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8760 ins_cost(VOLATILE_REF_COST);
8761 effect(TEMP_DEF res, KILL cr);
8762 format %{
8763 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8764 %}
8765 ins_encode %{
8766 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8767 Assembler::xword, /*acquire*/ true, /*release*/ true,
8768 /*weak*/ false, $res$$Register);
8769 %}
8770 ins_pipe(pipe_slow);
8771 %}
8772
8773 // This pattern is generated automatically from cas.m4.
8774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8775 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8776 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8777 ins_cost(2 * VOLATILE_REF_COST);
8778 effect(KILL cr);
8779 format %{
8780 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8781 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8782 %}
8783 ins_encode %{
8784 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8785 Assembler::byte, /*acquire*/ false, /*release*/ true,
8786 /*weak*/ true, noreg);
8787 __ csetw($res$$Register, Assembler::EQ);
8788 %}
8789 ins_pipe(pipe_slow);
8790 %}
8791
8792 // This pattern is generated automatically from cas.m4.
8793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8794 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8795 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8796 ins_cost(2 * VOLATILE_REF_COST);
8797 effect(KILL cr);
8798 format %{
8799 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8800 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8801 %}
8802 ins_encode %{
8803 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8804 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8805 /*weak*/ true, noreg);
8806 __ csetw($res$$Register, Assembler::EQ);
8807 %}
8808 ins_pipe(pipe_slow);
8809 %}
8810
8811 // This pattern is generated automatically from cas.m4.
8812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8813 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8814 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8815 ins_cost(2 * VOLATILE_REF_COST);
8816 effect(KILL cr);
8817 format %{
8818 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8819 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8820 %}
8821 ins_encode %{
8822 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8823 Assembler::word, /*acquire*/ false, /*release*/ true,
8824 /*weak*/ true, noreg);
8825 __ csetw($res$$Register, Assembler::EQ);
8826 %}
8827 ins_pipe(pipe_slow);
8828 %}
8829
8830 // This pattern is generated automatically from cas.m4.
8831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8832 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8833 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8834 ins_cost(2 * VOLATILE_REF_COST);
8835 effect(KILL cr);
8836 format %{
8837 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8838 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8839 %}
8840 ins_encode %{
8841 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8842 Assembler::xword, /*acquire*/ false, /*release*/ true,
8843 /*weak*/ true, noreg);
8844 __ csetw($res$$Register, Assembler::EQ);
8845 %}
8846 ins_pipe(pipe_slow);
8847 %}
8848
8849 // This pattern is generated automatically from cas.m4.
8850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8851 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8852 predicate(n->as_LoadStore()->barrier_data() == 0);
8853 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8854 ins_cost(2 * VOLATILE_REF_COST);
8855 effect(KILL cr);
8856 format %{
8857 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8858 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8859 %}
8860 ins_encode %{
8861 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8862 Assembler::word, /*acquire*/ false, /*release*/ true,
8863 /*weak*/ true, noreg);
8864 __ csetw($res$$Register, Assembler::EQ);
8865 %}
8866 ins_pipe(pipe_slow);
8867 %}
8868
8869 // This pattern is generated automatically from cas.m4.
8870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8871 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8872 predicate(n->as_LoadStore()->barrier_data() == 0);
8873 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8874 ins_cost(2 * VOLATILE_REF_COST);
8875 effect(KILL cr);
8876 format %{
8877 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8878 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8879 %}
8880 ins_encode %{
8881 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8882 Assembler::xword, /*acquire*/ false, /*release*/ true,
8883 /*weak*/ true, noreg);
8884 __ csetw($res$$Register, Assembler::EQ);
8885 %}
8886 ins_pipe(pipe_slow);
8887 %}
8888
8889 // This pattern is generated automatically from cas.m4.
8890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8891 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8892 predicate(needs_acquiring_load_exclusive(n));
8893 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8894 ins_cost(VOLATILE_REF_COST);
8895 effect(KILL cr);
8896 format %{
8897 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8898 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8899 %}
8900 ins_encode %{
8901 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8902 Assembler::byte, /*acquire*/ true, /*release*/ true,
8903 /*weak*/ true, noreg);
8904 __ csetw($res$$Register, Assembler::EQ);
8905 %}
8906 ins_pipe(pipe_slow);
8907 %}
8908
8909 // This pattern is generated automatically from cas.m4.
8910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8911 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8912 predicate(needs_acquiring_load_exclusive(n));
8913 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8914 ins_cost(VOLATILE_REF_COST);
8915 effect(KILL cr);
8916 format %{
8917 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8918 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8919 %}
8920 ins_encode %{
8921 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8922 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8923 /*weak*/ true, noreg);
8924 __ csetw($res$$Register, Assembler::EQ);
8925 %}
8926 ins_pipe(pipe_slow);
8927 %}
8928
8929 // This pattern is generated automatically from cas.m4.
8930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8931 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8932 predicate(needs_acquiring_load_exclusive(n));
8933 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8934 ins_cost(VOLATILE_REF_COST);
8935 effect(KILL cr);
8936 format %{
8937 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8938 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8939 %}
8940 ins_encode %{
8941 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8942 Assembler::word, /*acquire*/ true, /*release*/ true,
8943 /*weak*/ true, noreg);
8944 __ csetw($res$$Register, Assembler::EQ);
8945 %}
8946 ins_pipe(pipe_slow);
8947 %}
8948
8949 // This pattern is generated automatically from cas.m4.
8950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8951 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8952 predicate(needs_acquiring_load_exclusive(n));
8953 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8954 ins_cost(VOLATILE_REF_COST);
8955 effect(KILL cr);
8956 format %{
8957 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8958 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8959 %}
8960 ins_encode %{
8961 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8962 Assembler::xword, /*acquire*/ true, /*release*/ true,
8963 /*weak*/ true, noreg);
8964 __ csetw($res$$Register, Assembler::EQ);
8965 %}
8966 ins_pipe(pipe_slow);
8967 %}
8968
8969 // This pattern is generated automatically from cas.m4.
8970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8971 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8972 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8973 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8974 ins_cost(VOLATILE_REF_COST);
8975 effect(KILL cr);
8976 format %{
8977 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8978 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8979 %}
8980 ins_encode %{
8981 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8982 Assembler::word, /*acquire*/ true, /*release*/ true,
8983 /*weak*/ true, noreg);
8984 __ csetw($res$$Register, Assembler::EQ);
8985 %}
8986 ins_pipe(pipe_slow);
8987 %}
8988
8989 // This pattern is generated automatically from cas.m4.
8990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8991 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8992 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8993 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8994 ins_cost(VOLATILE_REF_COST);
8995 effect(KILL cr);
8996 format %{
8997 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8998 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8999 %}
9000 ins_encode %{
9001 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9002 Assembler::xword, /*acquire*/ true, /*release*/ true,
9003 /*weak*/ true, noreg);
9004 __ csetw($res$$Register, Assembler::EQ);
9005 %}
9006 ins_pipe(pipe_slow);
9007 %}
9008
9009 // END This section of the file is automatically generated. Do not edit --------------
9010 // ---------------------------------------------------------------------
9011
9012 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9013 match(Set prev (GetAndSetI mem newv));
9014 ins_cost(2 * VOLATILE_REF_COST);
9015 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9016 ins_encode %{
9017 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9018 %}
9019 ins_pipe(pipe_serial);
9020 %}
9021
9022 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9023 match(Set prev (GetAndSetL mem newv));
9024 ins_cost(2 * VOLATILE_REF_COST);
9025 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9026 ins_encode %{
9027 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9028 %}
9029 ins_pipe(pipe_serial);
9030 %}
9031
9032 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9033 predicate(n->as_LoadStore()->barrier_data() == 0);
9034 match(Set prev (GetAndSetN mem newv));
9035 ins_cost(2 * VOLATILE_REF_COST);
9036 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9037 ins_encode %{
9038 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9039 %}
9040 ins_pipe(pipe_serial);
9041 %}
9042
9043 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9044 predicate(n->as_LoadStore()->barrier_data() == 0);
9045 match(Set prev (GetAndSetP mem newv));
9046 ins_cost(2 * VOLATILE_REF_COST);
9047 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9048 ins_encode %{
9049 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9050 %}
9051 ins_pipe(pipe_serial);
9052 %}
9053
9054 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9055 predicate(needs_acquiring_load_exclusive(n));
9056 match(Set prev (GetAndSetI mem newv));
9057 ins_cost(VOLATILE_REF_COST);
9058 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9059 ins_encode %{
9060 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9061 %}
9062 ins_pipe(pipe_serial);
9063 %}
9064
9065 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9066 predicate(needs_acquiring_load_exclusive(n));
9067 match(Set prev (GetAndSetL mem newv));
9068 ins_cost(VOLATILE_REF_COST);
9069 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9070 ins_encode %{
9071 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9072 %}
9073 ins_pipe(pipe_serial);
9074 %}
9075
9076 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9077 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9078 match(Set prev (GetAndSetN mem newv));
9079 ins_cost(VOLATILE_REF_COST);
9080 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9081 ins_encode %{
9082 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9083 %}
9084 ins_pipe(pipe_serial);
9085 %}
9086
9087 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9088 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9089 match(Set prev (GetAndSetP mem newv));
9090 ins_cost(VOLATILE_REF_COST);
9091 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9092 ins_encode %{
9093 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9094 %}
9095 ins_pipe(pipe_serial);
9096 %}
9097
9098
9099 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9100 match(Set newval (GetAndAddL mem incr));
9101 ins_cost(2 * VOLATILE_REF_COST + 1);
9102 format %{ "get_and_addL $newval, [$mem], $incr" %}
9103 ins_encode %{
9104 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9105 %}
9106 ins_pipe(pipe_serial);
9107 %}
9108
9109 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9110 predicate(n->as_LoadStore()->result_not_used());
9111 match(Set dummy (GetAndAddL mem incr));
9112 ins_cost(2 * VOLATILE_REF_COST);
9113 format %{ "get_and_addL [$mem], $incr" %}
9114 ins_encode %{
9115 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9116 %}
9117 ins_pipe(pipe_serial);
9118 %}
9119
9120 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9121 match(Set newval (GetAndAddL mem incr));
9122 ins_cost(2 * VOLATILE_REF_COST + 1);
9123 format %{ "get_and_addL $newval, [$mem], $incr" %}
9124 ins_encode %{
9125 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9126 %}
9127 ins_pipe(pipe_serial);
9128 %}
9129
9130 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9131 predicate(n->as_LoadStore()->result_not_used());
9132 match(Set dummy (GetAndAddL mem incr));
9133 ins_cost(2 * VOLATILE_REF_COST);
9134 format %{ "get_and_addL [$mem], $incr" %}
9135 ins_encode %{
9136 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9137 %}
9138 ins_pipe(pipe_serial);
9139 %}
9140
9141 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9142 match(Set newval (GetAndAddI mem incr));
9143 ins_cost(2 * VOLATILE_REF_COST + 1);
9144 format %{ "get_and_addI $newval, [$mem], $incr" %}
9145 ins_encode %{
9146 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9147 %}
9148 ins_pipe(pipe_serial);
9149 %}
9150
9151 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9152 predicate(n->as_LoadStore()->result_not_used());
9153 match(Set dummy (GetAndAddI mem incr));
9154 ins_cost(2 * VOLATILE_REF_COST);
9155 format %{ "get_and_addI [$mem], $incr" %}
9156 ins_encode %{
9157 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9158 %}
9159 ins_pipe(pipe_serial);
9160 %}
9161
9162 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9163 match(Set newval (GetAndAddI mem incr));
9164 ins_cost(2 * VOLATILE_REF_COST + 1);
9165 format %{ "get_and_addI $newval, [$mem], $incr" %}
9166 ins_encode %{
9167 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9168 %}
9169 ins_pipe(pipe_serial);
9170 %}
9171
9172 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9173 predicate(n->as_LoadStore()->result_not_used());
9174 match(Set dummy (GetAndAddI mem incr));
9175 ins_cost(2 * VOLATILE_REF_COST);
9176 format %{ "get_and_addI [$mem], $incr" %}
9177 ins_encode %{
9178 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9179 %}
9180 ins_pipe(pipe_serial);
9181 %}
9182
9183 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9184 predicate(needs_acquiring_load_exclusive(n));
9185 match(Set newval (GetAndAddL mem incr));
9186 ins_cost(VOLATILE_REF_COST + 1);
9187 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9188 ins_encode %{
9189 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9190 %}
9191 ins_pipe(pipe_serial);
9192 %}
9193
9194 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9195 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9196 match(Set dummy (GetAndAddL mem incr));
9197 ins_cost(VOLATILE_REF_COST);
9198 format %{ "get_and_addL_acq [$mem], $incr" %}
9199 ins_encode %{
9200 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9201 %}
9202 ins_pipe(pipe_serial);
9203 %}
9204
9205 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9206 predicate(needs_acquiring_load_exclusive(n));
9207 match(Set newval (GetAndAddL mem incr));
9208 ins_cost(VOLATILE_REF_COST + 1);
9209 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9210 ins_encode %{
9211 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9212 %}
9213 ins_pipe(pipe_serial);
9214 %}
9215
9216 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9217 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9218 match(Set dummy (GetAndAddL mem incr));
9219 ins_cost(VOLATILE_REF_COST);
9220 format %{ "get_and_addL_acq [$mem], $incr" %}
9221 ins_encode %{
9222 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9223 %}
9224 ins_pipe(pipe_serial);
9225 %}
9226
9227 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9228 predicate(needs_acquiring_load_exclusive(n));
9229 match(Set newval (GetAndAddI mem incr));
9230 ins_cost(VOLATILE_REF_COST + 1);
9231 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9232 ins_encode %{
9233 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9234 %}
9235 ins_pipe(pipe_serial);
9236 %}
9237
9238 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9239 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9240 match(Set dummy (GetAndAddI mem incr));
9241 ins_cost(VOLATILE_REF_COST);
9242 format %{ "get_and_addI_acq [$mem], $incr" %}
9243 ins_encode %{
9244 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9245 %}
9246 ins_pipe(pipe_serial);
9247 %}
9248
9249 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9250 predicate(needs_acquiring_load_exclusive(n));
9251 match(Set newval (GetAndAddI mem incr));
9252 ins_cost(VOLATILE_REF_COST + 1);
9253 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9254 ins_encode %{
9255 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9256 %}
9257 ins_pipe(pipe_serial);
9258 %}
9259
9260 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9261 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9262 match(Set dummy (GetAndAddI mem incr));
9263 ins_cost(VOLATILE_REF_COST);
9264 format %{ "get_and_addI_acq [$mem], $incr" %}
9265 ins_encode %{
9266 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9267 %}
9268 ins_pipe(pipe_serial);
9269 %}
9270
9271 // Manifest a CmpU result in an integer register.
9272 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9273 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9274 %{
9275 match(Set dst (CmpU3 src1 src2));
9276 effect(KILL flags);
9277
9278 ins_cost(INSN_COST * 3);
9279 format %{
9280 "cmpw $src1, $src2\n\t"
9281 "csetw $dst, ne\n\t"
9282 "cnegw $dst, lo\t# CmpU3(reg)"
9283 %}
9284 ins_encode %{
9285 __ cmpw($src1$$Register, $src2$$Register);
9286 __ csetw($dst$$Register, Assembler::NE);
9287 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9288 %}
9289
9290 ins_pipe(pipe_class_default);
9291 %}
9292
9293 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9294 %{
9295 match(Set dst (CmpU3 src1 src2));
9296 effect(KILL flags);
9297
9298 ins_cost(INSN_COST * 3);
9299 format %{
9300 "subsw zr, $src1, $src2\n\t"
9301 "csetw $dst, ne\n\t"
9302 "cnegw $dst, lo\t# CmpU3(imm)"
9303 %}
9304 ins_encode %{
9305 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9306 __ csetw($dst$$Register, Assembler::NE);
9307 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9308 %}
9309
9310 ins_pipe(pipe_class_default);
9311 %}
9312
9313 // Manifest a CmpUL result in an integer register.
9314 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9315 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9316 %{
9317 match(Set dst (CmpUL3 src1 src2));
9318 effect(KILL flags);
9319
9320 ins_cost(INSN_COST * 3);
9321 format %{
9322 "cmp $src1, $src2\n\t"
9323 "csetw $dst, ne\n\t"
9324 "cnegw $dst, lo\t# CmpUL3(reg)"
9325 %}
9326 ins_encode %{
9327 __ cmp($src1$$Register, $src2$$Register);
9328 __ csetw($dst$$Register, Assembler::NE);
9329 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9330 %}
9331
9332 ins_pipe(pipe_class_default);
9333 %}
9334
9335 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9336 %{
9337 match(Set dst (CmpUL3 src1 src2));
9338 effect(KILL flags);
9339
9340 ins_cost(INSN_COST * 3);
9341 format %{
9342 "subs zr, $src1, $src2\n\t"
9343 "csetw $dst, ne\n\t"
9344 "cnegw $dst, lo\t# CmpUL3(imm)"
9345 %}
9346 ins_encode %{
9347 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9348 __ csetw($dst$$Register, Assembler::NE);
9349 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9350 %}
9351
9352 ins_pipe(pipe_class_default);
9353 %}
9354
9355 // Manifest a CmpL result in an integer register.
9356 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9357 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9358 %{
9359 match(Set dst (CmpL3 src1 src2));
9360 effect(KILL flags);
9361
9362 ins_cost(INSN_COST * 3);
9363 format %{
9364 "cmp $src1, $src2\n\t"
9365 "csetw $dst, ne\n\t"
9366 "cnegw $dst, lt\t# CmpL3(reg)"
9367 %}
9368 ins_encode %{
9369 __ cmp($src1$$Register, $src2$$Register);
9370 __ csetw($dst$$Register, Assembler::NE);
9371 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9372 %}
9373
9374 ins_pipe(pipe_class_default);
9375 %}
9376
9377 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9378 %{
9379 match(Set dst (CmpL3 src1 src2));
9380 effect(KILL flags);
9381
9382 ins_cost(INSN_COST * 3);
9383 format %{
9384 "subs zr, $src1, $src2\n\t"
9385 "csetw $dst, ne\n\t"
9386 "cnegw $dst, lt\t# CmpL3(imm)"
9387 %}
9388 ins_encode %{
9389 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9390 __ csetw($dst$$Register, Assembler::NE);
9391 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9392 %}
9393
9394 ins_pipe(pipe_class_default);
9395 %}
9396
9397 // ============================================================================
9398 // Conditional Move Instructions
9399
9400 // n.b. we have identical rules for both a signed compare op (cmpOp)
9401 // and an unsigned compare op (cmpOpU). it would be nice if we could
9402 // define an op class which merged both inputs and use it to type the
9403 // argument to a single rule. unfortunatelyt his fails because the
9404 // opclass does not live up to the COND_INTER interface of its
9405 // component operands. When the generic code tries to negate the
9406 // operand it ends up running the generci Machoper::negate method
9407 // which throws a ShouldNotHappen. So, we have to provide two flavours
9408 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9409
9410 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9411 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9412
9413 ins_cost(INSN_COST * 2);
9414 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9415
9416 ins_encode %{
9417 __ cselw(as_Register($dst$$reg),
9418 as_Register($src2$$reg),
9419 as_Register($src1$$reg),
9420 (Assembler::Condition)$cmp$$cmpcode);
9421 %}
9422
9423 ins_pipe(icond_reg_reg);
9424 %}
9425
9426 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9427 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9428
9429 ins_cost(INSN_COST * 2);
9430 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9431
9432 ins_encode %{
9433 __ cselw(as_Register($dst$$reg),
9434 as_Register($src2$$reg),
9435 as_Register($src1$$reg),
9436 (Assembler::Condition)$cmp$$cmpcode);
9437 %}
9438
9439 ins_pipe(icond_reg_reg);
9440 %}
9441
9442 // special cases where one arg is zero
9443
9444 // n.b. this is selected in preference to the rule above because it
9445 // avoids loading constant 0 into a source register
9446
9447 // TODO
9448 // we ought only to be able to cull one of these variants as the ideal
9449 // transforms ought always to order the zero consistently (to left/right?)
9450
9451 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9452 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9453
9454 ins_cost(INSN_COST * 2);
9455 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9456
9457 ins_encode %{
9458 __ cselw(as_Register($dst$$reg),
9459 as_Register($src$$reg),
9460 zr,
9461 (Assembler::Condition)$cmp$$cmpcode);
9462 %}
9463
9464 ins_pipe(icond_reg);
9465 %}
9466
9467 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9468 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9469
9470 ins_cost(INSN_COST * 2);
9471 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9472
9473 ins_encode %{
9474 __ cselw(as_Register($dst$$reg),
9475 as_Register($src$$reg),
9476 zr,
9477 (Assembler::Condition)$cmp$$cmpcode);
9478 %}
9479
9480 ins_pipe(icond_reg);
9481 %}
9482
9483 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9484 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9485
9486 ins_cost(INSN_COST * 2);
9487 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9488
9489 ins_encode %{
9490 __ cselw(as_Register($dst$$reg),
9491 zr,
9492 as_Register($src$$reg),
9493 (Assembler::Condition)$cmp$$cmpcode);
9494 %}
9495
9496 ins_pipe(icond_reg);
9497 %}
9498
9499 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9500 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9501
9502 ins_cost(INSN_COST * 2);
9503 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9504
9505 ins_encode %{
9506 __ cselw(as_Register($dst$$reg),
9507 zr,
9508 as_Register($src$$reg),
9509 (Assembler::Condition)$cmp$$cmpcode);
9510 %}
9511
9512 ins_pipe(icond_reg);
9513 %}
9514
9515 // special case for creating a boolean 0 or 1
9516
9517 // n.b. this is selected in preference to the rule above because it
9518 // avoids loading constants 0 and 1 into a source register
9519
9520 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9521 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9522
9523 ins_cost(INSN_COST * 2);
9524 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9525
9526 ins_encode %{
9527 // equivalently
9528 // cset(as_Register($dst$$reg),
9529 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9530 __ csincw(as_Register($dst$$reg),
9531 zr,
9532 zr,
9533 (Assembler::Condition)$cmp$$cmpcode);
9534 %}
9535
9536 ins_pipe(icond_none);
9537 %}
9538
9539 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9540 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9541
9542 ins_cost(INSN_COST * 2);
9543 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9544
9545 ins_encode %{
9546 // equivalently
9547 // cset(as_Register($dst$$reg),
9548 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9549 __ csincw(as_Register($dst$$reg),
9550 zr,
9551 zr,
9552 (Assembler::Condition)$cmp$$cmpcode);
9553 %}
9554
9555 ins_pipe(icond_none);
9556 %}
9557
9558 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9559 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9560
9561 ins_cost(INSN_COST * 2);
9562 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9563
9564 ins_encode %{
9565 __ csel(as_Register($dst$$reg),
9566 as_Register($src2$$reg),
9567 as_Register($src1$$reg),
9568 (Assembler::Condition)$cmp$$cmpcode);
9569 %}
9570
9571 ins_pipe(icond_reg_reg);
9572 %}
9573
9574 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9575 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9576
9577 ins_cost(INSN_COST * 2);
9578 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9579
9580 ins_encode %{
9581 __ csel(as_Register($dst$$reg),
9582 as_Register($src2$$reg),
9583 as_Register($src1$$reg),
9584 (Assembler::Condition)$cmp$$cmpcode);
9585 %}
9586
9587 ins_pipe(icond_reg_reg);
9588 %}
9589
9590 // special cases where one arg is zero
9591
9592 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9593 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9594
9595 ins_cost(INSN_COST * 2);
9596 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9597
9598 ins_encode %{
9599 __ csel(as_Register($dst$$reg),
9600 zr,
9601 as_Register($src$$reg),
9602 (Assembler::Condition)$cmp$$cmpcode);
9603 %}
9604
9605 ins_pipe(icond_reg);
9606 %}
9607
9608 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9609 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9610
9611 ins_cost(INSN_COST * 2);
9612 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9613
9614 ins_encode %{
9615 __ csel(as_Register($dst$$reg),
9616 zr,
9617 as_Register($src$$reg),
9618 (Assembler::Condition)$cmp$$cmpcode);
9619 %}
9620
9621 ins_pipe(icond_reg);
9622 %}
9623
9624 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9625 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9626
9627 ins_cost(INSN_COST * 2);
9628 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9629
9630 ins_encode %{
9631 __ csel(as_Register($dst$$reg),
9632 as_Register($src$$reg),
9633 zr,
9634 (Assembler::Condition)$cmp$$cmpcode);
9635 %}
9636
9637 ins_pipe(icond_reg);
9638 %}
9639
9640 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9641 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9642
9643 ins_cost(INSN_COST * 2);
9644 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9645
9646 ins_encode %{
9647 __ csel(as_Register($dst$$reg),
9648 as_Register($src$$reg),
9649 zr,
9650 (Assembler::Condition)$cmp$$cmpcode);
9651 %}
9652
9653 ins_pipe(icond_reg);
9654 %}
9655
9656 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9657 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9658
9659 ins_cost(INSN_COST * 2);
9660 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9661
9662 ins_encode %{
9663 __ csel(as_Register($dst$$reg),
9664 as_Register($src2$$reg),
9665 as_Register($src1$$reg),
9666 (Assembler::Condition)$cmp$$cmpcode);
9667 %}
9668
9669 ins_pipe(icond_reg_reg);
9670 %}
9671
9672 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9673 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9674
9675 ins_cost(INSN_COST * 2);
9676 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9677
9678 ins_encode %{
9679 __ csel(as_Register($dst$$reg),
9680 as_Register($src2$$reg),
9681 as_Register($src1$$reg),
9682 (Assembler::Condition)$cmp$$cmpcode);
9683 %}
9684
9685 ins_pipe(icond_reg_reg);
9686 %}
9687
9688 // special cases where one arg is zero
9689
9690 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9691 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9692
9693 ins_cost(INSN_COST * 2);
9694 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9695
9696 ins_encode %{
9697 __ csel(as_Register($dst$$reg),
9698 zr,
9699 as_Register($src$$reg),
9700 (Assembler::Condition)$cmp$$cmpcode);
9701 %}
9702
9703 ins_pipe(icond_reg);
9704 %}
9705
9706 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9707 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9708
9709 ins_cost(INSN_COST * 2);
9710 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9711
9712 ins_encode %{
9713 __ csel(as_Register($dst$$reg),
9714 zr,
9715 as_Register($src$$reg),
9716 (Assembler::Condition)$cmp$$cmpcode);
9717 %}
9718
9719 ins_pipe(icond_reg);
9720 %}
9721
9722 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9723 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9724
9725 ins_cost(INSN_COST * 2);
9726 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9727
9728 ins_encode %{
9729 __ csel(as_Register($dst$$reg),
9730 as_Register($src$$reg),
9731 zr,
9732 (Assembler::Condition)$cmp$$cmpcode);
9733 %}
9734
9735 ins_pipe(icond_reg);
9736 %}
9737
9738 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9739 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9740
9741 ins_cost(INSN_COST * 2);
9742 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9743
9744 ins_encode %{
9745 __ csel(as_Register($dst$$reg),
9746 as_Register($src$$reg),
9747 zr,
9748 (Assembler::Condition)$cmp$$cmpcode);
9749 %}
9750
9751 ins_pipe(icond_reg);
9752 %}
9753
9754 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9755 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9756
9757 ins_cost(INSN_COST * 2);
9758 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9759
9760 ins_encode %{
9761 __ cselw(as_Register($dst$$reg),
9762 as_Register($src2$$reg),
9763 as_Register($src1$$reg),
9764 (Assembler::Condition)$cmp$$cmpcode);
9765 %}
9766
9767 ins_pipe(icond_reg_reg);
9768 %}
9769
9770 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9771 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9772
9773 ins_cost(INSN_COST * 2);
9774 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9775
9776 ins_encode %{
9777 __ cselw(as_Register($dst$$reg),
9778 as_Register($src2$$reg),
9779 as_Register($src1$$reg),
9780 (Assembler::Condition)$cmp$$cmpcode);
9781 %}
9782
9783 ins_pipe(icond_reg_reg);
9784 %}
9785
9786 // special cases where one arg is zero
9787
9788 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9789 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9790
9791 ins_cost(INSN_COST * 2);
9792 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9793
9794 ins_encode %{
9795 __ cselw(as_Register($dst$$reg),
9796 zr,
9797 as_Register($src$$reg),
9798 (Assembler::Condition)$cmp$$cmpcode);
9799 %}
9800
9801 ins_pipe(icond_reg);
9802 %}
9803
9804 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9805 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9806
9807 ins_cost(INSN_COST * 2);
9808 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9809
9810 ins_encode %{
9811 __ cselw(as_Register($dst$$reg),
9812 zr,
9813 as_Register($src$$reg),
9814 (Assembler::Condition)$cmp$$cmpcode);
9815 %}
9816
9817 ins_pipe(icond_reg);
9818 %}
9819
9820 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9821 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9822
9823 ins_cost(INSN_COST * 2);
9824 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9825
9826 ins_encode %{
9827 __ cselw(as_Register($dst$$reg),
9828 as_Register($src$$reg),
9829 zr,
9830 (Assembler::Condition)$cmp$$cmpcode);
9831 %}
9832
9833 ins_pipe(icond_reg);
9834 %}
9835
9836 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9837 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9838
9839 ins_cost(INSN_COST * 2);
9840 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9841
9842 ins_encode %{
9843 __ cselw(as_Register($dst$$reg),
9844 as_Register($src$$reg),
9845 zr,
9846 (Assembler::Condition)$cmp$$cmpcode);
9847 %}
9848
9849 ins_pipe(icond_reg);
9850 %}
9851
9852 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9853 %{
9854 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9855
9856 ins_cost(INSN_COST * 3);
9857
9858 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9859 ins_encode %{
9860 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9861 __ fcsels(as_FloatRegister($dst$$reg),
9862 as_FloatRegister($src2$$reg),
9863 as_FloatRegister($src1$$reg),
9864 cond);
9865 %}
9866
9867 ins_pipe(fp_cond_reg_reg_s);
9868 %}
9869
9870 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9871 %{
9872 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9873
9874 ins_cost(INSN_COST * 3);
9875
9876 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9877 ins_encode %{
9878 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9879 __ fcsels(as_FloatRegister($dst$$reg),
9880 as_FloatRegister($src2$$reg),
9881 as_FloatRegister($src1$$reg),
9882 cond);
9883 %}
9884
9885 ins_pipe(fp_cond_reg_reg_s);
9886 %}
9887
9888 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9889 %{
9890 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9891
9892 ins_cost(INSN_COST * 3);
9893
9894 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9895 ins_encode %{
9896 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9897 __ fcseld(as_FloatRegister($dst$$reg),
9898 as_FloatRegister($src2$$reg),
9899 as_FloatRegister($src1$$reg),
9900 cond);
9901 %}
9902
9903 ins_pipe(fp_cond_reg_reg_d);
9904 %}
9905
9906 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9907 %{
9908 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9909
9910 ins_cost(INSN_COST * 3);
9911
9912 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9913 ins_encode %{
9914 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9915 __ fcseld(as_FloatRegister($dst$$reg),
9916 as_FloatRegister($src2$$reg),
9917 as_FloatRegister($src1$$reg),
9918 cond);
9919 %}
9920
9921 ins_pipe(fp_cond_reg_reg_d);
9922 %}
9923
9924 // ============================================================================
9925 // Arithmetic Instructions
9926 //
9927
9928 // Integer Addition
9929
9930 // TODO
9931 // these currently employ operations which do not set CR and hence are
9932 // not flagged as killing CR but we would like to isolate the cases
9933 // where we want to set flags from those where we don't. need to work
9934 // out how to do that.
9935
9936 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9937 match(Set dst (AddI src1 src2));
9938
9939 ins_cost(INSN_COST);
9940 format %{ "addw $dst, $src1, $src2" %}
9941
9942 ins_encode %{
9943 __ addw(as_Register($dst$$reg),
9944 as_Register($src1$$reg),
9945 as_Register($src2$$reg));
9946 %}
9947
9948 ins_pipe(ialu_reg_reg);
9949 %}
9950
9951 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9952 match(Set dst (AddI src1 src2));
9953
9954 ins_cost(INSN_COST);
9955 format %{ "addw $dst, $src1, $src2" %}
9956
9957 // use opcode to indicate that this is an add not a sub
9958 opcode(0x0);
9959
9960 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9961
9962 ins_pipe(ialu_reg_imm);
9963 %}
9964
9965 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9966 match(Set dst (AddI (ConvL2I src1) src2));
9967
9968 ins_cost(INSN_COST);
9969 format %{ "addw $dst, $src1, $src2" %}
9970
9971 // use opcode to indicate that this is an add not a sub
9972 opcode(0x0);
9973
9974 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9975
9976 ins_pipe(ialu_reg_imm);
9977 %}
9978
9979 // Pointer Addition
9980 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9981 match(Set dst (AddP src1 src2));
9982
9983 ins_cost(INSN_COST);
9984 format %{ "add $dst, $src1, $src2\t# ptr" %}
9985
9986 ins_encode %{
9987 __ add(as_Register($dst$$reg),
9988 as_Register($src1$$reg),
9989 as_Register($src2$$reg));
9990 %}
9991
9992 ins_pipe(ialu_reg_reg);
9993 %}
9994
9995 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9996 match(Set dst (AddP src1 (ConvI2L src2)));
9997
9998 ins_cost(1.9 * INSN_COST);
9999 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10000
10001 ins_encode %{
10002 __ add(as_Register($dst$$reg),
10003 as_Register($src1$$reg),
10004 as_Register($src2$$reg), ext::sxtw);
10005 %}
10006
10007 ins_pipe(ialu_reg_reg);
10008 %}
10009
10010 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10011 match(Set dst (AddP src1 (LShiftL src2 scale)));
10012
10013 ins_cost(1.9 * INSN_COST);
10014 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10015
10016 ins_encode %{
10017 __ lea(as_Register($dst$$reg),
10018 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10019 Address::lsl($scale$$constant)));
10020 %}
10021
10022 ins_pipe(ialu_reg_reg_shift);
10023 %}
10024
10025 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10026 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10027
10028 ins_cost(1.9 * INSN_COST);
10029 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10030
10031 ins_encode %{
10032 __ lea(as_Register($dst$$reg),
10033 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10034 Address::sxtw($scale$$constant)));
10035 %}
10036
10037 ins_pipe(ialu_reg_reg_shift);
10038 %}
10039
10040 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10041 match(Set dst (LShiftL (ConvI2L src) scale));
10042
10043 ins_cost(INSN_COST);
10044 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10045
10046 ins_encode %{
10047 __ sbfiz(as_Register($dst$$reg),
10048 as_Register($src$$reg),
10049 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10050 %}
10051
10052 ins_pipe(ialu_reg_shift);
10053 %}
10054
10055 // Pointer Immediate Addition
10056 // n.b. this needs to be more expensive than using an indirect memory
10057 // operand
10058 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10059 match(Set dst (AddP src1 src2));
10060
10061 ins_cost(INSN_COST);
10062 format %{ "add $dst, $src1, $src2\t# ptr" %}
10063
10064 // use opcode to indicate that this is an add not a sub
10065 opcode(0x0);
10066
10067 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10068
10069 ins_pipe(ialu_reg_imm);
10070 %}
10071
10072 // Long Addition
10073 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10074
10075 match(Set dst (AddL src1 src2));
10076
10077 ins_cost(INSN_COST);
10078 format %{ "add $dst, $src1, $src2" %}
10079
10080 ins_encode %{
10081 __ add(as_Register($dst$$reg),
10082 as_Register($src1$$reg),
10083 as_Register($src2$$reg));
10084 %}
10085
10086 ins_pipe(ialu_reg_reg);
10087 %}
10088
10089 // No constant pool entries requiredLong Immediate Addition.
10090 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10091 match(Set dst (AddL src1 src2));
10092
10093 ins_cost(INSN_COST);
10094 format %{ "add $dst, $src1, $src2" %}
10095
10096 // use opcode to indicate that this is an add not a sub
10097 opcode(0x0);
10098
10099 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10100
10101 ins_pipe(ialu_reg_imm);
10102 %}
10103
10104 // Integer Subtraction
10105 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10106 match(Set dst (SubI src1 src2));
10107
10108 ins_cost(INSN_COST);
10109 format %{ "subw $dst, $src1, $src2" %}
10110
10111 ins_encode %{
10112 __ subw(as_Register($dst$$reg),
10113 as_Register($src1$$reg),
10114 as_Register($src2$$reg));
10115 %}
10116
10117 ins_pipe(ialu_reg_reg);
10118 %}
10119
10120 // Immediate Subtraction
10121 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10122 match(Set dst (SubI src1 src2));
10123
10124 ins_cost(INSN_COST);
10125 format %{ "subw $dst, $src1, $src2" %}
10126
10127 // use opcode to indicate that this is a sub not an add
10128 opcode(0x1);
10129
10130 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10131
10132 ins_pipe(ialu_reg_imm);
10133 %}
10134
10135 // Long Subtraction
10136 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10137
10138 match(Set dst (SubL src1 src2));
10139
10140 ins_cost(INSN_COST);
10141 format %{ "sub $dst, $src1, $src2" %}
10142
10143 ins_encode %{
10144 __ sub(as_Register($dst$$reg),
10145 as_Register($src1$$reg),
10146 as_Register($src2$$reg));
10147 %}
10148
10149 ins_pipe(ialu_reg_reg);
10150 %}
10151
10152 // No constant pool entries requiredLong Immediate Subtraction.
10153 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10154 match(Set dst (SubL src1 src2));
10155
10156 ins_cost(INSN_COST);
10157 format %{ "sub$dst, $src1, $src2" %}
10158
10159 // use opcode to indicate that this is a sub not an add
10160 opcode(0x1);
10161
10162 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10163
10164 ins_pipe(ialu_reg_imm);
10165 %}
10166
10167 // Integer Negation (special case for sub)
10168
10169 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10170 match(Set dst (SubI zero src));
10171
10172 ins_cost(INSN_COST);
10173 format %{ "negw $dst, $src\t# int" %}
10174
10175 ins_encode %{
10176 __ negw(as_Register($dst$$reg),
10177 as_Register($src$$reg));
10178 %}
10179
10180 ins_pipe(ialu_reg);
10181 %}
10182
10183 // Long Negation
10184
10185 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10186 match(Set dst (SubL zero src));
10187
10188 ins_cost(INSN_COST);
10189 format %{ "neg $dst, $src\t# long" %}
10190
10191 ins_encode %{
10192 __ neg(as_Register($dst$$reg),
10193 as_Register($src$$reg));
10194 %}
10195
10196 ins_pipe(ialu_reg);
10197 %}
10198
10199 // Integer Multiply
10200
10201 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10202 match(Set dst (MulI src1 src2));
10203
10204 ins_cost(INSN_COST * 3);
10205 format %{ "mulw $dst, $src1, $src2" %}
10206
10207 ins_encode %{
10208 __ mulw(as_Register($dst$$reg),
10209 as_Register($src1$$reg),
10210 as_Register($src2$$reg));
10211 %}
10212
10213 ins_pipe(imul_reg_reg);
10214 %}
10215
10216 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10217 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10218
10219 ins_cost(INSN_COST * 3);
10220 format %{ "smull $dst, $src1, $src2" %}
10221
10222 ins_encode %{
10223 __ smull(as_Register($dst$$reg),
10224 as_Register($src1$$reg),
10225 as_Register($src2$$reg));
10226 %}
10227
10228 ins_pipe(imul_reg_reg);
10229 %}
10230
10231 // Long Multiply
10232
10233 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10234 match(Set dst (MulL src1 src2));
10235
10236 ins_cost(INSN_COST * 5);
10237 format %{ "mul $dst, $src1, $src2" %}
10238
10239 ins_encode %{
10240 __ mul(as_Register($dst$$reg),
10241 as_Register($src1$$reg),
10242 as_Register($src2$$reg));
10243 %}
10244
10245 ins_pipe(lmul_reg_reg);
10246 %}
10247
10248 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10249 %{
10250 match(Set dst (MulHiL src1 src2));
10251
10252 ins_cost(INSN_COST * 7);
10253 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10254
10255 ins_encode %{
10256 __ smulh(as_Register($dst$$reg),
10257 as_Register($src1$$reg),
10258 as_Register($src2$$reg));
10259 %}
10260
10261 ins_pipe(lmul_reg_reg);
10262 %}
10263
10264 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10265 %{
10266 match(Set dst (UMulHiL src1 src2));
10267
10268 ins_cost(INSN_COST * 7);
10269 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10270
10271 ins_encode %{
10272 __ umulh(as_Register($dst$$reg),
10273 as_Register($src1$$reg),
10274 as_Register($src2$$reg));
10275 %}
10276
10277 ins_pipe(lmul_reg_reg);
10278 %}
10279
10280 // Combined Integer Multiply & Add/Sub
10281
10282 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10283 match(Set dst (AddI src3 (MulI src1 src2)));
10284
10285 ins_cost(INSN_COST * 3);
10286 format %{ "madd $dst, $src1, $src2, $src3" %}
10287
10288 ins_encode %{
10289 __ maddw(as_Register($dst$$reg),
10290 as_Register($src1$$reg),
10291 as_Register($src2$$reg),
10292 as_Register($src3$$reg));
10293 %}
10294
10295 ins_pipe(imac_reg_reg);
10296 %}
10297
10298 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10299 match(Set dst (SubI src3 (MulI src1 src2)));
10300
10301 ins_cost(INSN_COST * 3);
10302 format %{ "msub $dst, $src1, $src2, $src3" %}
10303
10304 ins_encode %{
10305 __ msubw(as_Register($dst$$reg),
10306 as_Register($src1$$reg),
10307 as_Register($src2$$reg),
10308 as_Register($src3$$reg));
10309 %}
10310
10311 ins_pipe(imac_reg_reg);
10312 %}
10313
10314 // Combined Integer Multiply & Neg
10315
10316 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10317 match(Set dst (MulI (SubI zero src1) src2));
10318
10319 ins_cost(INSN_COST * 3);
10320 format %{ "mneg $dst, $src1, $src2" %}
10321
10322 ins_encode %{
10323 __ mnegw(as_Register($dst$$reg),
10324 as_Register($src1$$reg),
10325 as_Register($src2$$reg));
10326 %}
10327
10328 ins_pipe(imac_reg_reg);
10329 %}
10330
10331 // Combined Long Multiply & Add/Sub
10332
10333 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10334 match(Set dst (AddL src3 (MulL src1 src2)));
10335
10336 ins_cost(INSN_COST * 5);
10337 format %{ "madd $dst, $src1, $src2, $src3" %}
10338
10339 ins_encode %{
10340 __ madd(as_Register($dst$$reg),
10341 as_Register($src1$$reg),
10342 as_Register($src2$$reg),
10343 as_Register($src3$$reg));
10344 %}
10345
10346 ins_pipe(lmac_reg_reg);
10347 %}
10348
10349 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10350 match(Set dst (SubL src3 (MulL src1 src2)));
10351
10352 ins_cost(INSN_COST * 5);
10353 format %{ "msub $dst, $src1, $src2, $src3" %}
10354
10355 ins_encode %{
10356 __ msub(as_Register($dst$$reg),
10357 as_Register($src1$$reg),
10358 as_Register($src2$$reg),
10359 as_Register($src3$$reg));
10360 %}
10361
10362 ins_pipe(lmac_reg_reg);
10363 %}
10364
10365 // Combined Long Multiply & Neg
10366
10367 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10368 match(Set dst (MulL (SubL zero src1) src2));
10369
10370 ins_cost(INSN_COST * 5);
10371 format %{ "mneg $dst, $src1, $src2" %}
10372
10373 ins_encode %{
10374 __ mneg(as_Register($dst$$reg),
10375 as_Register($src1$$reg),
10376 as_Register($src2$$reg));
10377 %}
10378
10379 ins_pipe(lmac_reg_reg);
10380 %}
10381
10382 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10383
10384 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10385 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10386
10387 ins_cost(INSN_COST * 3);
10388 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10389
10390 ins_encode %{
10391 __ smaddl(as_Register($dst$$reg),
10392 as_Register($src1$$reg),
10393 as_Register($src2$$reg),
10394 as_Register($src3$$reg));
10395 %}
10396
10397 ins_pipe(imac_reg_reg);
10398 %}
10399
10400 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10401 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10402
10403 ins_cost(INSN_COST * 3);
10404 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10405
10406 ins_encode %{
10407 __ smsubl(as_Register($dst$$reg),
10408 as_Register($src1$$reg),
10409 as_Register($src2$$reg),
10410 as_Register($src3$$reg));
10411 %}
10412
10413 ins_pipe(imac_reg_reg);
10414 %}
10415
10416 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10417 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10418
10419 ins_cost(INSN_COST * 3);
10420 format %{ "smnegl $dst, $src1, $src2" %}
10421
10422 ins_encode %{
10423 __ smnegl(as_Register($dst$$reg),
10424 as_Register($src1$$reg),
10425 as_Register($src2$$reg));
10426 %}
10427
10428 ins_pipe(imac_reg_reg);
10429 %}
10430
10431 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10432
10433 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10434 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10435
10436 ins_cost(INSN_COST * 5);
10437 format %{ "mulw rscratch1, $src1, $src2\n\t"
10438 "maddw $dst, $src3, $src4, rscratch1" %}
10439
10440 ins_encode %{
10441 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10442 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10443
10444 ins_pipe(imac_reg_reg);
10445 %}
10446
10447 // Integer Divide
10448
10449 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10450 match(Set dst (DivI src1 src2));
10451
10452 ins_cost(INSN_COST * 19);
10453 format %{ "sdivw $dst, $src1, $src2" %}
10454
10455 ins_encode(aarch64_enc_divw(dst, src1, src2));
10456 ins_pipe(idiv_reg_reg);
10457 %}
10458
10459 // Long Divide
10460
10461 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10462 match(Set dst (DivL src1 src2));
10463
10464 ins_cost(INSN_COST * 35);
10465 format %{ "sdiv $dst, $src1, $src2" %}
10466
10467 ins_encode(aarch64_enc_div(dst, src1, src2));
10468 ins_pipe(ldiv_reg_reg);
10469 %}
10470
10471 // Integer Remainder
10472
10473 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10474 match(Set dst (ModI src1 src2));
10475
10476 ins_cost(INSN_COST * 22);
10477 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10478 "msubw $dst, rscratch1, $src2, $src1" %}
10479
10480 ins_encode(aarch64_enc_modw(dst, src1, src2));
10481 ins_pipe(idiv_reg_reg);
10482 %}
10483
10484 // Long Remainder
10485
10486 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10487 match(Set dst (ModL src1 src2));
10488
10489 ins_cost(INSN_COST * 38);
10490 format %{ "sdiv rscratch1, $src1, $src2\n"
10491 "msub $dst, rscratch1, $src2, $src1" %}
10492
10493 ins_encode(aarch64_enc_mod(dst, src1, src2));
10494 ins_pipe(ldiv_reg_reg);
10495 %}
10496
10497 // Unsigned Integer Divide
10498
10499 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10500 match(Set dst (UDivI src1 src2));
10501
10502 ins_cost(INSN_COST * 19);
10503 format %{ "udivw $dst, $src1, $src2" %}
10504
10505 ins_encode %{
10506 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10507 %}
10508
10509 ins_pipe(idiv_reg_reg);
10510 %}
10511
10512 // Unsigned Long Divide
10513
10514 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10515 match(Set dst (UDivL src1 src2));
10516
10517 ins_cost(INSN_COST * 35);
10518 format %{ "udiv $dst, $src1, $src2" %}
10519
10520 ins_encode %{
10521 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10522 %}
10523
10524 ins_pipe(ldiv_reg_reg);
10525 %}
10526
10527 // Unsigned Integer Remainder
10528
10529 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10530 match(Set dst (UModI src1 src2));
10531
10532 ins_cost(INSN_COST * 22);
10533 format %{ "udivw rscratch1, $src1, $src2\n\t"
10534 "msubw $dst, rscratch1, $src2, $src1" %}
10535
10536 ins_encode %{
10537 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10538 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10539 %}
10540
10541 ins_pipe(idiv_reg_reg);
10542 %}
10543
10544 // Unsigned Long Remainder
10545
10546 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10547 match(Set dst (UModL src1 src2));
10548
10549 ins_cost(INSN_COST * 38);
10550 format %{ "udiv rscratch1, $src1, $src2\n"
10551 "msub $dst, rscratch1, $src2, $src1" %}
10552
10553 ins_encode %{
10554 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10555 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10556 %}
10557
10558 ins_pipe(ldiv_reg_reg);
10559 %}
10560
10561 // Integer Shifts
10562
10563 // Shift Left Register
10564 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10565 match(Set dst (LShiftI src1 src2));
10566
10567 ins_cost(INSN_COST * 2);
10568 format %{ "lslvw $dst, $src1, $src2" %}
10569
10570 ins_encode %{
10571 __ lslvw(as_Register($dst$$reg),
10572 as_Register($src1$$reg),
10573 as_Register($src2$$reg));
10574 %}
10575
10576 ins_pipe(ialu_reg_reg_vshift);
10577 %}
10578
10579 // Shift Left Immediate
10580 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10581 match(Set dst (LShiftI src1 src2));
10582
10583 ins_cost(INSN_COST);
10584 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10585
10586 ins_encode %{
10587 __ lslw(as_Register($dst$$reg),
10588 as_Register($src1$$reg),
10589 $src2$$constant & 0x1f);
10590 %}
10591
10592 ins_pipe(ialu_reg_shift);
10593 %}
10594
10595 // Shift Right Logical Register
10596 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10597 match(Set dst (URShiftI src1 src2));
10598
10599 ins_cost(INSN_COST * 2);
10600 format %{ "lsrvw $dst, $src1, $src2" %}
10601
10602 ins_encode %{
10603 __ lsrvw(as_Register($dst$$reg),
10604 as_Register($src1$$reg),
10605 as_Register($src2$$reg));
10606 %}
10607
10608 ins_pipe(ialu_reg_reg_vshift);
10609 %}
10610
10611 // Shift Right Logical Immediate
10612 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10613 match(Set dst (URShiftI src1 src2));
10614
10615 ins_cost(INSN_COST);
10616 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10617
10618 ins_encode %{
10619 __ lsrw(as_Register($dst$$reg),
10620 as_Register($src1$$reg),
10621 $src2$$constant & 0x1f);
10622 %}
10623
10624 ins_pipe(ialu_reg_shift);
10625 %}
10626
10627 // Shift Right Arithmetic Register
10628 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10629 match(Set dst (RShiftI src1 src2));
10630
10631 ins_cost(INSN_COST * 2);
10632 format %{ "asrvw $dst, $src1, $src2" %}
10633
10634 ins_encode %{
10635 __ asrvw(as_Register($dst$$reg),
10636 as_Register($src1$$reg),
10637 as_Register($src2$$reg));
10638 %}
10639
10640 ins_pipe(ialu_reg_reg_vshift);
10641 %}
10642
10643 // Shift Right Arithmetic Immediate
10644 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10645 match(Set dst (RShiftI src1 src2));
10646
10647 ins_cost(INSN_COST);
10648 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10649
10650 ins_encode %{
10651 __ asrw(as_Register($dst$$reg),
10652 as_Register($src1$$reg),
10653 $src2$$constant & 0x1f);
10654 %}
10655
10656 ins_pipe(ialu_reg_shift);
10657 %}
10658
10659 // Combined Int Mask and Right Shift (using UBFM)
10660 // TODO
10661
10662 // Long Shifts
10663
10664 // Shift Left Register
10665 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10666 match(Set dst (LShiftL src1 src2));
10667
10668 ins_cost(INSN_COST * 2);
10669 format %{ "lslv $dst, $src1, $src2" %}
10670
10671 ins_encode %{
10672 __ lslv(as_Register($dst$$reg),
10673 as_Register($src1$$reg),
10674 as_Register($src2$$reg));
10675 %}
10676
10677 ins_pipe(ialu_reg_reg_vshift);
10678 %}
10679
10680 // Shift Left Immediate
10681 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10682 match(Set dst (LShiftL src1 src2));
10683
10684 ins_cost(INSN_COST);
10685 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10686
10687 ins_encode %{
10688 __ lsl(as_Register($dst$$reg),
10689 as_Register($src1$$reg),
10690 $src2$$constant & 0x3f);
10691 %}
10692
10693 ins_pipe(ialu_reg_shift);
10694 %}
10695
10696 // Shift Right Logical Register
10697 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10698 match(Set dst (URShiftL src1 src2));
10699
10700 ins_cost(INSN_COST * 2);
10701 format %{ "lsrv $dst, $src1, $src2" %}
10702
10703 ins_encode %{
10704 __ lsrv(as_Register($dst$$reg),
10705 as_Register($src1$$reg),
10706 as_Register($src2$$reg));
10707 %}
10708
10709 ins_pipe(ialu_reg_reg_vshift);
10710 %}
10711
10712 // Shift Right Logical Immediate
10713 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10714 match(Set dst (URShiftL src1 src2));
10715
10716 ins_cost(INSN_COST);
10717 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10718
10719 ins_encode %{
10720 __ lsr(as_Register($dst$$reg),
10721 as_Register($src1$$reg),
10722 $src2$$constant & 0x3f);
10723 %}
10724
10725 ins_pipe(ialu_reg_shift);
10726 %}
10727
10728 // A special-case pattern for card table stores.
10729 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10730 match(Set dst (URShiftL (CastP2X src1) src2));
10731
10732 ins_cost(INSN_COST);
10733 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10734
10735 ins_encode %{
10736 __ lsr(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 $src2$$constant & 0x3f);
10739 %}
10740
10741 ins_pipe(ialu_reg_shift);
10742 %}
10743
10744 // Shift Right Arithmetic Register
10745 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10746 match(Set dst (RShiftL src1 src2));
10747
10748 ins_cost(INSN_COST * 2);
10749 format %{ "asrv $dst, $src1, $src2" %}
10750
10751 ins_encode %{
10752 __ asrv(as_Register($dst$$reg),
10753 as_Register($src1$$reg),
10754 as_Register($src2$$reg));
10755 %}
10756
10757 ins_pipe(ialu_reg_reg_vshift);
10758 %}
10759
10760 // Shift Right Arithmetic Immediate
10761 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10762 match(Set dst (RShiftL src1 src2));
10763
10764 ins_cost(INSN_COST);
10765 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10766
10767 ins_encode %{
10768 __ asr(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 $src2$$constant & 0x3f);
10771 %}
10772
10773 ins_pipe(ialu_reg_shift);
10774 %}
10775
10776 // BEGIN This section of the file is automatically generated. Do not edit --------------
10777 // This section is generated from aarch64_ad.m4
10778
10779 // This pattern is automatically generated from aarch64_ad.m4
10780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10781 instruct regL_not_reg(iRegLNoSp dst,
10782 iRegL src1, immL_M1 m1,
10783 rFlagsReg cr) %{
10784 match(Set dst (XorL src1 m1));
10785 ins_cost(INSN_COST);
10786 format %{ "eon $dst, $src1, zr" %}
10787
10788 ins_encode %{
10789 __ eon(as_Register($dst$$reg),
10790 as_Register($src1$$reg),
10791 zr,
10792 Assembler::LSL, 0);
10793 %}
10794
10795 ins_pipe(ialu_reg);
10796 %}
10797
10798 // This pattern is automatically generated from aarch64_ad.m4
10799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10800 instruct regI_not_reg(iRegINoSp dst,
10801 iRegIorL2I src1, immI_M1 m1,
10802 rFlagsReg cr) %{
10803 match(Set dst (XorI src1 m1));
10804 ins_cost(INSN_COST);
10805 format %{ "eonw $dst, $src1, zr" %}
10806
10807 ins_encode %{
10808 __ eonw(as_Register($dst$$reg),
10809 as_Register($src1$$reg),
10810 zr,
10811 Assembler::LSL, 0);
10812 %}
10813
10814 ins_pipe(ialu_reg);
10815 %}
10816
10817 // This pattern is automatically generated from aarch64_ad.m4
10818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10819 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10820 immI0 zero, iRegIorL2I src1, immI src2) %{
10821 match(Set dst (SubI zero (URShiftI src1 src2)));
10822
10823 ins_cost(1.9 * INSN_COST);
10824 format %{ "negw $dst, $src1, LSR $src2" %}
10825
10826 ins_encode %{
10827 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10828 Assembler::LSR, $src2$$constant & 0x1f);
10829 %}
10830
10831 ins_pipe(ialu_reg_shift);
10832 %}
10833
10834 // This pattern is automatically generated from aarch64_ad.m4
10835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10836 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10837 immI0 zero, iRegIorL2I src1, immI src2) %{
10838 match(Set dst (SubI zero (RShiftI src1 src2)));
10839
10840 ins_cost(1.9 * INSN_COST);
10841 format %{ "negw $dst, $src1, ASR $src2" %}
10842
10843 ins_encode %{
10844 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10845 Assembler::ASR, $src2$$constant & 0x1f);
10846 %}
10847
10848 ins_pipe(ialu_reg_shift);
10849 %}
10850
10851 // This pattern is automatically generated from aarch64_ad.m4
10852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10853 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10854 immI0 zero, iRegIorL2I src1, immI src2) %{
10855 match(Set dst (SubI zero (LShiftI src1 src2)));
10856
10857 ins_cost(1.9 * INSN_COST);
10858 format %{ "negw $dst, $src1, LSL $src2" %}
10859
10860 ins_encode %{
10861 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10862 Assembler::LSL, $src2$$constant & 0x1f);
10863 %}
10864
10865 ins_pipe(ialu_reg_shift);
10866 %}
10867
10868 // This pattern is automatically generated from aarch64_ad.m4
10869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10870 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10871 immL0 zero, iRegL src1, immI src2) %{
10872 match(Set dst (SubL zero (URShiftL src1 src2)));
10873
10874 ins_cost(1.9 * INSN_COST);
10875 format %{ "neg $dst, $src1, LSR $src2" %}
10876
10877 ins_encode %{
10878 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10879 Assembler::LSR, $src2$$constant & 0x3f);
10880 %}
10881
10882 ins_pipe(ialu_reg_shift);
10883 %}
10884
10885 // This pattern is automatically generated from aarch64_ad.m4
10886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10887 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10888 immL0 zero, iRegL src1, immI src2) %{
10889 match(Set dst (SubL zero (RShiftL src1 src2)));
10890
10891 ins_cost(1.9 * INSN_COST);
10892 format %{ "neg $dst, $src1, ASR $src2" %}
10893
10894 ins_encode %{
10895 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10896 Assembler::ASR, $src2$$constant & 0x3f);
10897 %}
10898
10899 ins_pipe(ialu_reg_shift);
10900 %}
10901
10902 // This pattern is automatically generated from aarch64_ad.m4
10903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10904 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10905 immL0 zero, iRegL src1, immI src2) %{
10906 match(Set dst (SubL zero (LShiftL src1 src2)));
10907
10908 ins_cost(1.9 * INSN_COST);
10909 format %{ "neg $dst, $src1, LSL $src2" %}
10910
10911 ins_encode %{
10912 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10913 Assembler::LSL, $src2$$constant & 0x3f);
10914 %}
10915
10916 ins_pipe(ialu_reg_shift);
10917 %}
10918
10919 // This pattern is automatically generated from aarch64_ad.m4
10920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10921 instruct AndI_reg_not_reg(iRegINoSp dst,
10922 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10923 match(Set dst (AndI src1 (XorI src2 m1)));
10924 ins_cost(INSN_COST);
10925 format %{ "bicw $dst, $src1, $src2" %}
10926
10927 ins_encode %{
10928 __ bicw(as_Register($dst$$reg),
10929 as_Register($src1$$reg),
10930 as_Register($src2$$reg),
10931 Assembler::LSL, 0);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 instruct AndL_reg_not_reg(iRegLNoSp dst,
10940 iRegL src1, iRegL src2, immL_M1 m1) %{
10941 match(Set dst (AndL src1 (XorL src2 m1)));
10942 ins_cost(INSN_COST);
10943 format %{ "bic $dst, $src1, $src2" %}
10944
10945 ins_encode %{
10946 __ bic(as_Register($dst$$reg),
10947 as_Register($src1$$reg),
10948 as_Register($src2$$reg),
10949 Assembler::LSL, 0);
10950 %}
10951
10952 ins_pipe(ialu_reg_reg);
10953 %}
10954
10955 // This pattern is automatically generated from aarch64_ad.m4
10956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10957 instruct OrI_reg_not_reg(iRegINoSp dst,
10958 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10959 match(Set dst (OrI src1 (XorI src2 m1)));
10960 ins_cost(INSN_COST);
10961 format %{ "ornw $dst, $src1, $src2" %}
10962
10963 ins_encode %{
10964 __ ornw(as_Register($dst$$reg),
10965 as_Register($src1$$reg),
10966 as_Register($src2$$reg),
10967 Assembler::LSL, 0);
10968 %}
10969
10970 ins_pipe(ialu_reg_reg);
10971 %}
10972
10973 // This pattern is automatically generated from aarch64_ad.m4
10974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10975 instruct OrL_reg_not_reg(iRegLNoSp dst,
10976 iRegL src1, iRegL src2, immL_M1 m1) %{
10977 match(Set dst (OrL src1 (XorL src2 m1)));
10978 ins_cost(INSN_COST);
10979 format %{ "orn $dst, $src1, $src2" %}
10980
10981 ins_encode %{
10982 __ orn(as_Register($dst$$reg),
10983 as_Register($src1$$reg),
10984 as_Register($src2$$reg),
10985 Assembler::LSL, 0);
10986 %}
10987
10988 ins_pipe(ialu_reg_reg);
10989 %}
10990
10991 // This pattern is automatically generated from aarch64_ad.m4
10992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10993 instruct XorI_reg_not_reg(iRegINoSp dst,
10994 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10995 match(Set dst (XorI m1 (XorI src2 src1)));
10996 ins_cost(INSN_COST);
10997 format %{ "eonw $dst, $src1, $src2" %}
10998
10999 ins_encode %{
11000 __ eonw(as_Register($dst$$reg),
11001 as_Register($src1$$reg),
11002 as_Register($src2$$reg),
11003 Assembler::LSL, 0);
11004 %}
11005
11006 ins_pipe(ialu_reg_reg);
11007 %}
11008
11009 // This pattern is automatically generated from aarch64_ad.m4
11010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11011 instruct XorL_reg_not_reg(iRegLNoSp dst,
11012 iRegL src1, iRegL src2, immL_M1 m1) %{
11013 match(Set dst (XorL m1 (XorL src2 src1)));
11014 ins_cost(INSN_COST);
11015 format %{ "eon $dst, $src1, $src2" %}
11016
11017 ins_encode %{
11018 __ eon(as_Register($dst$$reg),
11019 as_Register($src1$$reg),
11020 as_Register($src2$$reg),
11021 Assembler::LSL, 0);
11022 %}
11023
11024 ins_pipe(ialu_reg_reg);
11025 %}
11026
11027 // This pattern is automatically generated from aarch64_ad.m4
11028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11029 // val & (-1 ^ (val >>> shift)) ==> bicw
11030 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11031 iRegIorL2I src1, iRegIorL2I src2,
11032 immI src3, immI_M1 src4) %{
11033 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11034 ins_cost(1.9 * INSN_COST);
11035 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11036
11037 ins_encode %{
11038 __ bicw(as_Register($dst$$reg),
11039 as_Register($src1$$reg),
11040 as_Register($src2$$reg),
11041 Assembler::LSR,
11042 $src3$$constant & 0x1f);
11043 %}
11044
11045 ins_pipe(ialu_reg_reg_shift);
11046 %}
11047
11048 // This pattern is automatically generated from aarch64_ad.m4
11049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11050 // val & (-1 ^ (val >>> shift)) ==> bic
11051 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11052 iRegL src1, iRegL src2,
11053 immI src3, immL_M1 src4) %{
11054 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11055 ins_cost(1.9 * INSN_COST);
11056 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11057
11058 ins_encode %{
11059 __ bic(as_Register($dst$$reg),
11060 as_Register($src1$$reg),
11061 as_Register($src2$$reg),
11062 Assembler::LSR,
11063 $src3$$constant & 0x3f);
11064 %}
11065
11066 ins_pipe(ialu_reg_reg_shift);
11067 %}
11068
11069 // This pattern is automatically generated from aarch64_ad.m4
11070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11071 // val & (-1 ^ (val >> shift)) ==> bicw
11072 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11073 iRegIorL2I src1, iRegIorL2I src2,
11074 immI src3, immI_M1 src4) %{
11075 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11076 ins_cost(1.9 * INSN_COST);
11077 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11078
11079 ins_encode %{
11080 __ bicw(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg),
11083 Assembler::ASR,
11084 $src3$$constant & 0x1f);
11085 %}
11086
11087 ins_pipe(ialu_reg_reg_shift);
11088 %}
11089
11090 // This pattern is automatically generated from aarch64_ad.m4
11091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11092 // val & (-1 ^ (val >> shift)) ==> bic
11093 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11094 iRegL src1, iRegL src2,
11095 immI src3, immL_M1 src4) %{
11096 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11097 ins_cost(1.9 * INSN_COST);
11098 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11099
11100 ins_encode %{
11101 __ bic(as_Register($dst$$reg),
11102 as_Register($src1$$reg),
11103 as_Register($src2$$reg),
11104 Assembler::ASR,
11105 $src3$$constant & 0x3f);
11106 %}
11107
11108 ins_pipe(ialu_reg_reg_shift);
11109 %}
11110
11111 // This pattern is automatically generated from aarch64_ad.m4
11112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11113 // val & (-1 ^ (val ror shift)) ==> bicw
11114 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11115 iRegIorL2I src1, iRegIorL2I src2,
11116 immI src3, immI_M1 src4) %{
11117 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11118 ins_cost(1.9 * INSN_COST);
11119 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11120
11121 ins_encode %{
11122 __ bicw(as_Register($dst$$reg),
11123 as_Register($src1$$reg),
11124 as_Register($src2$$reg),
11125 Assembler::ROR,
11126 $src3$$constant & 0x1f);
11127 %}
11128
11129 ins_pipe(ialu_reg_reg_shift);
11130 %}
11131
11132 // This pattern is automatically generated from aarch64_ad.m4
11133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11134 // val & (-1 ^ (val ror shift)) ==> bic
11135 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11136 iRegL src1, iRegL src2,
11137 immI src3, immL_M1 src4) %{
11138 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11139 ins_cost(1.9 * INSN_COST);
11140 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11141
11142 ins_encode %{
11143 __ bic(as_Register($dst$$reg),
11144 as_Register($src1$$reg),
11145 as_Register($src2$$reg),
11146 Assembler::ROR,
11147 $src3$$constant & 0x3f);
11148 %}
11149
11150 ins_pipe(ialu_reg_reg_shift);
11151 %}
11152
11153 // This pattern is automatically generated from aarch64_ad.m4
11154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11155 // val & (-1 ^ (val << shift)) ==> bicw
11156 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11157 iRegIorL2I src1, iRegIorL2I src2,
11158 immI src3, immI_M1 src4) %{
11159 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11160 ins_cost(1.9 * INSN_COST);
11161 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11162
11163 ins_encode %{
11164 __ bicw(as_Register($dst$$reg),
11165 as_Register($src1$$reg),
11166 as_Register($src2$$reg),
11167 Assembler::LSL,
11168 $src3$$constant & 0x1f);
11169 %}
11170
11171 ins_pipe(ialu_reg_reg_shift);
11172 %}
11173
11174 // This pattern is automatically generated from aarch64_ad.m4
11175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11176 // val & (-1 ^ (val << shift)) ==> bic
11177 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11178 iRegL src1, iRegL src2,
11179 immI src3, immL_M1 src4) %{
11180 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11181 ins_cost(1.9 * INSN_COST);
11182 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11183
11184 ins_encode %{
11185 __ bic(as_Register($dst$$reg),
11186 as_Register($src1$$reg),
11187 as_Register($src2$$reg),
11188 Assembler::LSL,
11189 $src3$$constant & 0x3f);
11190 %}
11191
11192 ins_pipe(ialu_reg_reg_shift);
11193 %}
11194
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11198 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11199 iRegIorL2I src1, iRegIorL2I src2,
11200 immI src3, immI_M1 src4) %{
11201 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11202 ins_cost(1.9 * INSN_COST);
11203 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11204
11205 ins_encode %{
11206 __ eonw(as_Register($dst$$reg),
11207 as_Register($src1$$reg),
11208 as_Register($src2$$reg),
11209 Assembler::LSR,
11210 $src3$$constant & 0x1f);
11211 %}
11212
11213 ins_pipe(ialu_reg_reg_shift);
11214 %}
11215
11216 // This pattern is automatically generated from aarch64_ad.m4
11217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11218 // val ^ (-1 ^ (val >>> shift)) ==> eon
11219 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11220 iRegL src1, iRegL src2,
11221 immI src3, immL_M1 src4) %{
11222 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11223 ins_cost(1.9 * INSN_COST);
11224 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11225
11226 ins_encode %{
11227 __ eon(as_Register($dst$$reg),
11228 as_Register($src1$$reg),
11229 as_Register($src2$$reg),
11230 Assembler::LSR,
11231 $src3$$constant & 0x3f);
11232 %}
11233
11234 ins_pipe(ialu_reg_reg_shift);
11235 %}
11236
11237 // This pattern is automatically generated from aarch64_ad.m4
11238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11239 // val ^ (-1 ^ (val >> shift)) ==> eonw
11240 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11241 iRegIorL2I src1, iRegIorL2I src2,
11242 immI src3, immI_M1 src4) %{
11243 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11244 ins_cost(1.9 * INSN_COST);
11245 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11246
11247 ins_encode %{
11248 __ eonw(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 as_Register($src2$$reg),
11251 Assembler::ASR,
11252 $src3$$constant & 0x1f);
11253 %}
11254
11255 ins_pipe(ialu_reg_reg_shift);
11256 %}
11257
11258 // This pattern is automatically generated from aarch64_ad.m4
11259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11260 // val ^ (-1 ^ (val >> shift)) ==> eon
11261 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11262 iRegL src1, iRegL src2,
11263 immI src3, immL_M1 src4) %{
11264 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11265 ins_cost(1.9 * INSN_COST);
11266 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11267
11268 ins_encode %{
11269 __ eon(as_Register($dst$$reg),
11270 as_Register($src1$$reg),
11271 as_Register($src2$$reg),
11272 Assembler::ASR,
11273 $src3$$constant & 0x3f);
11274 %}
11275
11276 ins_pipe(ialu_reg_reg_shift);
11277 %}
11278
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 // val ^ (-1 ^ (val ror shift)) ==> eonw
11282 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11283 iRegIorL2I src1, iRegIorL2I src2,
11284 immI src3, immI_M1 src4) %{
11285 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11286 ins_cost(1.9 * INSN_COST);
11287 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11288
11289 ins_encode %{
11290 __ eonw(as_Register($dst$$reg),
11291 as_Register($src1$$reg),
11292 as_Register($src2$$reg),
11293 Assembler::ROR,
11294 $src3$$constant & 0x1f);
11295 %}
11296
11297 ins_pipe(ialu_reg_reg_shift);
11298 %}
11299
11300 // This pattern is automatically generated from aarch64_ad.m4
11301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11302 // val ^ (-1 ^ (val ror shift)) ==> eon
11303 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11304 iRegL src1, iRegL src2,
11305 immI src3, immL_M1 src4) %{
11306 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11307 ins_cost(1.9 * INSN_COST);
11308 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11309
11310 ins_encode %{
11311 __ eon(as_Register($dst$$reg),
11312 as_Register($src1$$reg),
11313 as_Register($src2$$reg),
11314 Assembler::ROR,
11315 $src3$$constant & 0x3f);
11316 %}
11317
11318 ins_pipe(ialu_reg_reg_shift);
11319 %}
11320
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 // val ^ (-1 ^ (val << shift)) ==> eonw
11324 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11325 iRegIorL2I src1, iRegIorL2I src2,
11326 immI src3, immI_M1 src4) %{
11327 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11328 ins_cost(1.9 * INSN_COST);
11329 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11330
11331 ins_encode %{
11332 __ eonw(as_Register($dst$$reg),
11333 as_Register($src1$$reg),
11334 as_Register($src2$$reg),
11335 Assembler::LSL,
11336 $src3$$constant & 0x1f);
11337 %}
11338
11339 ins_pipe(ialu_reg_reg_shift);
11340 %}
11341
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 // val ^ (-1 ^ (val << shift)) ==> eon
11345 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11346 iRegL src1, iRegL src2,
11347 immI src3, immL_M1 src4) %{
11348 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11349 ins_cost(1.9 * INSN_COST);
11350 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11351
11352 ins_encode %{
11353 __ eon(as_Register($dst$$reg),
11354 as_Register($src1$$reg),
11355 as_Register($src2$$reg),
11356 Assembler::LSL,
11357 $src3$$constant & 0x3f);
11358 %}
11359
11360 ins_pipe(ialu_reg_reg_shift);
11361 %}
11362
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 // val | (-1 ^ (val >>> shift)) ==> ornw
11366 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11367 iRegIorL2I src1, iRegIorL2I src2,
11368 immI src3, immI_M1 src4) %{
11369 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11370 ins_cost(1.9 * INSN_COST);
11371 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11372
11373 ins_encode %{
11374 __ ornw(as_Register($dst$$reg),
11375 as_Register($src1$$reg),
11376 as_Register($src2$$reg),
11377 Assembler::LSR,
11378 $src3$$constant & 0x1f);
11379 %}
11380
11381 ins_pipe(ialu_reg_reg_shift);
11382 %}
11383
11384 // This pattern is automatically generated from aarch64_ad.m4
11385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11386 // val | (-1 ^ (val >>> shift)) ==> orn
11387 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11388 iRegL src1, iRegL src2,
11389 immI src3, immL_M1 src4) %{
11390 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11391 ins_cost(1.9 * INSN_COST);
11392 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11393
11394 ins_encode %{
11395 __ orn(as_Register($dst$$reg),
11396 as_Register($src1$$reg),
11397 as_Register($src2$$reg),
11398 Assembler::LSR,
11399 $src3$$constant & 0x3f);
11400 %}
11401
11402 ins_pipe(ialu_reg_reg_shift);
11403 %}
11404
11405 // This pattern is automatically generated from aarch64_ad.m4
11406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11407 // val | (-1 ^ (val >> shift)) ==> ornw
11408 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11409 iRegIorL2I src1, iRegIorL2I src2,
11410 immI src3, immI_M1 src4) %{
11411 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11412 ins_cost(1.9 * INSN_COST);
11413 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11414
11415 ins_encode %{
11416 __ ornw(as_Register($dst$$reg),
11417 as_Register($src1$$reg),
11418 as_Register($src2$$reg),
11419 Assembler::ASR,
11420 $src3$$constant & 0x1f);
11421 %}
11422
11423 ins_pipe(ialu_reg_reg_shift);
11424 %}
11425
11426 // This pattern is automatically generated from aarch64_ad.m4
11427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11428 // val | (-1 ^ (val >> shift)) ==> orn
11429 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11430 iRegL src1, iRegL src2,
11431 immI src3, immL_M1 src4) %{
11432 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11433 ins_cost(1.9 * INSN_COST);
11434 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11435
11436 ins_encode %{
11437 __ orn(as_Register($dst$$reg),
11438 as_Register($src1$$reg),
11439 as_Register($src2$$reg),
11440 Assembler::ASR,
11441 $src3$$constant & 0x3f);
11442 %}
11443
11444 ins_pipe(ialu_reg_reg_shift);
11445 %}
11446
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449 // val | (-1 ^ (val ror shift)) ==> ornw
11450 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11451 iRegIorL2I src1, iRegIorL2I src2,
11452 immI src3, immI_M1 src4) %{
11453 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11454 ins_cost(1.9 * INSN_COST);
11455 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11456
11457 ins_encode %{
11458 __ ornw(as_Register($dst$$reg),
11459 as_Register($src1$$reg),
11460 as_Register($src2$$reg),
11461 Assembler::ROR,
11462 $src3$$constant & 0x1f);
11463 %}
11464
11465 ins_pipe(ialu_reg_reg_shift);
11466 %}
11467
11468 // This pattern is automatically generated from aarch64_ad.m4
11469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11470 // val | (-1 ^ (val ror shift)) ==> orn
11471 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11472 iRegL src1, iRegL src2,
11473 immI src3, immL_M1 src4) %{
11474 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11475 ins_cost(1.9 * INSN_COST);
11476 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11477
11478 ins_encode %{
11479 __ orn(as_Register($dst$$reg),
11480 as_Register($src1$$reg),
11481 as_Register($src2$$reg),
11482 Assembler::ROR,
11483 $src3$$constant & 0x3f);
11484 %}
11485
11486 ins_pipe(ialu_reg_reg_shift);
11487 %}
11488
11489 // This pattern is automatically generated from aarch64_ad.m4
11490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11491 // val | (-1 ^ (val << shift)) ==> ornw
11492 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11493 iRegIorL2I src1, iRegIorL2I src2,
11494 immI src3, immI_M1 src4) %{
11495 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11496 ins_cost(1.9 * INSN_COST);
11497 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11498
11499 ins_encode %{
11500 __ ornw(as_Register($dst$$reg),
11501 as_Register($src1$$reg),
11502 as_Register($src2$$reg),
11503 Assembler::LSL,
11504 $src3$$constant & 0x1f);
11505 %}
11506
11507 ins_pipe(ialu_reg_reg_shift);
11508 %}
11509
11510 // This pattern is automatically generated from aarch64_ad.m4
11511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11512 // val | (-1 ^ (val << shift)) ==> orn
11513 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11514 iRegL src1, iRegL src2,
11515 immI src3, immL_M1 src4) %{
11516 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11517 ins_cost(1.9 * INSN_COST);
11518 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11519
11520 ins_encode %{
11521 __ orn(as_Register($dst$$reg),
11522 as_Register($src1$$reg),
11523 as_Register($src2$$reg),
11524 Assembler::LSL,
11525 $src3$$constant & 0x3f);
11526 %}
11527
11528 ins_pipe(ialu_reg_reg_shift);
11529 %}
11530
11531 // This pattern is automatically generated from aarch64_ad.m4
11532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11533 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11534 iRegIorL2I src1, iRegIorL2I src2,
11535 immI src3) %{
11536 match(Set dst (AndI src1 (URShiftI src2 src3)));
11537
11538 ins_cost(1.9 * INSN_COST);
11539 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11540
11541 ins_encode %{
11542 __ andw(as_Register($dst$$reg),
11543 as_Register($src1$$reg),
11544 as_Register($src2$$reg),
11545 Assembler::LSR,
11546 $src3$$constant & 0x1f);
11547 %}
11548
11549 ins_pipe(ialu_reg_reg_shift);
11550 %}
11551
11552 // This pattern is automatically generated from aarch64_ad.m4
11553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11554 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11555 iRegL src1, iRegL src2,
11556 immI src3) %{
11557 match(Set dst (AndL src1 (URShiftL src2 src3)));
11558
11559 ins_cost(1.9 * INSN_COST);
11560 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11561
11562 ins_encode %{
11563 __ andr(as_Register($dst$$reg),
11564 as_Register($src1$$reg),
11565 as_Register($src2$$reg),
11566 Assembler::LSR,
11567 $src3$$constant & 0x3f);
11568 %}
11569
11570 ins_pipe(ialu_reg_reg_shift);
11571 %}
11572
11573 // This pattern is automatically generated from aarch64_ad.m4
11574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11575 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11576 iRegIorL2I src1, iRegIorL2I src2,
11577 immI src3) %{
11578 match(Set dst (AndI src1 (RShiftI src2 src3)));
11579
11580 ins_cost(1.9 * INSN_COST);
11581 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11582
11583 ins_encode %{
11584 __ andw(as_Register($dst$$reg),
11585 as_Register($src1$$reg),
11586 as_Register($src2$$reg),
11587 Assembler::ASR,
11588 $src3$$constant & 0x1f);
11589 %}
11590
11591 ins_pipe(ialu_reg_reg_shift);
11592 %}
11593
11594 // This pattern is automatically generated from aarch64_ad.m4
11595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11596 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11597 iRegL src1, iRegL src2,
11598 immI src3) %{
11599 match(Set dst (AndL src1 (RShiftL src2 src3)));
11600
11601 ins_cost(1.9 * INSN_COST);
11602 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11603
11604 ins_encode %{
11605 __ andr(as_Register($dst$$reg),
11606 as_Register($src1$$reg),
11607 as_Register($src2$$reg),
11608 Assembler::ASR,
11609 $src3$$constant & 0x3f);
11610 %}
11611
11612 ins_pipe(ialu_reg_reg_shift);
11613 %}
11614
11615 // This pattern is automatically generated from aarch64_ad.m4
11616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11617 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11618 iRegIorL2I src1, iRegIorL2I src2,
11619 immI src3) %{
11620 match(Set dst (AndI src1 (LShiftI src2 src3)));
11621
11622 ins_cost(1.9 * INSN_COST);
11623 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11624
11625 ins_encode %{
11626 __ andw(as_Register($dst$$reg),
11627 as_Register($src1$$reg),
11628 as_Register($src2$$reg),
11629 Assembler::LSL,
11630 $src3$$constant & 0x1f);
11631 %}
11632
11633 ins_pipe(ialu_reg_reg_shift);
11634 %}
11635
11636 // This pattern is automatically generated from aarch64_ad.m4
11637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11638 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11639 iRegL src1, iRegL src2,
11640 immI src3) %{
11641 match(Set dst (AndL src1 (LShiftL src2 src3)));
11642
11643 ins_cost(1.9 * INSN_COST);
11644 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11645
11646 ins_encode %{
11647 __ andr(as_Register($dst$$reg),
11648 as_Register($src1$$reg),
11649 as_Register($src2$$reg),
11650 Assembler::LSL,
11651 $src3$$constant & 0x3f);
11652 %}
11653
11654 ins_pipe(ialu_reg_reg_shift);
11655 %}
11656
11657 // This pattern is automatically generated from aarch64_ad.m4
11658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11659 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11660 iRegIorL2I src1, iRegIorL2I src2,
11661 immI src3) %{
11662 match(Set dst (AndI src1 (RotateRight src2 src3)));
11663
11664 ins_cost(1.9 * INSN_COST);
11665 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11666
11667 ins_encode %{
11668 __ andw(as_Register($dst$$reg),
11669 as_Register($src1$$reg),
11670 as_Register($src2$$reg),
11671 Assembler::ROR,
11672 $src3$$constant & 0x1f);
11673 %}
11674
11675 ins_pipe(ialu_reg_reg_shift);
11676 %}
11677
11678 // This pattern is automatically generated from aarch64_ad.m4
11679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11680 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11681 iRegL src1, iRegL src2,
11682 immI src3) %{
11683 match(Set dst (AndL src1 (RotateRight src2 src3)));
11684
11685 ins_cost(1.9 * INSN_COST);
11686 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11687
11688 ins_encode %{
11689 __ andr(as_Register($dst$$reg),
11690 as_Register($src1$$reg),
11691 as_Register($src2$$reg),
11692 Assembler::ROR,
11693 $src3$$constant & 0x3f);
11694 %}
11695
11696 ins_pipe(ialu_reg_reg_shift);
11697 %}
11698
11699 // This pattern is automatically generated from aarch64_ad.m4
11700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11701 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11702 iRegIorL2I src1, iRegIorL2I src2,
11703 immI src3) %{
11704 match(Set dst (XorI src1 (URShiftI src2 src3)));
11705
11706 ins_cost(1.9 * INSN_COST);
11707 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11708
11709 ins_encode %{
11710 __ eorw(as_Register($dst$$reg),
11711 as_Register($src1$$reg),
11712 as_Register($src2$$reg),
11713 Assembler::LSR,
11714 $src3$$constant & 0x1f);
11715 %}
11716
11717 ins_pipe(ialu_reg_reg_shift);
11718 %}
11719
11720 // This pattern is automatically generated from aarch64_ad.m4
11721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11722 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11723 iRegL src1, iRegL src2,
11724 immI src3) %{
11725 match(Set dst (XorL src1 (URShiftL src2 src3)));
11726
11727 ins_cost(1.9 * INSN_COST);
11728 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11729
11730 ins_encode %{
11731 __ eor(as_Register($dst$$reg),
11732 as_Register($src1$$reg),
11733 as_Register($src2$$reg),
11734 Assembler::LSR,
11735 $src3$$constant & 0x3f);
11736 %}
11737
11738 ins_pipe(ialu_reg_reg_shift);
11739 %}
11740
11741 // This pattern is automatically generated from aarch64_ad.m4
11742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11743 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11744 iRegIorL2I src1, iRegIorL2I src2,
11745 immI src3) %{
11746 match(Set dst (XorI src1 (RShiftI src2 src3)));
11747
11748 ins_cost(1.9 * INSN_COST);
11749 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11750
11751 ins_encode %{
11752 __ eorw(as_Register($dst$$reg),
11753 as_Register($src1$$reg),
11754 as_Register($src2$$reg),
11755 Assembler::ASR,
11756 $src3$$constant & 0x1f);
11757 %}
11758
11759 ins_pipe(ialu_reg_reg_shift);
11760 %}
11761
11762 // This pattern is automatically generated from aarch64_ad.m4
11763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11764 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11765 iRegL src1, iRegL src2,
11766 immI src3) %{
11767 match(Set dst (XorL src1 (RShiftL src2 src3)));
11768
11769 ins_cost(1.9 * INSN_COST);
11770 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11771
11772 ins_encode %{
11773 __ eor(as_Register($dst$$reg),
11774 as_Register($src1$$reg),
11775 as_Register($src2$$reg),
11776 Assembler::ASR,
11777 $src3$$constant & 0x3f);
11778 %}
11779
11780 ins_pipe(ialu_reg_reg_shift);
11781 %}
11782
11783 // This pattern is automatically generated from aarch64_ad.m4
11784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11785 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11786 iRegIorL2I src1, iRegIorL2I src2,
11787 immI src3) %{
11788 match(Set dst (XorI src1 (LShiftI src2 src3)));
11789
11790 ins_cost(1.9 * INSN_COST);
11791 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11792
11793 ins_encode %{
11794 __ eorw(as_Register($dst$$reg),
11795 as_Register($src1$$reg),
11796 as_Register($src2$$reg),
11797 Assembler::LSL,
11798 $src3$$constant & 0x1f);
11799 %}
11800
11801 ins_pipe(ialu_reg_reg_shift);
11802 %}
11803
11804 // This pattern is automatically generated from aarch64_ad.m4
11805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11806 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11807 iRegL src1, iRegL src2,
11808 immI src3) %{
11809 match(Set dst (XorL src1 (LShiftL src2 src3)));
11810
11811 ins_cost(1.9 * INSN_COST);
11812 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11813
11814 ins_encode %{
11815 __ eor(as_Register($dst$$reg),
11816 as_Register($src1$$reg),
11817 as_Register($src2$$reg),
11818 Assembler::LSL,
11819 $src3$$constant & 0x3f);
11820 %}
11821
11822 ins_pipe(ialu_reg_reg_shift);
11823 %}
11824
11825 // This pattern is automatically generated from aarch64_ad.m4
11826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11827 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11828 iRegIorL2I src1, iRegIorL2I src2,
11829 immI src3) %{
11830 match(Set dst (XorI src1 (RotateRight src2 src3)));
11831
11832 ins_cost(1.9 * INSN_COST);
11833 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11834
11835 ins_encode %{
11836 __ eorw(as_Register($dst$$reg),
11837 as_Register($src1$$reg),
11838 as_Register($src2$$reg),
11839 Assembler::ROR,
11840 $src3$$constant & 0x1f);
11841 %}
11842
11843 ins_pipe(ialu_reg_reg_shift);
11844 %}
11845
11846 // This pattern is automatically generated from aarch64_ad.m4
11847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11848 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11849 iRegL src1, iRegL src2,
11850 immI src3) %{
11851 match(Set dst (XorL src1 (RotateRight src2 src3)));
11852
11853 ins_cost(1.9 * INSN_COST);
11854 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11855
11856 ins_encode %{
11857 __ eor(as_Register($dst$$reg),
11858 as_Register($src1$$reg),
11859 as_Register($src2$$reg),
11860 Assembler::ROR,
11861 $src3$$constant & 0x3f);
11862 %}
11863
11864 ins_pipe(ialu_reg_reg_shift);
11865 %}
11866
11867 // This pattern is automatically generated from aarch64_ad.m4
11868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11869 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11870 iRegIorL2I src1, iRegIorL2I src2,
11871 immI src3) %{
11872 match(Set dst (OrI src1 (URShiftI src2 src3)));
11873
11874 ins_cost(1.9 * INSN_COST);
11875 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11876
11877 ins_encode %{
11878 __ orrw(as_Register($dst$$reg),
11879 as_Register($src1$$reg),
11880 as_Register($src2$$reg),
11881 Assembler::LSR,
11882 $src3$$constant & 0x1f);
11883 %}
11884
11885 ins_pipe(ialu_reg_reg_shift);
11886 %}
11887
11888 // This pattern is automatically generated from aarch64_ad.m4
11889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11890 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11891 iRegL src1, iRegL src2,
11892 immI src3) %{
11893 match(Set dst (OrL src1 (URShiftL src2 src3)));
11894
11895 ins_cost(1.9 * INSN_COST);
11896 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11897
11898 ins_encode %{
11899 __ orr(as_Register($dst$$reg),
11900 as_Register($src1$$reg),
11901 as_Register($src2$$reg),
11902 Assembler::LSR,
11903 $src3$$constant & 0x3f);
11904 %}
11905
11906 ins_pipe(ialu_reg_reg_shift);
11907 %}
11908
11909 // This pattern is automatically generated from aarch64_ad.m4
11910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11911 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11912 iRegIorL2I src1, iRegIorL2I src2,
11913 immI src3) %{
11914 match(Set dst (OrI src1 (RShiftI src2 src3)));
11915
11916 ins_cost(1.9 * INSN_COST);
11917 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11918
11919 ins_encode %{
11920 __ orrw(as_Register($dst$$reg),
11921 as_Register($src1$$reg),
11922 as_Register($src2$$reg),
11923 Assembler::ASR,
11924 $src3$$constant & 0x1f);
11925 %}
11926
11927 ins_pipe(ialu_reg_reg_shift);
11928 %}
11929
11930 // This pattern is automatically generated from aarch64_ad.m4
11931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11932 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11933 iRegL src1, iRegL src2,
11934 immI src3) %{
11935 match(Set dst (OrL src1 (RShiftL src2 src3)));
11936
11937 ins_cost(1.9 * INSN_COST);
11938 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11939
11940 ins_encode %{
11941 __ orr(as_Register($dst$$reg),
11942 as_Register($src1$$reg),
11943 as_Register($src2$$reg),
11944 Assembler::ASR,
11945 $src3$$constant & 0x3f);
11946 %}
11947
11948 ins_pipe(ialu_reg_reg_shift);
11949 %}
11950
11951 // This pattern is automatically generated from aarch64_ad.m4
11952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11953 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11954 iRegIorL2I src1, iRegIorL2I src2,
11955 immI src3) %{
11956 match(Set dst (OrI src1 (LShiftI src2 src3)));
11957
11958 ins_cost(1.9 * INSN_COST);
11959 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11960
11961 ins_encode %{
11962 __ orrw(as_Register($dst$$reg),
11963 as_Register($src1$$reg),
11964 as_Register($src2$$reg),
11965 Assembler::LSL,
11966 $src3$$constant & 0x1f);
11967 %}
11968
11969 ins_pipe(ialu_reg_reg_shift);
11970 %}
11971
11972 // This pattern is automatically generated from aarch64_ad.m4
11973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11974 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11975 iRegL src1, iRegL src2,
11976 immI src3) %{
11977 match(Set dst (OrL src1 (LShiftL src2 src3)));
11978
11979 ins_cost(1.9 * INSN_COST);
11980 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11981
11982 ins_encode %{
11983 __ orr(as_Register($dst$$reg),
11984 as_Register($src1$$reg),
11985 as_Register($src2$$reg),
11986 Assembler::LSL,
11987 $src3$$constant & 0x3f);
11988 %}
11989
11990 ins_pipe(ialu_reg_reg_shift);
11991 %}
11992
11993 // This pattern is automatically generated from aarch64_ad.m4
11994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11995 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11996 iRegIorL2I src1, iRegIorL2I src2,
11997 immI src3) %{
11998 match(Set dst (OrI src1 (RotateRight src2 src3)));
11999
12000 ins_cost(1.9 * INSN_COST);
12001 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12002
12003 ins_encode %{
12004 __ orrw(as_Register($dst$$reg),
12005 as_Register($src1$$reg),
12006 as_Register($src2$$reg),
12007 Assembler::ROR,
12008 $src3$$constant & 0x1f);
12009 %}
12010
12011 ins_pipe(ialu_reg_reg_shift);
12012 %}
12013
12014 // This pattern is automatically generated from aarch64_ad.m4
12015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12016 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12017 iRegL src1, iRegL src2,
12018 immI src3) %{
12019 match(Set dst (OrL src1 (RotateRight src2 src3)));
12020
12021 ins_cost(1.9 * INSN_COST);
12022 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12023
12024 ins_encode %{
12025 __ orr(as_Register($dst$$reg),
12026 as_Register($src1$$reg),
12027 as_Register($src2$$reg),
12028 Assembler::ROR,
12029 $src3$$constant & 0x3f);
12030 %}
12031
12032 ins_pipe(ialu_reg_reg_shift);
12033 %}
12034
12035 // This pattern is automatically generated from aarch64_ad.m4
12036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12037 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12038 iRegIorL2I src1, iRegIorL2I src2,
12039 immI src3) %{
12040 match(Set dst (AddI src1 (URShiftI src2 src3)));
12041
12042 ins_cost(1.9 * INSN_COST);
12043 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12044
12045 ins_encode %{
12046 __ addw(as_Register($dst$$reg),
12047 as_Register($src1$$reg),
12048 as_Register($src2$$reg),
12049 Assembler::LSR,
12050 $src3$$constant & 0x1f);
12051 %}
12052
12053 ins_pipe(ialu_reg_reg_shift);
12054 %}
12055
12056 // This pattern is automatically generated from aarch64_ad.m4
12057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12058 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12059 iRegL src1, iRegL src2,
12060 immI src3) %{
12061 match(Set dst (AddL src1 (URShiftL src2 src3)));
12062
12063 ins_cost(1.9 * INSN_COST);
12064 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12065
12066 ins_encode %{
12067 __ add(as_Register($dst$$reg),
12068 as_Register($src1$$reg),
12069 as_Register($src2$$reg),
12070 Assembler::LSR,
12071 $src3$$constant & 0x3f);
12072 %}
12073
12074 ins_pipe(ialu_reg_reg_shift);
12075 %}
12076
12077 // This pattern is automatically generated from aarch64_ad.m4
12078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12079 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12080 iRegIorL2I src1, iRegIorL2I src2,
12081 immI src3) %{
12082 match(Set dst (AddI src1 (RShiftI src2 src3)));
12083
12084 ins_cost(1.9 * INSN_COST);
12085 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12086
12087 ins_encode %{
12088 __ addw(as_Register($dst$$reg),
12089 as_Register($src1$$reg),
12090 as_Register($src2$$reg),
12091 Assembler::ASR,
12092 $src3$$constant & 0x1f);
12093 %}
12094
12095 ins_pipe(ialu_reg_reg_shift);
12096 %}
12097
12098 // This pattern is automatically generated from aarch64_ad.m4
12099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12100 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12101 iRegL src1, iRegL src2,
12102 immI src3) %{
12103 match(Set dst (AddL src1 (RShiftL src2 src3)));
12104
12105 ins_cost(1.9 * INSN_COST);
12106 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12107
12108 ins_encode %{
12109 __ add(as_Register($dst$$reg),
12110 as_Register($src1$$reg),
12111 as_Register($src2$$reg),
12112 Assembler::ASR,
12113 $src3$$constant & 0x3f);
12114 %}
12115
12116 ins_pipe(ialu_reg_reg_shift);
12117 %}
12118
12119 // This pattern is automatically generated from aarch64_ad.m4
12120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12121 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12122 iRegIorL2I src1, iRegIorL2I src2,
12123 immI src3) %{
12124 match(Set dst (AddI src1 (LShiftI src2 src3)));
12125
12126 ins_cost(1.9 * INSN_COST);
12127 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12128
12129 ins_encode %{
12130 __ addw(as_Register($dst$$reg),
12131 as_Register($src1$$reg),
12132 as_Register($src2$$reg),
12133 Assembler::LSL,
12134 $src3$$constant & 0x1f);
12135 %}
12136
12137 ins_pipe(ialu_reg_reg_shift);
12138 %}
12139
12140 // This pattern is automatically generated from aarch64_ad.m4
12141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12142 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12143 iRegL src1, iRegL src2,
12144 immI src3) %{
12145 match(Set dst (AddL src1 (LShiftL src2 src3)));
12146
12147 ins_cost(1.9 * INSN_COST);
12148 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12149
12150 ins_encode %{
12151 __ add(as_Register($dst$$reg),
12152 as_Register($src1$$reg),
12153 as_Register($src2$$reg),
12154 Assembler::LSL,
12155 $src3$$constant & 0x3f);
12156 %}
12157
12158 ins_pipe(ialu_reg_reg_shift);
12159 %}
12160
12161 // This pattern is automatically generated from aarch64_ad.m4
12162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12163 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12164 iRegIorL2I src1, iRegIorL2I src2,
12165 immI src3) %{
12166 match(Set dst (SubI src1 (URShiftI src2 src3)));
12167
12168 ins_cost(1.9 * INSN_COST);
12169 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12170
12171 ins_encode %{
12172 __ subw(as_Register($dst$$reg),
12173 as_Register($src1$$reg),
12174 as_Register($src2$$reg),
12175 Assembler::LSR,
12176 $src3$$constant & 0x1f);
12177 %}
12178
12179 ins_pipe(ialu_reg_reg_shift);
12180 %}
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12185 iRegL src1, iRegL src2,
12186 immI src3) %{
12187 match(Set dst (SubL src1 (URShiftL src2 src3)));
12188
12189 ins_cost(1.9 * INSN_COST);
12190 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12191
12192 ins_encode %{
12193 __ sub(as_Register($dst$$reg),
12194 as_Register($src1$$reg),
12195 as_Register($src2$$reg),
12196 Assembler::LSR,
12197 $src3$$constant & 0x3f);
12198 %}
12199
12200 ins_pipe(ialu_reg_reg_shift);
12201 %}
12202
12203 // This pattern is automatically generated from aarch64_ad.m4
12204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12205 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12206 iRegIorL2I src1, iRegIorL2I src2,
12207 immI src3) %{
12208 match(Set dst (SubI src1 (RShiftI src2 src3)));
12209
12210 ins_cost(1.9 * INSN_COST);
12211 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12212
12213 ins_encode %{
12214 __ subw(as_Register($dst$$reg),
12215 as_Register($src1$$reg),
12216 as_Register($src2$$reg),
12217 Assembler::ASR,
12218 $src3$$constant & 0x1f);
12219 %}
12220
12221 ins_pipe(ialu_reg_reg_shift);
12222 %}
12223
12224 // This pattern is automatically generated from aarch64_ad.m4
12225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12226 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12227 iRegL src1, iRegL src2,
12228 immI src3) %{
12229 match(Set dst (SubL src1 (RShiftL src2 src3)));
12230
12231 ins_cost(1.9 * INSN_COST);
12232 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12233
12234 ins_encode %{
12235 __ sub(as_Register($dst$$reg),
12236 as_Register($src1$$reg),
12237 as_Register($src2$$reg),
12238 Assembler::ASR,
12239 $src3$$constant & 0x3f);
12240 %}
12241
12242 ins_pipe(ialu_reg_reg_shift);
12243 %}
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12248 iRegIorL2I src1, iRegIorL2I src2,
12249 immI src3) %{
12250 match(Set dst (SubI src1 (LShiftI src2 src3)));
12251
12252 ins_cost(1.9 * INSN_COST);
12253 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12254
12255 ins_encode %{
12256 __ subw(as_Register($dst$$reg),
12257 as_Register($src1$$reg),
12258 as_Register($src2$$reg),
12259 Assembler::LSL,
12260 $src3$$constant & 0x1f);
12261 %}
12262
12263 ins_pipe(ialu_reg_reg_shift);
12264 %}
12265
12266 // This pattern is automatically generated from aarch64_ad.m4
12267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12268 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12269 iRegL src1, iRegL src2,
12270 immI src3) %{
12271 match(Set dst (SubL src1 (LShiftL src2 src3)));
12272
12273 ins_cost(1.9 * INSN_COST);
12274 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12275
12276 ins_encode %{
12277 __ sub(as_Register($dst$$reg),
12278 as_Register($src1$$reg),
12279 as_Register($src2$$reg),
12280 Assembler::LSL,
12281 $src3$$constant & 0x3f);
12282 %}
12283
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289
12290 // Shift Left followed by Shift Right.
12291 // This idiom is used by the compiler for the i2b bytecode etc.
12292 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12293 %{
12294 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12295 ins_cost(INSN_COST * 2);
12296 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12297 ins_encode %{
12298 int lshift = $lshift_count$$constant & 63;
12299 int rshift = $rshift_count$$constant & 63;
12300 int s = 63 - lshift;
12301 int r = (rshift - lshift) & 63;
12302 __ sbfm(as_Register($dst$$reg),
12303 as_Register($src$$reg),
12304 r, s);
12305 %}
12306
12307 ins_pipe(ialu_reg_shift);
12308 %}
12309
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312
12313 // Shift Left followed by Shift Right.
12314 // This idiom is used by the compiler for the i2b bytecode etc.
12315 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12316 %{
12317 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12318 ins_cost(INSN_COST * 2);
12319 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12320 ins_encode %{
12321 int lshift = $lshift_count$$constant & 31;
12322 int rshift = $rshift_count$$constant & 31;
12323 int s = 31 - lshift;
12324 int r = (rshift - lshift) & 31;
12325 __ sbfmw(as_Register($dst$$reg),
12326 as_Register($src$$reg),
12327 r, s);
12328 %}
12329
12330 ins_pipe(ialu_reg_shift);
12331 %}
12332
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335
12336 // Shift Left followed by Shift Right.
12337 // This idiom is used by the compiler for the i2b bytecode etc.
12338 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12339 %{
12340 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12341 ins_cost(INSN_COST * 2);
12342 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12343 ins_encode %{
12344 int lshift = $lshift_count$$constant & 63;
12345 int rshift = $rshift_count$$constant & 63;
12346 int s = 63 - lshift;
12347 int r = (rshift - lshift) & 63;
12348 __ ubfm(as_Register($dst$$reg),
12349 as_Register($src$$reg),
12350 r, s);
12351 %}
12352
12353 ins_pipe(ialu_reg_shift);
12354 %}
12355
12356 // This pattern is automatically generated from aarch64_ad.m4
12357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12358
12359 // Shift Left followed by Shift Right.
12360 // This idiom is used by the compiler for the i2b bytecode etc.
12361 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12362 %{
12363 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12364 ins_cost(INSN_COST * 2);
12365 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12366 ins_encode %{
12367 int lshift = $lshift_count$$constant & 31;
12368 int rshift = $rshift_count$$constant & 31;
12369 int s = 31 - lshift;
12370 int r = (rshift - lshift) & 31;
12371 __ ubfmw(as_Register($dst$$reg),
12372 as_Register($src$$reg),
12373 r, s);
12374 %}
12375
12376 ins_pipe(ialu_reg_shift);
12377 %}
12378
12379 // Bitfield extract with shift & mask
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12384 %{
12385 match(Set dst (AndI (URShiftI src rshift) mask));
12386 // Make sure we are not going to exceed what ubfxw can do.
12387 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12388
12389 ins_cost(INSN_COST);
12390 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12391 ins_encode %{
12392 int rshift = $rshift$$constant & 31;
12393 intptr_t mask = $mask$$constant;
12394 int width = exact_log2(mask+1);
12395 __ ubfxw(as_Register($dst$$reg),
12396 as_Register($src$$reg), rshift, width);
12397 %}
12398 ins_pipe(ialu_reg_shift);
12399 %}
12400
12401 // This pattern is automatically generated from aarch64_ad.m4
12402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12403 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12404 %{
12405 match(Set dst (AndL (URShiftL src rshift) mask));
12406 // Make sure we are not going to exceed what ubfx can do.
12407 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12408
12409 ins_cost(INSN_COST);
12410 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12411 ins_encode %{
12412 int rshift = $rshift$$constant & 63;
12413 intptr_t mask = $mask$$constant;
12414 int width = exact_log2_long(mask+1);
12415 __ ubfx(as_Register($dst$$reg),
12416 as_Register($src$$reg), rshift, width);
12417 %}
12418 ins_pipe(ialu_reg_shift);
12419 %}
12420
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424
12425 // We can use ubfx when extending an And with a mask when we know mask
12426 // is positive. We know that because immI_bitmask guarantees it.
12427 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12428 %{
12429 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12430 // Make sure we are not going to exceed what ubfxw can do.
12431 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12432
12433 ins_cost(INSN_COST * 2);
12434 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12435 ins_encode %{
12436 int rshift = $rshift$$constant & 31;
12437 intptr_t mask = $mask$$constant;
12438 int width = exact_log2(mask+1);
12439 __ ubfx(as_Register($dst$$reg),
12440 as_Register($src$$reg), rshift, width);
12441 %}
12442 ins_pipe(ialu_reg_shift);
12443 %}
12444
12445
12446 // This pattern is automatically generated from aarch64_ad.m4
12447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12448
12449 // We can use ubfiz when masking by a positive number and then left shifting the result.
12450 // We know that the mask is positive because immI_bitmask guarantees it.
12451 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12452 %{
12453 match(Set dst (LShiftI (AndI src mask) lshift));
12454 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12455
12456 ins_cost(INSN_COST);
12457 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12458 ins_encode %{
12459 int lshift = $lshift$$constant & 31;
12460 intptr_t mask = $mask$$constant;
12461 int width = exact_log2(mask+1);
12462 __ ubfizw(as_Register($dst$$reg),
12463 as_Register($src$$reg), lshift, width);
12464 %}
12465 ins_pipe(ialu_reg_shift);
12466 %}
12467
12468 // This pattern is automatically generated from aarch64_ad.m4
12469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12470
12471 // We can use ubfiz when masking by a positive number and then left shifting the result.
12472 // We know that the mask is positive because immL_bitmask guarantees it.
12473 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12474 %{
12475 match(Set dst (LShiftL (AndL src mask) lshift));
12476 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12477
12478 ins_cost(INSN_COST);
12479 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12480 ins_encode %{
12481 int lshift = $lshift$$constant & 63;
12482 intptr_t mask = $mask$$constant;
12483 int width = exact_log2_long(mask+1);
12484 __ ubfiz(as_Register($dst$$reg),
12485 as_Register($src$$reg), lshift, width);
12486 %}
12487 ins_pipe(ialu_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492
12493 // We can use ubfiz when masking by a positive number and then left shifting the result.
12494 // We know that the mask is positive because immI_bitmask guarantees it.
12495 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12496 %{
12497 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12498 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12499
12500 ins_cost(INSN_COST);
12501 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12502 ins_encode %{
12503 int lshift = $lshift$$constant & 31;
12504 intptr_t mask = $mask$$constant;
12505 int width = exact_log2(mask+1);
12506 __ ubfizw(as_Register($dst$$reg),
12507 as_Register($src$$reg), lshift, width);
12508 %}
12509 ins_pipe(ialu_reg_shift);
12510 %}
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514
12515 // We can use ubfiz when masking by a positive number and then left shifting the result.
12516 // We know that the mask is positive because immL_bitmask guarantees it.
12517 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12518 %{
12519 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12520 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12521
12522 ins_cost(INSN_COST);
12523 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12524 ins_encode %{
12525 int lshift = $lshift$$constant & 63;
12526 intptr_t mask = $mask$$constant;
12527 int width = exact_log2_long(mask+1);
12528 __ ubfiz(as_Register($dst$$reg),
12529 as_Register($src$$reg), lshift, width);
12530 %}
12531 ins_pipe(ialu_reg_shift);
12532 %}
12533
12534
12535 // This pattern is automatically generated from aarch64_ad.m4
12536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12537
12538 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12539 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12540 %{
12541 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12542 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12543
12544 ins_cost(INSN_COST);
12545 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12546 ins_encode %{
12547 int lshift = $lshift$$constant & 63;
12548 intptr_t mask = $mask$$constant;
12549 int width = exact_log2(mask+1);
12550 __ ubfiz(as_Register($dst$$reg),
12551 as_Register($src$$reg), lshift, width);
12552 %}
12553 ins_pipe(ialu_reg_shift);
12554 %}
12555
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558
12559 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12560 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12561 %{
12562 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12563 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12564
12565 ins_cost(INSN_COST);
12566 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12567 ins_encode %{
12568 int lshift = $lshift$$constant & 31;
12569 intptr_t mask = $mask$$constant;
12570 int width = exact_log2(mask+1);
12571 __ ubfiz(as_Register($dst$$reg),
12572 as_Register($src$$reg), lshift, width);
12573 %}
12574 ins_pipe(ialu_reg_shift);
12575 %}
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579
12580 // Can skip int2long conversions after AND with small bitmask
12581 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12582 %{
12583 match(Set dst (ConvI2L (AndI src msk)));
12584 ins_cost(INSN_COST);
12585 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12586 ins_encode %{
12587 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12588 %}
12589 ins_pipe(ialu_reg_shift);
12590 %}
12591
12592
12593 // Rotations
12594
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12598 %{
12599 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12600 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12601
12602 ins_cost(INSN_COST);
12603 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12604
12605 ins_encode %{
12606 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12607 $rshift$$constant & 63);
12608 %}
12609 ins_pipe(ialu_reg_reg_extr);
12610 %}
12611
12612
12613 // This pattern is automatically generated from aarch64_ad.m4
12614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12615 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12616 %{
12617 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12618 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12619
12620 ins_cost(INSN_COST);
12621 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12622
12623 ins_encode %{
12624 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12625 $rshift$$constant & 31);
12626 %}
12627 ins_pipe(ialu_reg_reg_extr);
12628 %}
12629
12630
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12634 %{
12635 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12636 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12637
12638 ins_cost(INSN_COST);
12639 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12640
12641 ins_encode %{
12642 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12643 $rshift$$constant & 63);
12644 %}
12645 ins_pipe(ialu_reg_reg_extr);
12646 %}
12647
12648
12649 // This pattern is automatically generated from aarch64_ad.m4
12650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12651 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12652 %{
12653 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12654 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12655
12656 ins_cost(INSN_COST);
12657 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12658
12659 ins_encode %{
12660 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12661 $rshift$$constant & 31);
12662 %}
12663 ins_pipe(ialu_reg_reg_extr);
12664 %}
12665
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12669 %{
12670 match(Set dst (RotateRight src shift));
12671
12672 ins_cost(INSN_COST);
12673 format %{ "ror $dst, $src, $shift" %}
12674
12675 ins_encode %{
12676 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12677 $shift$$constant & 0x1f);
12678 %}
12679 ins_pipe(ialu_reg_reg_vshift);
12680 %}
12681
12682 // This pattern is automatically generated from aarch64_ad.m4
12683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12684 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12685 %{
12686 match(Set dst (RotateRight src shift));
12687
12688 ins_cost(INSN_COST);
12689 format %{ "ror $dst, $src, $shift" %}
12690
12691 ins_encode %{
12692 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12693 $shift$$constant & 0x3f);
12694 %}
12695 ins_pipe(ialu_reg_reg_vshift);
12696 %}
12697
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12701 %{
12702 match(Set dst (RotateRight src shift));
12703
12704 ins_cost(INSN_COST);
12705 format %{ "ror $dst, $src, $shift" %}
12706
12707 ins_encode %{
12708 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12709 %}
12710 ins_pipe(ialu_reg_reg_vshift);
12711 %}
12712
12713 // This pattern is automatically generated from aarch64_ad.m4
12714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12715 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12716 %{
12717 match(Set dst (RotateRight src shift));
12718
12719 ins_cost(INSN_COST);
12720 format %{ "ror $dst, $src, $shift" %}
12721
12722 ins_encode %{
12723 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12724 %}
12725 ins_pipe(ialu_reg_reg_vshift);
12726 %}
12727
12728 // This pattern is automatically generated from aarch64_ad.m4
12729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12730 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12731 %{
12732 match(Set dst (RotateLeft src shift));
12733
12734 ins_cost(INSN_COST);
12735 format %{ "rol $dst, $src, $shift" %}
12736
12737 ins_encode %{
12738 __ subw(rscratch1, zr, as_Register($shift$$reg));
12739 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12740 %}
12741 ins_pipe(ialu_reg_reg_vshift);
12742 %}
12743
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12747 %{
12748 match(Set dst (RotateLeft src shift));
12749
12750 ins_cost(INSN_COST);
12751 format %{ "rol $dst, $src, $shift" %}
12752
12753 ins_encode %{
12754 __ subw(rscratch1, zr, as_Register($shift$$reg));
12755 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12756 %}
12757 ins_pipe(ialu_reg_reg_vshift);
12758 %}
12759
12760
12761 // Add/subtract (extended)
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12766 %{
12767 match(Set dst (AddL src1 (ConvI2L src2)));
12768 ins_cost(INSN_COST);
12769 format %{ "add $dst, $src1, $src2, sxtw" %}
12770
12771 ins_encode %{
12772 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12773 as_Register($src2$$reg), ext::sxtw);
12774 %}
12775 ins_pipe(ialu_reg_reg);
12776 %}
12777
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12781 %{
12782 match(Set dst (SubL src1 (ConvI2L src2)));
12783 ins_cost(INSN_COST);
12784 format %{ "sub $dst, $src1, $src2, sxtw" %}
12785
12786 ins_encode %{
12787 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12788 as_Register($src2$$reg), ext::sxtw);
12789 %}
12790 ins_pipe(ialu_reg_reg);
12791 %}
12792
12793 // This pattern is automatically generated from aarch64_ad.m4
12794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12795 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12796 %{
12797 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12798 ins_cost(INSN_COST);
12799 format %{ "add $dst, $src1, $src2, sxth" %}
12800
12801 ins_encode %{
12802 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12803 as_Register($src2$$reg), ext::sxth);
12804 %}
12805 ins_pipe(ialu_reg_reg);
12806 %}
12807
12808 // This pattern is automatically generated from aarch64_ad.m4
12809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12810 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12811 %{
12812 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12813 ins_cost(INSN_COST);
12814 format %{ "add $dst, $src1, $src2, sxtb" %}
12815
12816 ins_encode %{
12817 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12818 as_Register($src2$$reg), ext::sxtb);
12819 %}
12820 ins_pipe(ialu_reg_reg);
12821 %}
12822
12823 // This pattern is automatically generated from aarch64_ad.m4
12824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12825 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12826 %{
12827 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12828 ins_cost(INSN_COST);
12829 format %{ "add $dst, $src1, $src2, uxtb" %}
12830
12831 ins_encode %{
12832 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12833 as_Register($src2$$reg), ext::uxtb);
12834 %}
12835 ins_pipe(ialu_reg_reg);
12836 %}
12837
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12841 %{
12842 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12843 ins_cost(INSN_COST);
12844 format %{ "add $dst, $src1, $src2, sxth" %}
12845
12846 ins_encode %{
12847 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12848 as_Register($src2$$reg), ext::sxth);
12849 %}
12850 ins_pipe(ialu_reg_reg);
12851 %}
12852
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12856 %{
12857 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12858 ins_cost(INSN_COST);
12859 format %{ "add $dst, $src1, $src2, sxtw" %}
12860
12861 ins_encode %{
12862 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12863 as_Register($src2$$reg), ext::sxtw);
12864 %}
12865 ins_pipe(ialu_reg_reg);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12871 %{
12872 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12873 ins_cost(INSN_COST);
12874 format %{ "add $dst, $src1, $src2, sxtb" %}
12875
12876 ins_encode %{
12877 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12878 as_Register($src2$$reg), ext::sxtb);
12879 %}
12880 ins_pipe(ialu_reg_reg);
12881 %}
12882
12883 // This pattern is automatically generated from aarch64_ad.m4
12884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12886 %{
12887 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12888 ins_cost(INSN_COST);
12889 format %{ "add $dst, $src1, $src2, uxtb" %}
12890
12891 ins_encode %{
12892 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12893 as_Register($src2$$reg), ext::uxtb);
12894 %}
12895 ins_pipe(ialu_reg_reg);
12896 %}
12897
12898 // This pattern is automatically generated from aarch64_ad.m4
12899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12900 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12901 %{
12902 match(Set dst (AddI src1 (AndI src2 mask)));
12903 ins_cost(INSN_COST);
12904 format %{ "addw $dst, $src1, $src2, uxtb" %}
12905
12906 ins_encode %{
12907 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12908 as_Register($src2$$reg), ext::uxtb);
12909 %}
12910 ins_pipe(ialu_reg_reg);
12911 %}
12912
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12916 %{
12917 match(Set dst (AddI src1 (AndI src2 mask)));
12918 ins_cost(INSN_COST);
12919 format %{ "addw $dst, $src1, $src2, uxth" %}
12920
12921 ins_encode %{
12922 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12923 as_Register($src2$$reg), ext::uxth);
12924 %}
12925 ins_pipe(ialu_reg_reg);
12926 %}
12927
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12931 %{
12932 match(Set dst (AddL src1 (AndL src2 mask)));
12933 ins_cost(INSN_COST);
12934 format %{ "add $dst, $src1, $src2, uxtb" %}
12935
12936 ins_encode %{
12937 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12938 as_Register($src2$$reg), ext::uxtb);
12939 %}
12940 ins_pipe(ialu_reg_reg);
12941 %}
12942
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12946 %{
12947 match(Set dst (AddL src1 (AndL src2 mask)));
12948 ins_cost(INSN_COST);
12949 format %{ "add $dst, $src1, $src2, uxth" %}
12950
12951 ins_encode %{
12952 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12953 as_Register($src2$$reg), ext::uxth);
12954 %}
12955 ins_pipe(ialu_reg_reg);
12956 %}
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12961 %{
12962 match(Set dst (AddL src1 (AndL src2 mask)));
12963 ins_cost(INSN_COST);
12964 format %{ "add $dst, $src1, $src2, uxtw" %}
12965
12966 ins_encode %{
12967 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12968 as_Register($src2$$reg), ext::uxtw);
12969 %}
12970 ins_pipe(ialu_reg_reg);
12971 %}
12972
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12976 %{
12977 match(Set dst (SubI src1 (AndI src2 mask)));
12978 ins_cost(INSN_COST);
12979 format %{ "subw $dst, $src1, $src2, uxtb" %}
12980
12981 ins_encode %{
12982 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12983 as_Register($src2$$reg), ext::uxtb);
12984 %}
12985 ins_pipe(ialu_reg_reg);
12986 %}
12987
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12991 %{
12992 match(Set dst (SubI src1 (AndI src2 mask)));
12993 ins_cost(INSN_COST);
12994 format %{ "subw $dst, $src1, $src2, uxth" %}
12995
12996 ins_encode %{
12997 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12998 as_Register($src2$$reg), ext::uxth);
12999 %}
13000 ins_pipe(ialu_reg_reg);
13001 %}
13002
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13006 %{
13007 match(Set dst (SubL src1 (AndL src2 mask)));
13008 ins_cost(INSN_COST);
13009 format %{ "sub $dst, $src1, $src2, uxtb" %}
13010
13011 ins_encode %{
13012 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13013 as_Register($src2$$reg), ext::uxtb);
13014 %}
13015 ins_pipe(ialu_reg_reg);
13016 %}
13017
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13021 %{
13022 match(Set dst (SubL src1 (AndL src2 mask)));
13023 ins_cost(INSN_COST);
13024 format %{ "sub $dst, $src1, $src2, uxth" %}
13025
13026 ins_encode %{
13027 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13028 as_Register($src2$$reg), ext::uxth);
13029 %}
13030 ins_pipe(ialu_reg_reg);
13031 %}
13032
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13036 %{
13037 match(Set dst (SubL src1 (AndL src2 mask)));
13038 ins_cost(INSN_COST);
13039 format %{ "sub $dst, $src1, $src2, uxtw" %}
13040
13041 ins_encode %{
13042 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13043 as_Register($src2$$reg), ext::uxtw);
13044 %}
13045 ins_pipe(ialu_reg_reg);
13046 %}
13047
13048
13049 // This pattern is automatically generated from aarch64_ad.m4
13050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13051 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13052 %{
13053 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13054 ins_cost(1.9 * INSN_COST);
13055 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13056
13057 ins_encode %{
13058 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13059 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13060 %}
13061 ins_pipe(ialu_reg_reg_shift);
13062 %}
13063
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13067 %{
13068 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13069 ins_cost(1.9 * INSN_COST);
13070 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13071
13072 ins_encode %{
13073 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13074 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13075 %}
13076 ins_pipe(ialu_reg_reg_shift);
13077 %}
13078
13079 // This pattern is automatically generated from aarch64_ad.m4
13080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13081 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13082 %{
13083 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13084 ins_cost(1.9 * INSN_COST);
13085 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13086
13087 ins_encode %{
13088 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13089 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13090 %}
13091 ins_pipe(ialu_reg_reg_shift);
13092 %}
13093
13094 // This pattern is automatically generated from aarch64_ad.m4
13095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13096 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13097 %{
13098 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13099 ins_cost(1.9 * INSN_COST);
13100 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13101
13102 ins_encode %{
13103 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13104 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13105 %}
13106 ins_pipe(ialu_reg_reg_shift);
13107 %}
13108
13109 // This pattern is automatically generated from aarch64_ad.m4
13110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13111 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13112 %{
13113 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13114 ins_cost(1.9 * INSN_COST);
13115 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13116
13117 ins_encode %{
13118 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13119 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13120 %}
13121 ins_pipe(ialu_reg_reg_shift);
13122 %}
13123
13124 // This pattern is automatically generated from aarch64_ad.m4
13125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13126 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13127 %{
13128 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13129 ins_cost(1.9 * INSN_COST);
13130 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13131
13132 ins_encode %{
13133 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13134 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13135 %}
13136 ins_pipe(ialu_reg_reg_shift);
13137 %}
13138
13139 // This pattern is automatically generated from aarch64_ad.m4
13140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13141 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13142 %{
13143 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13144 ins_cost(1.9 * INSN_COST);
13145 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13146
13147 ins_encode %{
13148 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13149 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13150 %}
13151 ins_pipe(ialu_reg_reg_shift);
13152 %}
13153
13154 // This pattern is automatically generated from aarch64_ad.m4
13155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13156 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13157 %{
13158 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13159 ins_cost(1.9 * INSN_COST);
13160 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13161
13162 ins_encode %{
13163 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13164 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13165 %}
13166 ins_pipe(ialu_reg_reg_shift);
13167 %}
13168
13169 // This pattern is automatically generated from aarch64_ad.m4
13170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13171 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13172 %{
13173 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13174 ins_cost(1.9 * INSN_COST);
13175 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13176
13177 ins_encode %{
13178 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13179 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13180 %}
13181 ins_pipe(ialu_reg_reg_shift);
13182 %}
13183
13184 // This pattern is automatically generated from aarch64_ad.m4
13185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13186 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13187 %{
13188 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13189 ins_cost(1.9 * INSN_COST);
13190 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13191
13192 ins_encode %{
13193 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13194 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13195 %}
13196 ins_pipe(ialu_reg_reg_shift);
13197 %}
13198
13199 // This pattern is automatically generated from aarch64_ad.m4
13200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13201 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13202 %{
13203 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13204 ins_cost(1.9 * INSN_COST);
13205 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13206
13207 ins_encode %{
13208 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13209 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13210 %}
13211 ins_pipe(ialu_reg_reg_shift);
13212 %}
13213
13214 // This pattern is automatically generated from aarch64_ad.m4
13215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13216 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13217 %{
13218 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13219 ins_cost(1.9 * INSN_COST);
13220 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13221
13222 ins_encode %{
13223 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13224 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13225 %}
13226 ins_pipe(ialu_reg_reg_shift);
13227 %}
13228
13229 // This pattern is automatically generated from aarch64_ad.m4
13230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13231 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13232 %{
13233 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13234 ins_cost(1.9 * INSN_COST);
13235 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13236
13237 ins_encode %{
13238 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13239 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13240 %}
13241 ins_pipe(ialu_reg_reg_shift);
13242 %}
13243
13244 // This pattern is automatically generated from aarch64_ad.m4
13245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13246 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13247 %{
13248 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13249 ins_cost(1.9 * INSN_COST);
13250 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13251
13252 ins_encode %{
13253 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13254 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13255 %}
13256 ins_pipe(ialu_reg_reg_shift);
13257 %}
13258
13259 // This pattern is automatically generated from aarch64_ad.m4
13260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13261 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13262 %{
13263 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13264 ins_cost(1.9 * INSN_COST);
13265 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13266
13267 ins_encode %{
13268 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13269 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13270 %}
13271 ins_pipe(ialu_reg_reg_shift);
13272 %}
13273
13274 // This pattern is automatically generated from aarch64_ad.m4
13275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13276 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13277 %{
13278 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13279 ins_cost(1.9 * INSN_COST);
13280 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13281
13282 ins_encode %{
13283 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13284 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13285 %}
13286 ins_pipe(ialu_reg_reg_shift);
13287 %}
13288
13289 // This pattern is automatically generated from aarch64_ad.m4
13290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13291 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13292 %{
13293 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13294 ins_cost(1.9 * INSN_COST);
13295 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13296
13297 ins_encode %{
13298 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13299 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13300 %}
13301 ins_pipe(ialu_reg_reg_shift);
13302 %}
13303
13304 // This pattern is automatically generated from aarch64_ad.m4
13305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13306 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13307 %{
13308 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13309 ins_cost(1.9 * INSN_COST);
13310 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13311
13312 ins_encode %{
13313 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13314 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13315 %}
13316 ins_pipe(ialu_reg_reg_shift);
13317 %}
13318
13319 // This pattern is automatically generated from aarch64_ad.m4
13320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13321 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13322 %{
13323 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13324 ins_cost(1.9 * INSN_COST);
13325 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13326
13327 ins_encode %{
13328 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13329 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13330 %}
13331 ins_pipe(ialu_reg_reg_shift);
13332 %}
13333
13334 // This pattern is automatically generated from aarch64_ad.m4
13335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13336 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13337 %{
13338 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13339 ins_cost(1.9 * INSN_COST);
13340 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13341
13342 ins_encode %{
13343 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13344 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13345 %}
13346 ins_pipe(ialu_reg_reg_shift);
13347 %}
13348
13349 // This pattern is automatically generated from aarch64_ad.m4
13350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13351 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13352 %{
13353 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13354 ins_cost(1.9 * INSN_COST);
13355 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13356
13357 ins_encode %{
13358 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13359 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13360 %}
13361 ins_pipe(ialu_reg_reg_shift);
13362 %}
13363
13364 // This pattern is automatically generated from aarch64_ad.m4
13365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13366 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13367 %{
13368 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13369 ins_cost(1.9 * INSN_COST);
13370 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13371
13372 ins_encode %{
13373 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13374 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13375 %}
13376 ins_pipe(ialu_reg_reg_shift);
13377 %}
13378
13379 // This pattern is automatically generated from aarch64_ad.m4
13380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13381 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13382 %{
13383 effect(DEF dst, USE src1, USE src2, USE cr);
13384 ins_cost(INSN_COST * 2);
13385 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13386
13387 ins_encode %{
13388 __ cselw($dst$$Register,
13389 $src1$$Register,
13390 $src2$$Register,
13391 Assembler::LT);
13392 %}
13393 ins_pipe(icond_reg_reg);
13394 %}
13395
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13399 %{
13400 effect(DEF dst, USE src1, USE src2, USE cr);
13401 ins_cost(INSN_COST * 2);
13402 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13403
13404 ins_encode %{
13405 __ cselw($dst$$Register,
13406 $src1$$Register,
13407 $src2$$Register,
13408 Assembler::GT);
13409 %}
13410 ins_pipe(icond_reg_reg);
13411 %}
13412
13413 // This pattern is automatically generated from aarch64_ad.m4
13414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13415 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13416 %{
13417 effect(DEF dst, USE src1, USE cr);
13418 ins_cost(INSN_COST * 2);
13419 format %{ "cselw $dst, $src1, zr lt\t" %}
13420
13421 ins_encode %{
13422 __ cselw($dst$$Register,
13423 $src1$$Register,
13424 zr,
13425 Assembler::LT);
13426 %}
13427 ins_pipe(icond_reg);
13428 %}
13429
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13433 %{
13434 effect(DEF dst, USE src1, USE cr);
13435 ins_cost(INSN_COST * 2);
13436 format %{ "cselw $dst, $src1, zr gt\t" %}
13437
13438 ins_encode %{
13439 __ cselw($dst$$Register,
13440 $src1$$Register,
13441 zr,
13442 Assembler::GT);
13443 %}
13444 ins_pipe(icond_reg);
13445 %}
13446
13447 // This pattern is automatically generated from aarch64_ad.m4
13448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13449 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13450 %{
13451 effect(DEF dst, USE src1, USE cr);
13452 ins_cost(INSN_COST * 2);
13453 format %{ "csincw $dst, $src1, zr le\t" %}
13454
13455 ins_encode %{
13456 __ csincw($dst$$Register,
13457 $src1$$Register,
13458 zr,
13459 Assembler::LE);
13460 %}
13461 ins_pipe(icond_reg);
13462 %}
13463
13464 // This pattern is automatically generated from aarch64_ad.m4
13465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13466 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13467 %{
13468 effect(DEF dst, USE src1, USE cr);
13469 ins_cost(INSN_COST * 2);
13470 format %{ "csincw $dst, $src1, zr gt\t" %}
13471
13472 ins_encode %{
13473 __ csincw($dst$$Register,
13474 $src1$$Register,
13475 zr,
13476 Assembler::GT);
13477 %}
13478 ins_pipe(icond_reg);
13479 %}
13480
13481 // This pattern is automatically generated from aarch64_ad.m4
13482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13483 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13484 %{
13485 effect(DEF dst, USE src1, USE cr);
13486 ins_cost(INSN_COST * 2);
13487 format %{ "csinvw $dst, $src1, zr lt\t" %}
13488
13489 ins_encode %{
13490 __ csinvw($dst$$Register,
13491 $src1$$Register,
13492 zr,
13493 Assembler::LT);
13494 %}
13495 ins_pipe(icond_reg);
13496 %}
13497
13498 // This pattern is automatically generated from aarch64_ad.m4
13499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13500 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13501 %{
13502 effect(DEF dst, USE src1, USE cr);
13503 ins_cost(INSN_COST * 2);
13504 format %{ "csinvw $dst, $src1, zr ge\t" %}
13505
13506 ins_encode %{
13507 __ csinvw($dst$$Register,
13508 $src1$$Register,
13509 zr,
13510 Assembler::GE);
13511 %}
13512 ins_pipe(icond_reg);
13513 %}
13514
13515 // This pattern is automatically generated from aarch64_ad.m4
13516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13517 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13518 %{
13519 match(Set dst (MinI src imm));
13520 ins_cost(INSN_COST * 3);
13521 expand %{
13522 rFlagsReg cr;
13523 compI_reg_imm0(cr, src);
13524 cmovI_reg_imm0_lt(dst, src, cr);
13525 %}
13526 %}
13527
13528 // This pattern is automatically generated from aarch64_ad.m4
13529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13530 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13531 %{
13532 match(Set dst (MinI imm src));
13533 ins_cost(INSN_COST * 3);
13534 expand %{
13535 rFlagsReg cr;
13536 compI_reg_imm0(cr, src);
13537 cmovI_reg_imm0_lt(dst, src, cr);
13538 %}
13539 %}
13540
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13544 %{
13545 match(Set dst (MinI src imm));
13546 ins_cost(INSN_COST * 3);
13547 expand %{
13548 rFlagsReg cr;
13549 compI_reg_imm0(cr, src);
13550 cmovI_reg_imm1_le(dst, src, cr);
13551 %}
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13557 %{
13558 match(Set dst (MinI imm src));
13559 ins_cost(INSN_COST * 3);
13560 expand %{
13561 rFlagsReg cr;
13562 compI_reg_imm0(cr, src);
13563 cmovI_reg_imm1_le(dst, src, cr);
13564 %}
13565 %}
13566
13567 // This pattern is automatically generated from aarch64_ad.m4
13568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13569 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13570 %{
13571 match(Set dst (MinI src imm));
13572 ins_cost(INSN_COST * 3);
13573 expand %{
13574 rFlagsReg cr;
13575 compI_reg_imm0(cr, src);
13576 cmovI_reg_immM1_lt(dst, src, cr);
13577 %}
13578 %}
13579
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13583 %{
13584 match(Set dst (MinI imm src));
13585 ins_cost(INSN_COST * 3);
13586 expand %{
13587 rFlagsReg cr;
13588 compI_reg_imm0(cr, src);
13589 cmovI_reg_immM1_lt(dst, src, cr);
13590 %}
13591 %}
13592
13593 // This pattern is automatically generated from aarch64_ad.m4
13594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13595 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13596 %{
13597 match(Set dst (MaxI src imm));
13598 ins_cost(INSN_COST * 3);
13599 expand %{
13600 rFlagsReg cr;
13601 compI_reg_imm0(cr, src);
13602 cmovI_reg_imm0_gt(dst, src, cr);
13603 %}
13604 %}
13605
13606 // This pattern is automatically generated from aarch64_ad.m4
13607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13608 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13609 %{
13610 match(Set dst (MaxI imm src));
13611 ins_cost(INSN_COST * 3);
13612 expand %{
13613 rFlagsReg cr;
13614 compI_reg_imm0(cr, src);
13615 cmovI_reg_imm0_gt(dst, src, cr);
13616 %}
13617 %}
13618
13619 // This pattern is automatically generated from aarch64_ad.m4
13620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13621 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13622 %{
13623 match(Set dst (MaxI src imm));
13624 ins_cost(INSN_COST * 3);
13625 expand %{
13626 rFlagsReg cr;
13627 compI_reg_imm0(cr, src);
13628 cmovI_reg_imm1_gt(dst, src, cr);
13629 %}
13630 %}
13631
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13635 %{
13636 match(Set dst (MaxI imm src));
13637 ins_cost(INSN_COST * 3);
13638 expand %{
13639 rFlagsReg cr;
13640 compI_reg_imm0(cr, src);
13641 cmovI_reg_imm1_gt(dst, src, cr);
13642 %}
13643 %}
13644
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13648 %{
13649 match(Set dst (MaxI src imm));
13650 ins_cost(INSN_COST * 3);
13651 expand %{
13652 rFlagsReg cr;
13653 compI_reg_imm0(cr, src);
13654 cmovI_reg_immM1_ge(dst, src, cr);
13655 %}
13656 %}
13657
13658 // This pattern is automatically generated from aarch64_ad.m4
13659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13660 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13661 %{
13662 match(Set dst (MaxI imm src));
13663 ins_cost(INSN_COST * 3);
13664 expand %{
13665 rFlagsReg cr;
13666 compI_reg_imm0(cr, src);
13667 cmovI_reg_immM1_ge(dst, src, cr);
13668 %}
13669 %}
13670
13671 // This pattern is automatically generated from aarch64_ad.m4
13672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13673 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13674 %{
13675 match(Set dst (ReverseI src));
13676 ins_cost(INSN_COST);
13677 format %{ "rbitw $dst, $src" %}
13678 ins_encode %{
13679 __ rbitw($dst$$Register, $src$$Register);
13680 %}
13681 ins_pipe(ialu_reg);
13682 %}
13683
13684 // This pattern is automatically generated from aarch64_ad.m4
13685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13686 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13687 %{
13688 match(Set dst (ReverseL src));
13689 ins_cost(INSN_COST);
13690 format %{ "rbit $dst, $src" %}
13691 ins_encode %{
13692 __ rbit($dst$$Register, $src$$Register);
13693 %}
13694 ins_pipe(ialu_reg);
13695 %}
13696
13697
13698 // END This section of the file is automatically generated. Do not edit --------------
13699
13700
13701 // ============================================================================
13702 // Floating Point Arithmetic Instructions
13703
13704 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13705 match(Set dst (AddHF src1 src2));
13706 format %{ "faddh $dst, $src1, $src2" %}
13707 ins_encode %{
13708 __ faddh($dst$$FloatRegister,
13709 $src1$$FloatRegister,
13710 $src2$$FloatRegister);
13711 %}
13712 ins_pipe(fp_dop_reg_reg_s);
13713 %}
13714
13715 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13716 match(Set dst (AddF src1 src2));
13717
13718 ins_cost(INSN_COST * 5);
13719 format %{ "fadds $dst, $src1, $src2" %}
13720
13721 ins_encode %{
13722 __ fadds(as_FloatRegister($dst$$reg),
13723 as_FloatRegister($src1$$reg),
13724 as_FloatRegister($src2$$reg));
13725 %}
13726
13727 ins_pipe(fp_dop_reg_reg_s);
13728 %}
13729
13730 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13731 match(Set dst (AddD src1 src2));
13732
13733 ins_cost(INSN_COST * 5);
13734 format %{ "faddd $dst, $src1, $src2" %}
13735
13736 ins_encode %{
13737 __ faddd(as_FloatRegister($dst$$reg),
13738 as_FloatRegister($src1$$reg),
13739 as_FloatRegister($src2$$reg));
13740 %}
13741
13742 ins_pipe(fp_dop_reg_reg_d);
13743 %}
13744
13745 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13746 match(Set dst (SubHF src1 src2));
13747 format %{ "fsubh $dst, $src1, $src2" %}
13748 ins_encode %{
13749 __ fsubh($dst$$FloatRegister,
13750 $src1$$FloatRegister,
13751 $src2$$FloatRegister);
13752 %}
13753 ins_pipe(fp_dop_reg_reg_s);
13754 %}
13755
13756 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13757 match(Set dst (SubF src1 src2));
13758
13759 ins_cost(INSN_COST * 5);
13760 format %{ "fsubs $dst, $src1, $src2" %}
13761
13762 ins_encode %{
13763 __ fsubs(as_FloatRegister($dst$$reg),
13764 as_FloatRegister($src1$$reg),
13765 as_FloatRegister($src2$$reg));
13766 %}
13767
13768 ins_pipe(fp_dop_reg_reg_s);
13769 %}
13770
13771 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13772 match(Set dst (SubD src1 src2));
13773
13774 ins_cost(INSN_COST * 5);
13775 format %{ "fsubd $dst, $src1, $src2" %}
13776
13777 ins_encode %{
13778 __ fsubd(as_FloatRegister($dst$$reg),
13779 as_FloatRegister($src1$$reg),
13780 as_FloatRegister($src2$$reg));
13781 %}
13782
13783 ins_pipe(fp_dop_reg_reg_d);
13784 %}
13785
13786 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13787 match(Set dst (MulHF src1 src2));
13788 format %{ "fmulh $dst, $src1, $src2" %}
13789 ins_encode %{
13790 __ fmulh($dst$$FloatRegister,
13791 $src1$$FloatRegister,
13792 $src2$$FloatRegister);
13793 %}
13794 ins_pipe(fp_dop_reg_reg_s);
13795 %}
13796
13797 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13798 match(Set dst (MulF src1 src2));
13799
13800 ins_cost(INSN_COST * 6);
13801 format %{ "fmuls $dst, $src1, $src2" %}
13802
13803 ins_encode %{
13804 __ fmuls(as_FloatRegister($dst$$reg),
13805 as_FloatRegister($src1$$reg),
13806 as_FloatRegister($src2$$reg));
13807 %}
13808
13809 ins_pipe(fp_dop_reg_reg_s);
13810 %}
13811
13812 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13813 match(Set dst (MulD src1 src2));
13814
13815 ins_cost(INSN_COST * 6);
13816 format %{ "fmuld $dst, $src1, $src2" %}
13817
13818 ins_encode %{
13819 __ fmuld(as_FloatRegister($dst$$reg),
13820 as_FloatRegister($src1$$reg),
13821 as_FloatRegister($src2$$reg));
13822 %}
13823
13824 ins_pipe(fp_dop_reg_reg_d);
13825 %}
13826
13827 // src1 * src2 + src3 (half-precision float)
13828 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13829 match(Set dst (FmaHF src3 (Binary src1 src2)));
13830 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13831 ins_encode %{
13832 assert(UseFMA, "Needs FMA instructions support.");
13833 __ fmaddh($dst$$FloatRegister,
13834 $src1$$FloatRegister,
13835 $src2$$FloatRegister,
13836 $src3$$FloatRegister);
13837 %}
13838 ins_pipe(pipe_class_default);
13839 %}
13840
13841 // src1 * src2 + src3
13842 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13843 match(Set dst (FmaF src3 (Binary src1 src2)));
13844
13845 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13846
13847 ins_encode %{
13848 assert(UseFMA, "Needs FMA instructions support.");
13849 __ fmadds(as_FloatRegister($dst$$reg),
13850 as_FloatRegister($src1$$reg),
13851 as_FloatRegister($src2$$reg),
13852 as_FloatRegister($src3$$reg));
13853 %}
13854
13855 ins_pipe(pipe_class_default);
13856 %}
13857
13858 // src1 * src2 + src3
13859 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13860 match(Set dst (FmaD src3 (Binary src1 src2)));
13861
13862 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13863
13864 ins_encode %{
13865 assert(UseFMA, "Needs FMA instructions support.");
13866 __ fmaddd(as_FloatRegister($dst$$reg),
13867 as_FloatRegister($src1$$reg),
13868 as_FloatRegister($src2$$reg),
13869 as_FloatRegister($src3$$reg));
13870 %}
13871
13872 ins_pipe(pipe_class_default);
13873 %}
13874
13875 // src1 * (-src2) + src3
13876 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13877 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13878 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13879
13880 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13881
13882 ins_encode %{
13883 assert(UseFMA, "Needs FMA instructions support.");
13884 __ fmsubs(as_FloatRegister($dst$$reg),
13885 as_FloatRegister($src1$$reg),
13886 as_FloatRegister($src2$$reg),
13887 as_FloatRegister($src3$$reg));
13888 %}
13889
13890 ins_pipe(pipe_class_default);
13891 %}
13892
13893 // src1 * (-src2) + src3
13894 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13895 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13896 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13897
13898 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13899
13900 ins_encode %{
13901 assert(UseFMA, "Needs FMA instructions support.");
13902 __ fmsubd(as_FloatRegister($dst$$reg),
13903 as_FloatRegister($src1$$reg),
13904 as_FloatRegister($src2$$reg),
13905 as_FloatRegister($src3$$reg));
13906 %}
13907
13908 ins_pipe(pipe_class_default);
13909 %}
13910
13911 // src1 * (-src2) - src3
13912 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13913 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13914 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13915
13916 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13917
13918 ins_encode %{
13919 assert(UseFMA, "Needs FMA instructions support.");
13920 __ fnmadds(as_FloatRegister($dst$$reg),
13921 as_FloatRegister($src1$$reg),
13922 as_FloatRegister($src2$$reg),
13923 as_FloatRegister($src3$$reg));
13924 %}
13925
13926 ins_pipe(pipe_class_default);
13927 %}
13928
13929 // src1 * (-src2) - src3
13930 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13931 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13932 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13933
13934 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13935
13936 ins_encode %{
13937 assert(UseFMA, "Needs FMA instructions support.");
13938 __ fnmaddd(as_FloatRegister($dst$$reg),
13939 as_FloatRegister($src1$$reg),
13940 as_FloatRegister($src2$$reg),
13941 as_FloatRegister($src3$$reg));
13942 %}
13943
13944 ins_pipe(pipe_class_default);
13945 %}
13946
13947 // src1 * src2 - src3
13948 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13949 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13950
13951 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13952
13953 ins_encode %{
13954 assert(UseFMA, "Needs FMA instructions support.");
13955 __ fnmsubs(as_FloatRegister($dst$$reg),
13956 as_FloatRegister($src1$$reg),
13957 as_FloatRegister($src2$$reg),
13958 as_FloatRegister($src3$$reg));
13959 %}
13960
13961 ins_pipe(pipe_class_default);
13962 %}
13963
13964 // src1 * src2 - src3
13965 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13966 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13967
13968 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13969
13970 ins_encode %{
13971 assert(UseFMA, "Needs FMA instructions support.");
13972 // n.b. insn name should be fnmsubd
13973 __ fnmsub(as_FloatRegister($dst$$reg),
13974 as_FloatRegister($src1$$reg),
13975 as_FloatRegister($src2$$reg),
13976 as_FloatRegister($src3$$reg));
13977 %}
13978
13979 ins_pipe(pipe_class_default);
13980 %}
13981
13982 // Math.max(HH)H (half-precision float)
13983 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13984 match(Set dst (MaxHF src1 src2));
13985 format %{ "fmaxh $dst, $src1, $src2" %}
13986 ins_encode %{
13987 __ fmaxh($dst$$FloatRegister,
13988 $src1$$FloatRegister,
13989 $src2$$FloatRegister);
13990 %}
13991 ins_pipe(fp_dop_reg_reg_s);
13992 %}
13993
13994 // Math.min(HH)H (half-precision float)
13995 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13996 match(Set dst (MinHF src1 src2));
13997 format %{ "fminh $dst, $src1, $src2" %}
13998 ins_encode %{
13999 __ fminh($dst$$FloatRegister,
14000 $src1$$FloatRegister,
14001 $src2$$FloatRegister);
14002 %}
14003 ins_pipe(fp_dop_reg_reg_s);
14004 %}
14005
14006 // Math.max(FF)F
14007 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14008 match(Set dst (MaxF src1 src2));
14009
14010 format %{ "fmaxs $dst, $src1, $src2" %}
14011 ins_encode %{
14012 __ fmaxs(as_FloatRegister($dst$$reg),
14013 as_FloatRegister($src1$$reg),
14014 as_FloatRegister($src2$$reg));
14015 %}
14016
14017 ins_pipe(fp_dop_reg_reg_s);
14018 %}
14019
14020 // Math.min(FF)F
14021 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14022 match(Set dst (MinF src1 src2));
14023
14024 format %{ "fmins $dst, $src1, $src2" %}
14025 ins_encode %{
14026 __ fmins(as_FloatRegister($dst$$reg),
14027 as_FloatRegister($src1$$reg),
14028 as_FloatRegister($src2$$reg));
14029 %}
14030
14031 ins_pipe(fp_dop_reg_reg_s);
14032 %}
14033
14034 // Math.max(DD)D
14035 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14036 match(Set dst (MaxD src1 src2));
14037
14038 format %{ "fmaxd $dst, $src1, $src2" %}
14039 ins_encode %{
14040 __ fmaxd(as_FloatRegister($dst$$reg),
14041 as_FloatRegister($src1$$reg),
14042 as_FloatRegister($src2$$reg));
14043 %}
14044
14045 ins_pipe(fp_dop_reg_reg_d);
14046 %}
14047
14048 // Math.min(DD)D
14049 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14050 match(Set dst (MinD src1 src2));
14051
14052 format %{ "fmind $dst, $src1, $src2" %}
14053 ins_encode %{
14054 __ fmind(as_FloatRegister($dst$$reg),
14055 as_FloatRegister($src1$$reg),
14056 as_FloatRegister($src2$$reg));
14057 %}
14058
14059 ins_pipe(fp_dop_reg_reg_d);
14060 %}
14061
14062 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14063 match(Set dst (DivHF src1 src2));
14064 format %{ "fdivh $dst, $src1, $src2" %}
14065 ins_encode %{
14066 __ fdivh($dst$$FloatRegister,
14067 $src1$$FloatRegister,
14068 $src2$$FloatRegister);
14069 %}
14070 ins_pipe(fp_div_s);
14071 %}
14072
14073 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14074 match(Set dst (DivF src1 src2));
14075
14076 ins_cost(INSN_COST * 18);
14077 format %{ "fdivs $dst, $src1, $src2" %}
14078
14079 ins_encode %{
14080 __ fdivs(as_FloatRegister($dst$$reg),
14081 as_FloatRegister($src1$$reg),
14082 as_FloatRegister($src2$$reg));
14083 %}
14084
14085 ins_pipe(fp_div_s);
14086 %}
14087
14088 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14089 match(Set dst (DivD src1 src2));
14090
14091 ins_cost(INSN_COST * 32);
14092 format %{ "fdivd $dst, $src1, $src2" %}
14093
14094 ins_encode %{
14095 __ fdivd(as_FloatRegister($dst$$reg),
14096 as_FloatRegister($src1$$reg),
14097 as_FloatRegister($src2$$reg));
14098 %}
14099
14100 ins_pipe(fp_div_d);
14101 %}
14102
14103 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14104 match(Set dst (NegF src));
14105
14106 ins_cost(INSN_COST * 3);
14107 format %{ "fneg $dst, $src" %}
14108
14109 ins_encode %{
14110 __ fnegs(as_FloatRegister($dst$$reg),
14111 as_FloatRegister($src$$reg));
14112 %}
14113
14114 ins_pipe(fp_uop_s);
14115 %}
14116
14117 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14118 match(Set dst (NegD src));
14119
14120 ins_cost(INSN_COST * 3);
14121 format %{ "fnegd $dst, $src" %}
14122
14123 ins_encode %{
14124 __ fnegd(as_FloatRegister($dst$$reg),
14125 as_FloatRegister($src$$reg));
14126 %}
14127
14128 ins_pipe(fp_uop_d);
14129 %}
14130
14131 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14132 %{
14133 match(Set dst (AbsI src));
14134
14135 effect(KILL cr);
14136 ins_cost(INSN_COST * 2);
14137 format %{ "cmpw $src, zr\n\t"
14138 "cnegw $dst, $src, Assembler::LT\t# int abs"
14139 %}
14140
14141 ins_encode %{
14142 __ cmpw(as_Register($src$$reg), zr);
14143 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14144 %}
14145 ins_pipe(pipe_class_default);
14146 %}
14147
14148 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14149 %{
14150 match(Set dst (AbsL src));
14151
14152 effect(KILL cr);
14153 ins_cost(INSN_COST * 2);
14154 format %{ "cmp $src, zr\n\t"
14155 "cneg $dst, $src, Assembler::LT\t# long abs"
14156 %}
14157
14158 ins_encode %{
14159 __ cmp(as_Register($src$$reg), zr);
14160 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14161 %}
14162 ins_pipe(pipe_class_default);
14163 %}
14164
14165 instruct absF_reg(vRegF dst, vRegF src) %{
14166 match(Set dst (AbsF src));
14167
14168 ins_cost(INSN_COST * 3);
14169 format %{ "fabss $dst, $src" %}
14170 ins_encode %{
14171 __ fabss(as_FloatRegister($dst$$reg),
14172 as_FloatRegister($src$$reg));
14173 %}
14174
14175 ins_pipe(fp_uop_s);
14176 %}
14177
14178 instruct absD_reg(vRegD dst, vRegD src) %{
14179 match(Set dst (AbsD src));
14180
14181 ins_cost(INSN_COST * 3);
14182 format %{ "fabsd $dst, $src" %}
14183 ins_encode %{
14184 __ fabsd(as_FloatRegister($dst$$reg),
14185 as_FloatRegister($src$$reg));
14186 %}
14187
14188 ins_pipe(fp_uop_d);
14189 %}
14190
14191 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14192 match(Set dst (AbsF (SubF src1 src2)));
14193
14194 ins_cost(INSN_COST * 3);
14195 format %{ "fabds $dst, $src1, $src2" %}
14196 ins_encode %{
14197 __ fabds(as_FloatRegister($dst$$reg),
14198 as_FloatRegister($src1$$reg),
14199 as_FloatRegister($src2$$reg));
14200 %}
14201
14202 ins_pipe(fp_uop_s);
14203 %}
14204
14205 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14206 match(Set dst (AbsD (SubD src1 src2)));
14207
14208 ins_cost(INSN_COST * 3);
14209 format %{ "fabdd $dst, $src1, $src2" %}
14210 ins_encode %{
14211 __ fabdd(as_FloatRegister($dst$$reg),
14212 as_FloatRegister($src1$$reg),
14213 as_FloatRegister($src2$$reg));
14214 %}
14215
14216 ins_pipe(fp_uop_d);
14217 %}
14218
14219 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14220 match(Set dst (SqrtD src));
14221
14222 ins_cost(INSN_COST * 50);
14223 format %{ "fsqrtd $dst, $src" %}
14224 ins_encode %{
14225 __ fsqrtd(as_FloatRegister($dst$$reg),
14226 as_FloatRegister($src$$reg));
14227 %}
14228
14229 ins_pipe(fp_div_s);
14230 %}
14231
14232 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14233 match(Set dst (SqrtF src));
14234
14235 ins_cost(INSN_COST * 50);
14236 format %{ "fsqrts $dst, $src" %}
14237 ins_encode %{
14238 __ fsqrts(as_FloatRegister($dst$$reg),
14239 as_FloatRegister($src$$reg));
14240 %}
14241
14242 ins_pipe(fp_div_d);
14243 %}
14244
14245 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14246 match(Set dst (SqrtHF src));
14247 format %{ "fsqrth $dst, $src" %}
14248 ins_encode %{
14249 __ fsqrth($dst$$FloatRegister,
14250 $src$$FloatRegister);
14251 %}
14252 ins_pipe(fp_div_s);
14253 %}
14254
14255 // Math.rint, floor, ceil
14256 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14257 match(Set dst (RoundDoubleMode src rmode));
14258 format %{ "frint $dst, $src, $rmode" %}
14259 ins_encode %{
14260 switch ($rmode$$constant) {
14261 case RoundDoubleModeNode::rmode_rint:
14262 __ frintnd(as_FloatRegister($dst$$reg),
14263 as_FloatRegister($src$$reg));
14264 break;
14265 case RoundDoubleModeNode::rmode_floor:
14266 __ frintmd(as_FloatRegister($dst$$reg),
14267 as_FloatRegister($src$$reg));
14268 break;
14269 case RoundDoubleModeNode::rmode_ceil:
14270 __ frintpd(as_FloatRegister($dst$$reg),
14271 as_FloatRegister($src$$reg));
14272 break;
14273 }
14274 %}
14275 ins_pipe(fp_uop_d);
14276 %}
14277
14278 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14279 match(Set dst (CopySignD src1 (Binary src2 zero)));
14280 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14281 format %{ "CopySignD $dst $src1 $src2" %}
14282 ins_encode %{
14283 FloatRegister dst = as_FloatRegister($dst$$reg),
14284 src1 = as_FloatRegister($src1$$reg),
14285 src2 = as_FloatRegister($src2$$reg),
14286 zero = as_FloatRegister($zero$$reg);
14287 __ fnegd(dst, zero);
14288 __ bsl(dst, __ T8B, src2, src1);
14289 %}
14290 ins_pipe(fp_uop_d);
14291 %}
14292
14293 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14294 match(Set dst (CopySignF src1 src2));
14295 effect(TEMP_DEF dst, USE src1, USE src2);
14296 format %{ "CopySignF $dst $src1 $src2" %}
14297 ins_encode %{
14298 FloatRegister dst = as_FloatRegister($dst$$reg),
14299 src1 = as_FloatRegister($src1$$reg),
14300 src2 = as_FloatRegister($src2$$reg);
14301 __ movi(dst, __ T2S, 0x80, 24);
14302 __ bsl(dst, __ T8B, src2, src1);
14303 %}
14304 ins_pipe(fp_uop_d);
14305 %}
14306
14307 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14308 match(Set dst (SignumD src (Binary zero one)));
14309 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14310 format %{ "signumD $dst, $src" %}
14311 ins_encode %{
14312 FloatRegister src = as_FloatRegister($src$$reg),
14313 dst = as_FloatRegister($dst$$reg),
14314 zero = as_FloatRegister($zero$$reg),
14315 one = as_FloatRegister($one$$reg);
14316 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14317 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14318 // Bit selection instruction gets bit from "one" for each enabled bit in
14319 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14320 // NaN the whole "src" will be copied because "dst" is zero. For all other
14321 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14322 // from "src", and all other bits are copied from 1.0.
14323 __ bsl(dst, __ T8B, one, src);
14324 %}
14325 ins_pipe(fp_uop_d);
14326 %}
14327
14328 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14329 match(Set dst (SignumF src (Binary zero one)));
14330 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14331 format %{ "signumF $dst, $src" %}
14332 ins_encode %{
14333 FloatRegister src = as_FloatRegister($src$$reg),
14334 dst = as_FloatRegister($dst$$reg),
14335 zero = as_FloatRegister($zero$$reg),
14336 one = as_FloatRegister($one$$reg);
14337 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14338 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14339 // Bit selection instruction gets bit from "one" for each enabled bit in
14340 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14341 // NaN the whole "src" will be copied because "dst" is zero. For all other
14342 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14343 // from "src", and all other bits are copied from 1.0.
14344 __ bsl(dst, __ T8B, one, src);
14345 %}
14346 ins_pipe(fp_uop_d);
14347 %}
14348
14349 instruct onspinwait() %{
14350 match(OnSpinWait);
14351 ins_cost(INSN_COST);
14352
14353 format %{ "onspinwait" %}
14354
14355 ins_encode %{
14356 __ spin_wait();
14357 %}
14358 ins_pipe(pipe_class_empty);
14359 %}
14360
14361 // ============================================================================
14362 // Logical Instructions
14363
14364 // Integer Logical Instructions
14365
14366 // And Instructions
14367
14368
14369 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14370 match(Set dst (AndI src1 src2));
14371
14372 format %{ "andw $dst, $src1, $src2\t# int" %}
14373
14374 ins_cost(INSN_COST);
14375 ins_encode %{
14376 __ andw(as_Register($dst$$reg),
14377 as_Register($src1$$reg),
14378 as_Register($src2$$reg));
14379 %}
14380
14381 ins_pipe(ialu_reg_reg);
14382 %}
14383
14384 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14385 match(Set dst (AndI src1 src2));
14386
14387 format %{ "andsw $dst, $src1, $src2\t# int" %}
14388
14389 ins_cost(INSN_COST);
14390 ins_encode %{
14391 __ andw(as_Register($dst$$reg),
14392 as_Register($src1$$reg),
14393 (uint64_t)($src2$$constant));
14394 %}
14395
14396 ins_pipe(ialu_reg_imm);
14397 %}
14398
14399 // Or Instructions
14400
14401 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14402 match(Set dst (OrI src1 src2));
14403
14404 format %{ "orrw $dst, $src1, $src2\t# int" %}
14405
14406 ins_cost(INSN_COST);
14407 ins_encode %{
14408 __ orrw(as_Register($dst$$reg),
14409 as_Register($src1$$reg),
14410 as_Register($src2$$reg));
14411 %}
14412
14413 ins_pipe(ialu_reg_reg);
14414 %}
14415
14416 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14417 match(Set dst (OrI src1 src2));
14418
14419 format %{ "orrw $dst, $src1, $src2\t# int" %}
14420
14421 ins_cost(INSN_COST);
14422 ins_encode %{
14423 __ orrw(as_Register($dst$$reg),
14424 as_Register($src1$$reg),
14425 (uint64_t)($src2$$constant));
14426 %}
14427
14428 ins_pipe(ialu_reg_imm);
14429 %}
14430
14431 // Xor Instructions
14432
14433 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14434 match(Set dst (XorI src1 src2));
14435
14436 format %{ "eorw $dst, $src1, $src2\t# int" %}
14437
14438 ins_cost(INSN_COST);
14439 ins_encode %{
14440 __ eorw(as_Register($dst$$reg),
14441 as_Register($src1$$reg),
14442 as_Register($src2$$reg));
14443 %}
14444
14445 ins_pipe(ialu_reg_reg);
14446 %}
14447
14448 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14449 match(Set dst (XorI src1 src2));
14450
14451 format %{ "eorw $dst, $src1, $src2\t# int" %}
14452
14453 ins_cost(INSN_COST);
14454 ins_encode %{
14455 __ eorw(as_Register($dst$$reg),
14456 as_Register($src1$$reg),
14457 (uint64_t)($src2$$constant));
14458 %}
14459
14460 ins_pipe(ialu_reg_imm);
14461 %}
14462
14463 // Long Logical Instructions
14464 // TODO
14465
14466 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14467 match(Set dst (AndL src1 src2));
14468
14469 format %{ "and $dst, $src1, $src2\t# int" %}
14470
14471 ins_cost(INSN_COST);
14472 ins_encode %{
14473 __ andr(as_Register($dst$$reg),
14474 as_Register($src1$$reg),
14475 as_Register($src2$$reg));
14476 %}
14477
14478 ins_pipe(ialu_reg_reg);
14479 %}
14480
14481 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14482 match(Set dst (AndL src1 src2));
14483
14484 format %{ "and $dst, $src1, $src2\t# int" %}
14485
14486 ins_cost(INSN_COST);
14487 ins_encode %{
14488 __ andr(as_Register($dst$$reg),
14489 as_Register($src1$$reg),
14490 (uint64_t)($src2$$constant));
14491 %}
14492
14493 ins_pipe(ialu_reg_imm);
14494 %}
14495
14496 // Or Instructions
14497
14498 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14499 match(Set dst (OrL src1 src2));
14500
14501 format %{ "orr $dst, $src1, $src2\t# int" %}
14502
14503 ins_cost(INSN_COST);
14504 ins_encode %{
14505 __ orr(as_Register($dst$$reg),
14506 as_Register($src1$$reg),
14507 as_Register($src2$$reg));
14508 %}
14509
14510 ins_pipe(ialu_reg_reg);
14511 %}
14512
14513 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14514 match(Set dst (OrL src1 src2));
14515
14516 format %{ "orr $dst, $src1, $src2\t# int" %}
14517
14518 ins_cost(INSN_COST);
14519 ins_encode %{
14520 __ orr(as_Register($dst$$reg),
14521 as_Register($src1$$reg),
14522 (uint64_t)($src2$$constant));
14523 %}
14524
14525 ins_pipe(ialu_reg_imm);
14526 %}
14527
14528 // Xor Instructions
14529
14530 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14531 match(Set dst (XorL src1 src2));
14532
14533 format %{ "eor $dst, $src1, $src2\t# int" %}
14534
14535 ins_cost(INSN_COST);
14536 ins_encode %{
14537 __ eor(as_Register($dst$$reg),
14538 as_Register($src1$$reg),
14539 as_Register($src2$$reg));
14540 %}
14541
14542 ins_pipe(ialu_reg_reg);
14543 %}
14544
14545 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14546 match(Set dst (XorL src1 src2));
14547
14548 ins_cost(INSN_COST);
14549 format %{ "eor $dst, $src1, $src2\t# int" %}
14550
14551 ins_encode %{
14552 __ eor(as_Register($dst$$reg),
14553 as_Register($src1$$reg),
14554 (uint64_t)($src2$$constant));
14555 %}
14556
14557 ins_pipe(ialu_reg_imm);
14558 %}
14559
14560 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14561 %{
14562 match(Set dst (ConvI2L src));
14563
14564 ins_cost(INSN_COST);
14565 format %{ "sxtw $dst, $src\t# i2l" %}
14566 ins_encode %{
14567 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14568 %}
14569 ins_pipe(ialu_reg_shift);
14570 %}
14571
14572 // this pattern occurs in bigmath arithmetic
14573 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14574 %{
14575 match(Set dst (AndL (ConvI2L src) mask));
14576
14577 ins_cost(INSN_COST);
14578 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14579 ins_encode %{
14580 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14581 %}
14582
14583 ins_pipe(ialu_reg_shift);
14584 %}
14585
14586 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14587 match(Set dst (ConvL2I src));
14588
14589 ins_cost(INSN_COST);
14590 format %{ "movw $dst, $src \t// l2i" %}
14591
14592 ins_encode %{
14593 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14594 %}
14595
14596 ins_pipe(ialu_reg);
14597 %}
14598
14599 instruct convD2F_reg(vRegF dst, vRegD src) %{
14600 match(Set dst (ConvD2F src));
14601
14602 ins_cost(INSN_COST * 5);
14603 format %{ "fcvtd $dst, $src \t// d2f" %}
14604
14605 ins_encode %{
14606 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14607 %}
14608
14609 ins_pipe(fp_d2f);
14610 %}
14611
14612 instruct convF2D_reg(vRegD dst, vRegF src) %{
14613 match(Set dst (ConvF2D src));
14614
14615 ins_cost(INSN_COST * 5);
14616 format %{ "fcvts $dst, $src \t// f2d" %}
14617
14618 ins_encode %{
14619 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14620 %}
14621
14622 ins_pipe(fp_f2d);
14623 %}
14624
14625 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14626 match(Set dst (ConvF2I src));
14627
14628 ins_cost(INSN_COST * 5);
14629 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14630
14631 ins_encode %{
14632 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14633 %}
14634
14635 ins_pipe(fp_f2i);
14636 %}
14637
14638 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14639 match(Set dst (ConvF2L src));
14640
14641 ins_cost(INSN_COST * 5);
14642 format %{ "fcvtzs $dst, $src \t// f2l" %}
14643
14644 ins_encode %{
14645 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14646 %}
14647
14648 ins_pipe(fp_f2l);
14649 %}
14650
14651 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14652 match(Set dst (ConvF2HF src));
14653 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14654 "smov $dst, $tmp\t# move result from $tmp to $dst"
14655 %}
14656 effect(TEMP tmp);
14657 ins_encode %{
14658 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14659 %}
14660 ins_pipe(pipe_slow);
14661 %}
14662
14663 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14664 match(Set dst (ConvHF2F src));
14665 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14666 "fcvt $dst, $tmp\t# convert half to single precision"
14667 %}
14668 effect(TEMP tmp);
14669 ins_encode %{
14670 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14671 %}
14672 ins_pipe(pipe_slow);
14673 %}
14674
14675 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14676 match(Set dst (ConvI2F src));
14677
14678 ins_cost(INSN_COST * 5);
14679 format %{ "scvtfws $dst, $src \t// i2f" %}
14680
14681 ins_encode %{
14682 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14683 %}
14684
14685 ins_pipe(fp_i2f);
14686 %}
14687
14688 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14689 match(Set dst (ConvL2F src));
14690
14691 ins_cost(INSN_COST * 5);
14692 format %{ "scvtfs $dst, $src \t// l2f" %}
14693
14694 ins_encode %{
14695 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14696 %}
14697
14698 ins_pipe(fp_l2f);
14699 %}
14700
14701 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14702 match(Set dst (ConvD2I src));
14703
14704 ins_cost(INSN_COST * 5);
14705 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14706
14707 ins_encode %{
14708 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14709 %}
14710
14711 ins_pipe(fp_d2i);
14712 %}
14713
14714 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14715 match(Set dst (ConvD2L src));
14716
14717 ins_cost(INSN_COST * 5);
14718 format %{ "fcvtzd $dst, $src \t// d2l" %}
14719
14720 ins_encode %{
14721 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14722 %}
14723
14724 ins_pipe(fp_d2l);
14725 %}
14726
14727 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14728 match(Set dst (ConvI2D src));
14729
14730 ins_cost(INSN_COST * 5);
14731 format %{ "scvtfwd $dst, $src \t// i2d" %}
14732
14733 ins_encode %{
14734 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14735 %}
14736
14737 ins_pipe(fp_i2d);
14738 %}
14739
14740 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14741 match(Set dst (ConvL2D src));
14742
14743 ins_cost(INSN_COST * 5);
14744 format %{ "scvtfd $dst, $src \t// l2d" %}
14745
14746 ins_encode %{
14747 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14748 %}
14749
14750 ins_pipe(fp_l2d);
14751 %}
14752
14753 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14754 %{
14755 match(Set dst (RoundD src));
14756 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14757 format %{ "java_round_double $dst,$src"%}
14758 ins_encode %{
14759 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14760 as_FloatRegister($ftmp$$reg));
14761 %}
14762 ins_pipe(pipe_slow);
14763 %}
14764
14765 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14766 %{
14767 match(Set dst (RoundF src));
14768 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14769 format %{ "java_round_float $dst,$src"%}
14770 ins_encode %{
14771 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14772 as_FloatRegister($ftmp$$reg));
14773 %}
14774 ins_pipe(pipe_slow);
14775 %}
14776
14777 // stack <-> reg and reg <-> reg shuffles with no conversion
14778
14779 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14780
14781 match(Set dst (MoveF2I src));
14782
14783 effect(DEF dst, USE src);
14784
14785 ins_cost(4 * INSN_COST);
14786
14787 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14788
14789 ins_encode %{
14790 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14791 %}
14792
14793 ins_pipe(iload_reg_reg);
14794
14795 %}
14796
14797 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14798
14799 match(Set dst (MoveI2F src));
14800
14801 effect(DEF dst, USE src);
14802
14803 ins_cost(4 * INSN_COST);
14804
14805 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14806
14807 ins_encode %{
14808 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14809 %}
14810
14811 ins_pipe(pipe_class_memory);
14812
14813 %}
14814
14815 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14816
14817 match(Set dst (MoveD2L src));
14818
14819 effect(DEF dst, USE src);
14820
14821 ins_cost(4 * INSN_COST);
14822
14823 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14824
14825 ins_encode %{
14826 __ ldr($dst$$Register, Address(sp, $src$$disp));
14827 %}
14828
14829 ins_pipe(iload_reg_reg);
14830
14831 %}
14832
14833 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14834
14835 match(Set dst (MoveL2D src));
14836
14837 effect(DEF dst, USE src);
14838
14839 ins_cost(4 * INSN_COST);
14840
14841 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14842
14843 ins_encode %{
14844 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14845 %}
14846
14847 ins_pipe(pipe_class_memory);
14848
14849 %}
14850
14851 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14852
14853 match(Set dst (MoveF2I src));
14854
14855 effect(DEF dst, USE src);
14856
14857 ins_cost(INSN_COST);
14858
14859 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14860
14861 ins_encode %{
14862 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14863 %}
14864
14865 ins_pipe(pipe_class_memory);
14866
14867 %}
14868
14869 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14870
14871 match(Set dst (MoveI2F src));
14872
14873 effect(DEF dst, USE src);
14874
14875 ins_cost(INSN_COST);
14876
14877 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14878
14879 ins_encode %{
14880 __ strw($src$$Register, Address(sp, $dst$$disp));
14881 %}
14882
14883 ins_pipe(istore_reg_reg);
14884
14885 %}
14886
14887 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14888
14889 match(Set dst (MoveD2L src));
14890
14891 effect(DEF dst, USE src);
14892
14893 ins_cost(INSN_COST);
14894
14895 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14896
14897 ins_encode %{
14898 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14899 %}
14900
14901 ins_pipe(pipe_class_memory);
14902
14903 %}
14904
14905 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14906
14907 match(Set dst (MoveL2D src));
14908
14909 effect(DEF dst, USE src);
14910
14911 ins_cost(INSN_COST);
14912
14913 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14914
14915 ins_encode %{
14916 __ str($src$$Register, Address(sp, $dst$$disp));
14917 %}
14918
14919 ins_pipe(istore_reg_reg);
14920
14921 %}
14922
14923 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14924
14925 match(Set dst (MoveF2I src));
14926
14927 effect(DEF dst, USE src);
14928
14929 ins_cost(INSN_COST);
14930
14931 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14932
14933 ins_encode %{
14934 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14935 %}
14936
14937 ins_pipe(fp_f2i);
14938
14939 %}
14940
14941 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14942
14943 match(Set dst (MoveI2F src));
14944
14945 effect(DEF dst, USE src);
14946
14947 ins_cost(INSN_COST);
14948
14949 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14950
14951 ins_encode %{
14952 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14953 %}
14954
14955 ins_pipe(fp_i2f);
14956
14957 %}
14958
14959 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14960
14961 match(Set dst (MoveD2L src));
14962
14963 effect(DEF dst, USE src);
14964
14965 ins_cost(INSN_COST);
14966
14967 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14968
14969 ins_encode %{
14970 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14971 %}
14972
14973 ins_pipe(fp_d2l);
14974
14975 %}
14976
14977 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14978
14979 match(Set dst (MoveL2D src));
14980
14981 effect(DEF dst, USE src);
14982
14983 ins_cost(INSN_COST);
14984
14985 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14986
14987 ins_encode %{
14988 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14989 %}
14990
14991 ins_pipe(fp_l2d);
14992
14993 %}
14994
14995 // ============================================================================
14996 // clearing of an array
14997
14998 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14999 %{
15000 match(Set dummy (ClearArray cnt base));
15001 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15002
15003 ins_cost(4 * INSN_COST);
15004 format %{ "ClearArray $cnt, $base" %}
15005
15006 ins_encode %{
15007 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15008 if (tpc == nullptr) {
15009 ciEnv::current()->record_failure("CodeCache is full");
15010 return;
15011 }
15012 %}
15013
15014 ins_pipe(pipe_class_memory);
15015 %}
15016
15017 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15018 %{
15019 predicate((uint64_t)n->in(2)->get_long()
15020 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15021 match(Set dummy (ClearArray cnt base));
15022 effect(TEMP temp, USE_KILL base, KILL cr);
15023
15024 ins_cost(4 * INSN_COST);
15025 format %{ "ClearArray $cnt, $base" %}
15026
15027 ins_encode %{
15028 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15029 if (tpc == nullptr) {
15030 ciEnv::current()->record_failure("CodeCache is full");
15031 return;
15032 }
15033 %}
15034
15035 ins_pipe(pipe_class_memory);
15036 %}
15037
15038 // ============================================================================
15039 // Overflow Math Instructions
15040
15041 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15042 %{
15043 match(Set cr (OverflowAddI op1 op2));
15044
15045 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15046 ins_cost(INSN_COST);
15047 ins_encode %{
15048 __ cmnw($op1$$Register, $op2$$Register);
15049 %}
15050
15051 ins_pipe(icmp_reg_reg);
15052 %}
15053
15054 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15055 %{
15056 match(Set cr (OverflowAddI op1 op2));
15057
15058 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15059 ins_cost(INSN_COST);
15060 ins_encode %{
15061 __ cmnw($op1$$Register, $op2$$constant);
15062 %}
15063
15064 ins_pipe(icmp_reg_imm);
15065 %}
15066
15067 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15068 %{
15069 match(Set cr (OverflowAddL op1 op2));
15070
15071 format %{ "cmn $op1, $op2\t# overflow check long" %}
15072 ins_cost(INSN_COST);
15073 ins_encode %{
15074 __ cmn($op1$$Register, $op2$$Register);
15075 %}
15076
15077 ins_pipe(icmp_reg_reg);
15078 %}
15079
15080 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15081 %{
15082 match(Set cr (OverflowAddL op1 op2));
15083
15084 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15085 ins_cost(INSN_COST);
15086 ins_encode %{
15087 __ adds(zr, $op1$$Register, $op2$$constant);
15088 %}
15089
15090 ins_pipe(icmp_reg_imm);
15091 %}
15092
15093 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15094 %{
15095 match(Set cr (OverflowSubI op1 op2));
15096
15097 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15098 ins_cost(INSN_COST);
15099 ins_encode %{
15100 __ cmpw($op1$$Register, $op2$$Register);
15101 %}
15102
15103 ins_pipe(icmp_reg_reg);
15104 %}
15105
15106 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15107 %{
15108 match(Set cr (OverflowSubI op1 op2));
15109
15110 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15111 ins_cost(INSN_COST);
15112 ins_encode %{
15113 __ cmpw($op1$$Register, $op2$$constant);
15114 %}
15115
15116 ins_pipe(icmp_reg_imm);
15117 %}
15118
15119 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15120 %{
15121 match(Set cr (OverflowSubL op1 op2));
15122
15123 format %{ "cmp $op1, $op2\t# overflow check long" %}
15124 ins_cost(INSN_COST);
15125 ins_encode %{
15126 __ cmp($op1$$Register, $op2$$Register);
15127 %}
15128
15129 ins_pipe(icmp_reg_reg);
15130 %}
15131
15132 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15133 %{
15134 match(Set cr (OverflowSubL op1 op2));
15135
15136 format %{ "cmp $op1, $op2\t# overflow check long" %}
15137 ins_cost(INSN_COST);
15138 ins_encode %{
15139 __ subs(zr, $op1$$Register, $op2$$constant);
15140 %}
15141
15142 ins_pipe(icmp_reg_imm);
15143 %}
15144
15145 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15146 %{
15147 match(Set cr (OverflowSubI zero op1));
15148
15149 format %{ "cmpw zr, $op1\t# overflow check int" %}
15150 ins_cost(INSN_COST);
15151 ins_encode %{
15152 __ cmpw(zr, $op1$$Register);
15153 %}
15154
15155 ins_pipe(icmp_reg_imm);
15156 %}
15157
15158 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15159 %{
15160 match(Set cr (OverflowSubL zero op1));
15161
15162 format %{ "cmp zr, $op1\t# overflow check long" %}
15163 ins_cost(INSN_COST);
15164 ins_encode %{
15165 __ cmp(zr, $op1$$Register);
15166 %}
15167
15168 ins_pipe(icmp_reg_imm);
15169 %}
15170
15171 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15172 %{
15173 match(Set cr (OverflowMulI op1 op2));
15174
15175 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15176 "cmp rscratch1, rscratch1, sxtw\n\t"
15177 "movw rscratch1, #0x80000000\n\t"
15178 "cselw rscratch1, rscratch1, zr, NE\n\t"
15179 "cmpw rscratch1, #1" %}
15180 ins_cost(5 * INSN_COST);
15181 ins_encode %{
15182 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15183 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15184 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15185 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15186 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15187 %}
15188
15189 ins_pipe(pipe_slow);
15190 %}
15191
15192 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15193 %{
15194 match(If cmp (OverflowMulI op1 op2));
15195 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15196 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15197 effect(USE labl, KILL cr);
15198
15199 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15200 "cmp rscratch1, rscratch1, sxtw\n\t"
15201 "b$cmp $labl" %}
15202 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15203 ins_encode %{
15204 Label* L = $labl$$label;
15205 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15206 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15207 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15208 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15209 %}
15210
15211 ins_pipe(pipe_serial);
15212 %}
15213
15214 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15215 %{
15216 match(Set cr (OverflowMulL op1 op2));
15217
15218 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15219 "smulh rscratch2, $op1, $op2\n\t"
15220 "cmp rscratch2, rscratch1, ASR #63\n\t"
15221 "movw rscratch1, #0x80000000\n\t"
15222 "cselw rscratch1, rscratch1, zr, NE\n\t"
15223 "cmpw rscratch1, #1" %}
15224 ins_cost(6 * INSN_COST);
15225 ins_encode %{
15226 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15227 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15228 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15229 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15230 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15231 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15232 %}
15233
15234 ins_pipe(pipe_slow);
15235 %}
15236
15237 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15238 %{
15239 match(If cmp (OverflowMulL op1 op2));
15240 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15241 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15242 effect(USE labl, KILL cr);
15243
15244 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15245 "smulh rscratch2, $op1, $op2\n\t"
15246 "cmp rscratch2, rscratch1, ASR #63\n\t"
15247 "b$cmp $labl" %}
15248 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15249 ins_encode %{
15250 Label* L = $labl$$label;
15251 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15252 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15253 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15254 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15255 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15256 %}
15257
15258 ins_pipe(pipe_serial);
15259 %}
15260
15261 // ============================================================================
15262 // Compare Instructions
15263
15264 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15265 %{
15266 match(Set cr (CmpI op1 op2));
15267
15268 effect(DEF cr, USE op1, USE op2);
15269
15270 ins_cost(INSN_COST);
15271 format %{ "cmpw $op1, $op2" %}
15272
15273 ins_encode(aarch64_enc_cmpw(op1, op2));
15274
15275 ins_pipe(icmp_reg_reg);
15276 %}
15277
15278 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15279 %{
15280 match(Set cr (CmpI op1 zero));
15281
15282 effect(DEF cr, USE op1);
15283
15284 ins_cost(INSN_COST);
15285 format %{ "cmpw $op1, 0" %}
15286
15287 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15288
15289 ins_pipe(icmp_reg_imm);
15290 %}
15291
15292 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15293 %{
15294 match(Set cr (CmpI op1 op2));
15295
15296 effect(DEF cr, USE op1);
15297
15298 ins_cost(INSN_COST);
15299 format %{ "cmpw $op1, $op2" %}
15300
15301 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15302
15303 ins_pipe(icmp_reg_imm);
15304 %}
15305
15306 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15307 %{
15308 match(Set cr (CmpI op1 op2));
15309
15310 effect(DEF cr, USE op1);
15311
15312 ins_cost(INSN_COST * 2);
15313 format %{ "cmpw $op1, $op2" %}
15314
15315 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15316
15317 ins_pipe(icmp_reg_imm);
15318 %}
15319
15320 // Unsigned compare Instructions; really, same as signed compare
15321 // except it should only be used to feed an If or a CMovI which takes a
15322 // cmpOpU.
15323
15324 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15325 %{
15326 match(Set cr (CmpU op1 op2));
15327
15328 effect(DEF cr, USE op1, USE op2);
15329
15330 ins_cost(INSN_COST);
15331 format %{ "cmpw $op1, $op2\t# unsigned" %}
15332
15333 ins_encode(aarch64_enc_cmpw(op1, op2));
15334
15335 ins_pipe(icmp_reg_reg);
15336 %}
15337
15338 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15339 %{
15340 match(Set cr (CmpU op1 zero));
15341
15342 effect(DEF cr, USE op1);
15343
15344 ins_cost(INSN_COST);
15345 format %{ "cmpw $op1, #0\t# unsigned" %}
15346
15347 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15348
15349 ins_pipe(icmp_reg_imm);
15350 %}
15351
15352 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15353 %{
15354 match(Set cr (CmpU op1 op2));
15355
15356 effect(DEF cr, USE op1);
15357
15358 ins_cost(INSN_COST);
15359 format %{ "cmpw $op1, $op2\t# unsigned" %}
15360
15361 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15362
15363 ins_pipe(icmp_reg_imm);
15364 %}
15365
15366 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15367 %{
15368 match(Set cr (CmpU op1 op2));
15369
15370 effect(DEF cr, USE op1);
15371
15372 ins_cost(INSN_COST * 2);
15373 format %{ "cmpw $op1, $op2\t# unsigned" %}
15374
15375 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15376
15377 ins_pipe(icmp_reg_imm);
15378 %}
15379
15380 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15381 %{
15382 match(Set cr (CmpL op1 op2));
15383
15384 effect(DEF cr, USE op1, USE op2);
15385
15386 ins_cost(INSN_COST);
15387 format %{ "cmp $op1, $op2" %}
15388
15389 ins_encode(aarch64_enc_cmp(op1, op2));
15390
15391 ins_pipe(icmp_reg_reg);
15392 %}
15393
15394 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15395 %{
15396 match(Set cr (CmpL op1 zero));
15397
15398 effect(DEF cr, USE op1);
15399
15400 ins_cost(INSN_COST);
15401 format %{ "tst $op1" %}
15402
15403 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15404
15405 ins_pipe(icmp_reg_imm);
15406 %}
15407
15408 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15409 %{
15410 match(Set cr (CmpL op1 op2));
15411
15412 effect(DEF cr, USE op1);
15413
15414 ins_cost(INSN_COST);
15415 format %{ "cmp $op1, $op2" %}
15416
15417 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15418
15419 ins_pipe(icmp_reg_imm);
15420 %}
15421
15422 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15423 %{
15424 match(Set cr (CmpL op1 op2));
15425
15426 effect(DEF cr, USE op1);
15427
15428 ins_cost(INSN_COST * 2);
15429 format %{ "cmp $op1, $op2" %}
15430
15431 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15432
15433 ins_pipe(icmp_reg_imm);
15434 %}
15435
15436 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15437 %{
15438 match(Set cr (CmpUL op1 op2));
15439
15440 effect(DEF cr, USE op1, USE op2);
15441
15442 ins_cost(INSN_COST);
15443 format %{ "cmp $op1, $op2" %}
15444
15445 ins_encode(aarch64_enc_cmp(op1, op2));
15446
15447 ins_pipe(icmp_reg_reg);
15448 %}
15449
15450 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15451 %{
15452 match(Set cr (CmpUL op1 zero));
15453
15454 effect(DEF cr, USE op1);
15455
15456 ins_cost(INSN_COST);
15457 format %{ "tst $op1" %}
15458
15459 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15460
15461 ins_pipe(icmp_reg_imm);
15462 %}
15463
15464 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15465 %{
15466 match(Set cr (CmpUL op1 op2));
15467
15468 effect(DEF cr, USE op1);
15469
15470 ins_cost(INSN_COST);
15471 format %{ "cmp $op1, $op2" %}
15472
15473 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15474
15475 ins_pipe(icmp_reg_imm);
15476 %}
15477
15478 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15479 %{
15480 match(Set cr (CmpUL op1 op2));
15481
15482 effect(DEF cr, USE op1);
15483
15484 ins_cost(INSN_COST * 2);
15485 format %{ "cmp $op1, $op2" %}
15486
15487 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15488
15489 ins_pipe(icmp_reg_imm);
15490 %}
15491
15492 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15493 %{
15494 match(Set cr (CmpP op1 op2));
15495
15496 effect(DEF cr, USE op1, USE op2);
15497
15498 ins_cost(INSN_COST);
15499 format %{ "cmp $op1, $op2\t // ptr" %}
15500
15501 ins_encode(aarch64_enc_cmpp(op1, op2));
15502
15503 ins_pipe(icmp_reg_reg);
15504 %}
15505
15506 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15507 %{
15508 match(Set cr (CmpN op1 op2));
15509
15510 effect(DEF cr, USE op1, USE op2);
15511
15512 ins_cost(INSN_COST);
15513 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15514
15515 ins_encode(aarch64_enc_cmpn(op1, op2));
15516
15517 ins_pipe(icmp_reg_reg);
15518 %}
15519
15520 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15521 %{
15522 match(Set cr (CmpP op1 zero));
15523
15524 effect(DEF cr, USE op1, USE zero);
15525
15526 ins_cost(INSN_COST);
15527 format %{ "cmp $op1, 0\t // ptr" %}
15528
15529 ins_encode(aarch64_enc_testp(op1));
15530
15531 ins_pipe(icmp_reg_imm);
15532 %}
15533
15534 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15535 %{
15536 match(Set cr (CmpN op1 zero));
15537
15538 effect(DEF cr, USE op1, USE zero);
15539
15540 ins_cost(INSN_COST);
15541 format %{ "cmp $op1, 0\t // compressed ptr" %}
15542
15543 ins_encode(aarch64_enc_testn(op1));
15544
15545 ins_pipe(icmp_reg_imm);
15546 %}
15547
15548 // FP comparisons
15549 //
15550 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15551 // using normal cmpOp. See declaration of rFlagsReg for details.
15552
15553 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15554 %{
15555 match(Set cr (CmpF src1 src2));
15556
15557 ins_cost(3 * INSN_COST);
15558 format %{ "fcmps $src1, $src2" %}
15559
15560 ins_encode %{
15561 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15562 %}
15563
15564 ins_pipe(pipe_class_compare);
15565 %}
15566
15567 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15568 %{
15569 match(Set cr (CmpF src1 src2));
15570
15571 ins_cost(3 * INSN_COST);
15572 format %{ "fcmps $src1, 0.0" %}
15573
15574 ins_encode %{
15575 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15576 %}
15577
15578 ins_pipe(pipe_class_compare);
15579 %}
15580 // FROM HERE
15581
15582 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15583 %{
15584 match(Set cr (CmpD src1 src2));
15585
15586 ins_cost(3 * INSN_COST);
15587 format %{ "fcmpd $src1, $src2" %}
15588
15589 ins_encode %{
15590 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15591 %}
15592
15593 ins_pipe(pipe_class_compare);
15594 %}
15595
15596 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15597 %{
15598 match(Set cr (CmpD src1 src2));
15599
15600 ins_cost(3 * INSN_COST);
15601 format %{ "fcmpd $src1, 0.0" %}
15602
15603 ins_encode %{
15604 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15605 %}
15606
15607 ins_pipe(pipe_class_compare);
15608 %}
15609
15610 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15611 %{
15612 match(Set dst (CmpF3 src1 src2));
15613 effect(KILL cr);
15614
15615 ins_cost(5 * INSN_COST);
15616 format %{ "fcmps $src1, $src2\n\t"
15617 "csinvw($dst, zr, zr, eq\n\t"
15618 "csnegw($dst, $dst, $dst, lt)"
15619 %}
15620
15621 ins_encode %{
15622 Label done;
15623 FloatRegister s1 = as_FloatRegister($src1$$reg);
15624 FloatRegister s2 = as_FloatRegister($src2$$reg);
15625 Register d = as_Register($dst$$reg);
15626 __ fcmps(s1, s2);
15627 // installs 0 if EQ else -1
15628 __ csinvw(d, zr, zr, Assembler::EQ);
15629 // keeps -1 if less or unordered else installs 1
15630 __ csnegw(d, d, d, Assembler::LT);
15631 __ bind(done);
15632 %}
15633
15634 ins_pipe(pipe_class_default);
15635
15636 %}
15637
15638 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15639 %{
15640 match(Set dst (CmpD3 src1 src2));
15641 effect(KILL cr);
15642
15643 ins_cost(5 * INSN_COST);
15644 format %{ "fcmpd $src1, $src2\n\t"
15645 "csinvw($dst, zr, zr, eq\n\t"
15646 "csnegw($dst, $dst, $dst, lt)"
15647 %}
15648
15649 ins_encode %{
15650 Label done;
15651 FloatRegister s1 = as_FloatRegister($src1$$reg);
15652 FloatRegister s2 = as_FloatRegister($src2$$reg);
15653 Register d = as_Register($dst$$reg);
15654 __ fcmpd(s1, s2);
15655 // installs 0 if EQ else -1
15656 __ csinvw(d, zr, zr, Assembler::EQ);
15657 // keeps -1 if less or unordered else installs 1
15658 __ csnegw(d, d, d, Assembler::LT);
15659 __ bind(done);
15660 %}
15661 ins_pipe(pipe_class_default);
15662
15663 %}
15664
15665 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15666 %{
15667 match(Set dst (CmpF3 src1 zero));
15668 effect(KILL cr);
15669
15670 ins_cost(5 * INSN_COST);
15671 format %{ "fcmps $src1, 0.0\n\t"
15672 "csinvw($dst, zr, zr, eq\n\t"
15673 "csnegw($dst, $dst, $dst, lt)"
15674 %}
15675
15676 ins_encode %{
15677 Label done;
15678 FloatRegister s1 = as_FloatRegister($src1$$reg);
15679 Register d = as_Register($dst$$reg);
15680 __ fcmps(s1, 0.0);
15681 // installs 0 if EQ else -1
15682 __ csinvw(d, zr, zr, Assembler::EQ);
15683 // keeps -1 if less or unordered else installs 1
15684 __ csnegw(d, d, d, Assembler::LT);
15685 __ bind(done);
15686 %}
15687
15688 ins_pipe(pipe_class_default);
15689
15690 %}
15691
15692 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15693 %{
15694 match(Set dst (CmpD3 src1 zero));
15695 effect(KILL cr);
15696
15697 ins_cost(5 * INSN_COST);
15698 format %{ "fcmpd $src1, 0.0\n\t"
15699 "csinvw($dst, zr, zr, eq\n\t"
15700 "csnegw($dst, $dst, $dst, lt)"
15701 %}
15702
15703 ins_encode %{
15704 Label done;
15705 FloatRegister s1 = as_FloatRegister($src1$$reg);
15706 Register d = as_Register($dst$$reg);
15707 __ fcmpd(s1, 0.0);
15708 // installs 0 if EQ else -1
15709 __ csinvw(d, zr, zr, Assembler::EQ);
15710 // keeps -1 if less or unordered else installs 1
15711 __ csnegw(d, d, d, Assembler::LT);
15712 __ bind(done);
15713 %}
15714 ins_pipe(pipe_class_default);
15715
15716 %}
15717
15718 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15719 %{
15720 match(Set dst (CmpLTMask p q));
15721 effect(KILL cr);
15722
15723 ins_cost(3 * INSN_COST);
15724
15725 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15726 "csetw $dst, lt\n\t"
15727 "subw $dst, zr, $dst"
15728 %}
15729
15730 ins_encode %{
15731 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15732 __ csetw(as_Register($dst$$reg), Assembler::LT);
15733 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15734 %}
15735
15736 ins_pipe(ialu_reg_reg);
15737 %}
15738
15739 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15740 %{
15741 match(Set dst (CmpLTMask src zero));
15742 effect(KILL cr);
15743
15744 ins_cost(INSN_COST);
15745
15746 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15747
15748 ins_encode %{
15749 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15750 %}
15751
15752 ins_pipe(ialu_reg_shift);
15753 %}
15754
15755 // ============================================================================
15756 // Max and Min
15757
15758 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15759
15760 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15761 %{
15762 effect(DEF cr, USE src);
15763 ins_cost(INSN_COST);
15764 format %{ "cmpw $src, 0" %}
15765
15766 ins_encode %{
15767 __ cmpw($src$$Register, 0);
15768 %}
15769 ins_pipe(icmp_reg_imm);
15770 %}
15771
15772 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15773 %{
15774 match(Set dst (MinI src1 src2));
15775 ins_cost(INSN_COST * 3);
15776
15777 expand %{
15778 rFlagsReg cr;
15779 compI_reg_reg(cr, src1, src2);
15780 cmovI_reg_reg_lt(dst, src1, src2, cr);
15781 %}
15782 %}
15783
15784 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15785 %{
15786 match(Set dst (MaxI src1 src2));
15787 ins_cost(INSN_COST * 3);
15788
15789 expand %{
15790 rFlagsReg cr;
15791 compI_reg_reg(cr, src1, src2);
15792 cmovI_reg_reg_gt(dst, src1, src2, cr);
15793 %}
15794 %}
15795
15796
15797 // ============================================================================
15798 // Branch Instructions
15799
15800 // Direct Branch.
15801 instruct branch(label lbl)
15802 %{
15803 match(Goto);
15804
15805 effect(USE lbl);
15806
15807 ins_cost(BRANCH_COST);
15808 format %{ "b $lbl" %}
15809
15810 ins_encode(aarch64_enc_b(lbl));
15811
15812 ins_pipe(pipe_branch);
15813 %}
15814
15815 // Conditional Near Branch
15816 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15817 %{
15818 // Same match rule as `branchConFar'.
15819 match(If cmp cr);
15820
15821 effect(USE lbl);
15822
15823 ins_cost(BRANCH_COST);
15824 // If set to 1 this indicates that the current instruction is a
15825 // short variant of a long branch. This avoids using this
15826 // instruction in first-pass matching. It will then only be used in
15827 // the `Shorten_branches' pass.
15828 // ins_short_branch(1);
15829 format %{ "b$cmp $lbl" %}
15830
15831 ins_encode(aarch64_enc_br_con(cmp, lbl));
15832
15833 ins_pipe(pipe_branch_cond);
15834 %}
15835
15836 // Conditional Near Branch Unsigned
15837 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15838 %{
15839 // Same match rule as `branchConFar'.
15840 match(If cmp cr);
15841
15842 effect(USE lbl);
15843
15844 ins_cost(BRANCH_COST);
15845 // If set to 1 this indicates that the current instruction is a
15846 // short variant of a long branch. This avoids using this
15847 // instruction in first-pass matching. It will then only be used in
15848 // the `Shorten_branches' pass.
15849 // ins_short_branch(1);
15850 format %{ "b$cmp $lbl\t# unsigned" %}
15851
15852 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15853
15854 ins_pipe(pipe_branch_cond);
15855 %}
15856
15857 // Make use of CBZ and CBNZ. These instructions, as well as being
15858 // shorter than (cmp; branch), have the additional benefit of not
15859 // killing the flags.
15860
15861 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15862 match(If cmp (CmpI op1 op2));
15863 effect(USE labl);
15864
15865 ins_cost(BRANCH_COST);
15866 format %{ "cbw$cmp $op1, $labl" %}
15867 ins_encode %{
15868 Label* L = $labl$$label;
15869 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15870 if (cond == Assembler::EQ)
15871 __ cbzw($op1$$Register, *L);
15872 else
15873 __ cbnzw($op1$$Register, *L);
15874 %}
15875 ins_pipe(pipe_cmp_branch);
15876 %}
15877
15878 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15879 match(If cmp (CmpL op1 op2));
15880 effect(USE labl);
15881
15882 ins_cost(BRANCH_COST);
15883 format %{ "cb$cmp $op1, $labl" %}
15884 ins_encode %{
15885 Label* L = $labl$$label;
15886 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15887 if (cond == Assembler::EQ)
15888 __ cbz($op1$$Register, *L);
15889 else
15890 __ cbnz($op1$$Register, *L);
15891 %}
15892 ins_pipe(pipe_cmp_branch);
15893 %}
15894
15895 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15896 match(If cmp (CmpP op1 op2));
15897 effect(USE labl);
15898
15899 ins_cost(BRANCH_COST);
15900 format %{ "cb$cmp $op1, $labl" %}
15901 ins_encode %{
15902 Label* L = $labl$$label;
15903 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15904 if (cond == Assembler::EQ)
15905 __ cbz($op1$$Register, *L);
15906 else
15907 __ cbnz($op1$$Register, *L);
15908 %}
15909 ins_pipe(pipe_cmp_branch);
15910 %}
15911
15912 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15913 match(If cmp (CmpN op1 op2));
15914 effect(USE labl);
15915
15916 ins_cost(BRANCH_COST);
15917 format %{ "cbw$cmp $op1, $labl" %}
15918 ins_encode %{
15919 Label* L = $labl$$label;
15920 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15921 if (cond == Assembler::EQ)
15922 __ cbzw($op1$$Register, *L);
15923 else
15924 __ cbnzw($op1$$Register, *L);
15925 %}
15926 ins_pipe(pipe_cmp_branch);
15927 %}
15928
15929 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15930 match(If cmp (CmpP (DecodeN oop) zero));
15931 effect(USE labl);
15932
15933 ins_cost(BRANCH_COST);
15934 format %{ "cb$cmp $oop, $labl" %}
15935 ins_encode %{
15936 Label* L = $labl$$label;
15937 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15938 if (cond == Assembler::EQ)
15939 __ cbzw($oop$$Register, *L);
15940 else
15941 __ cbnzw($oop$$Register, *L);
15942 %}
15943 ins_pipe(pipe_cmp_branch);
15944 %}
15945
15946 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15947 match(If cmp (CmpU op1 op2));
15948 effect(USE labl);
15949
15950 ins_cost(BRANCH_COST);
15951 format %{ "cbw$cmp $op1, $labl" %}
15952 ins_encode %{
15953 Label* L = $labl$$label;
15954 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15955 if (cond == Assembler::EQ || cond == Assembler::LS) {
15956 __ cbzw($op1$$Register, *L);
15957 } else {
15958 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15959 __ cbnzw($op1$$Register, *L);
15960 }
15961 %}
15962 ins_pipe(pipe_cmp_branch);
15963 %}
15964
15965 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15966 match(If cmp (CmpUL op1 op2));
15967 effect(USE labl);
15968
15969 ins_cost(BRANCH_COST);
15970 format %{ "cb$cmp $op1, $labl" %}
15971 ins_encode %{
15972 Label* L = $labl$$label;
15973 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15974 if (cond == Assembler::EQ || cond == Assembler::LS) {
15975 __ cbz($op1$$Register, *L);
15976 } else {
15977 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15978 __ cbnz($op1$$Register, *L);
15979 }
15980 %}
15981 ins_pipe(pipe_cmp_branch);
15982 %}
15983
15984 // Test bit and Branch
15985
15986 // Patterns for short (< 32KiB) variants
15987 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15988 match(If cmp (CmpL op1 op2));
15989 effect(USE labl);
15990
15991 ins_cost(BRANCH_COST);
15992 format %{ "cb$cmp $op1, $labl # long" %}
15993 ins_encode %{
15994 Label* L = $labl$$label;
15995 Assembler::Condition cond =
15996 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15997 __ tbr(cond, $op1$$Register, 63, *L);
15998 %}
15999 ins_pipe(pipe_cmp_branch);
16000 ins_short_branch(1);
16001 %}
16002
16003 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16004 match(If cmp (CmpI op1 op2));
16005 effect(USE labl);
16006
16007 ins_cost(BRANCH_COST);
16008 format %{ "cb$cmp $op1, $labl # int" %}
16009 ins_encode %{
16010 Label* L = $labl$$label;
16011 Assembler::Condition cond =
16012 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16013 __ tbr(cond, $op1$$Register, 31, *L);
16014 %}
16015 ins_pipe(pipe_cmp_branch);
16016 ins_short_branch(1);
16017 %}
16018
16019 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16020 match(If cmp (CmpL (AndL op1 op2) op3));
16021 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16022 effect(USE labl);
16023
16024 ins_cost(BRANCH_COST);
16025 format %{ "tb$cmp $op1, $op2, $labl" %}
16026 ins_encode %{
16027 Label* L = $labl$$label;
16028 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16029 int bit = exact_log2_long($op2$$constant);
16030 __ tbr(cond, $op1$$Register, bit, *L);
16031 %}
16032 ins_pipe(pipe_cmp_branch);
16033 ins_short_branch(1);
16034 %}
16035
16036 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16037 match(If cmp (CmpI (AndI op1 op2) op3));
16038 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16039 effect(USE labl);
16040
16041 ins_cost(BRANCH_COST);
16042 format %{ "tb$cmp $op1, $op2, $labl" %}
16043 ins_encode %{
16044 Label* L = $labl$$label;
16045 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16046 int bit = exact_log2((juint)$op2$$constant);
16047 __ tbr(cond, $op1$$Register, bit, *L);
16048 %}
16049 ins_pipe(pipe_cmp_branch);
16050 ins_short_branch(1);
16051 %}
16052
16053 // And far variants
16054 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16055 match(If cmp (CmpL op1 op2));
16056 effect(USE labl);
16057
16058 ins_cost(BRANCH_COST);
16059 format %{ "cb$cmp $op1, $labl # long" %}
16060 ins_encode %{
16061 Label* L = $labl$$label;
16062 Assembler::Condition cond =
16063 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16064 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16065 %}
16066 ins_pipe(pipe_cmp_branch);
16067 %}
16068
16069 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16070 match(If cmp (CmpI op1 op2));
16071 effect(USE labl);
16072
16073 ins_cost(BRANCH_COST);
16074 format %{ "cb$cmp $op1, $labl # int" %}
16075 ins_encode %{
16076 Label* L = $labl$$label;
16077 Assembler::Condition cond =
16078 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16079 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16080 %}
16081 ins_pipe(pipe_cmp_branch);
16082 %}
16083
16084 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16085 match(If cmp (CmpL (AndL op1 op2) op3));
16086 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16087 effect(USE labl);
16088
16089 ins_cost(BRANCH_COST);
16090 format %{ "tb$cmp $op1, $op2, $labl" %}
16091 ins_encode %{
16092 Label* L = $labl$$label;
16093 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16094 int bit = exact_log2_long($op2$$constant);
16095 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16096 %}
16097 ins_pipe(pipe_cmp_branch);
16098 %}
16099
16100 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16101 match(If cmp (CmpI (AndI op1 op2) op3));
16102 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16103 effect(USE labl);
16104
16105 ins_cost(BRANCH_COST);
16106 format %{ "tb$cmp $op1, $op2, $labl" %}
16107 ins_encode %{
16108 Label* L = $labl$$label;
16109 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16110 int bit = exact_log2((juint)$op2$$constant);
16111 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16112 %}
16113 ins_pipe(pipe_cmp_branch);
16114 %}
16115
16116 // Test bits
16117
16118 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16119 match(Set cr (CmpL (AndL op1 op2) op3));
16120 predicate(Assembler::operand_valid_for_logical_immediate
16121 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16122
16123 ins_cost(INSN_COST);
16124 format %{ "tst $op1, $op2 # long" %}
16125 ins_encode %{
16126 __ tst($op1$$Register, $op2$$constant);
16127 %}
16128 ins_pipe(ialu_reg_reg);
16129 %}
16130
16131 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16132 match(Set cr (CmpI (AndI op1 op2) op3));
16133 predicate(Assembler::operand_valid_for_logical_immediate
16134 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16135
16136 ins_cost(INSN_COST);
16137 format %{ "tst $op1, $op2 # int" %}
16138 ins_encode %{
16139 __ tstw($op1$$Register, $op2$$constant);
16140 %}
16141 ins_pipe(ialu_reg_reg);
16142 %}
16143
16144 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16145 match(Set cr (CmpL (AndL op1 op2) op3));
16146
16147 ins_cost(INSN_COST);
16148 format %{ "tst $op1, $op2 # long" %}
16149 ins_encode %{
16150 __ tst($op1$$Register, $op2$$Register);
16151 %}
16152 ins_pipe(ialu_reg_reg);
16153 %}
16154
16155 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16156 match(Set cr (CmpI (AndI op1 op2) op3));
16157
16158 ins_cost(INSN_COST);
16159 format %{ "tstw $op1, $op2 # int" %}
16160 ins_encode %{
16161 __ tstw($op1$$Register, $op2$$Register);
16162 %}
16163 ins_pipe(ialu_reg_reg);
16164 %}
16165
16166
16167 // Conditional Far Branch
16168 // Conditional Far Branch Unsigned
16169 // TODO: fixme
16170
16171 // counted loop end branch near
16172 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16173 %{
16174 match(CountedLoopEnd cmp cr);
16175
16176 effect(USE lbl);
16177
16178 ins_cost(BRANCH_COST);
16179 // short variant.
16180 // ins_short_branch(1);
16181 format %{ "b$cmp $lbl \t// counted loop end" %}
16182
16183 ins_encode(aarch64_enc_br_con(cmp, lbl));
16184
16185 ins_pipe(pipe_branch);
16186 %}
16187
16188 // counted loop end branch far
16189 // TODO: fixme
16190
16191 // ============================================================================
16192 // inlined locking and unlocking
16193
16194 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16195 %{
16196 predicate(LockingMode != LM_LIGHTWEIGHT);
16197 match(Set cr (FastLock object box));
16198 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16199
16200 ins_cost(5 * INSN_COST);
16201 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16202
16203 ins_encode %{
16204 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16205 %}
16206
16207 ins_pipe(pipe_serial);
16208 %}
16209
16210 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16211 %{
16212 predicate(LockingMode != LM_LIGHTWEIGHT);
16213 match(Set cr (FastUnlock object box));
16214 effect(TEMP tmp, TEMP tmp2);
16215
16216 ins_cost(5 * INSN_COST);
16217 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16218
16219 ins_encode %{
16220 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16221 %}
16222
16223 ins_pipe(pipe_serial);
16224 %}
16225
16226 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16227 %{
16228 predicate(LockingMode == LM_LIGHTWEIGHT);
16229 match(Set cr (FastLock object box));
16230 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16231
16232 ins_cost(5 * INSN_COST);
16233 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16234
16235 ins_encode %{
16236 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16237 %}
16238
16239 ins_pipe(pipe_serial);
16240 %}
16241
16242 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16243 %{
16244 predicate(LockingMode == LM_LIGHTWEIGHT);
16245 match(Set cr (FastUnlock object box));
16246 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16247
16248 ins_cost(5 * INSN_COST);
16249 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16250
16251 ins_encode %{
16252 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16253 %}
16254
16255 ins_pipe(pipe_serial);
16256 %}
16257
16258 // ============================================================================
16259 // Safepoint Instructions
16260
16261 // TODO
16262 // provide a near and far version of this code
16263
16264 instruct safePoint(rFlagsReg cr, iRegP poll)
16265 %{
16266 match(SafePoint poll);
16267 effect(KILL cr);
16268
16269 format %{
16270 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16271 %}
16272 ins_encode %{
16273 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16274 %}
16275 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16276 %}
16277
16278
16279 // ============================================================================
16280 // Procedure Call/Return Instructions
16281
16282 // Call Java Static Instruction
16283
16284 instruct CallStaticJavaDirect(method meth)
16285 %{
16286 match(CallStaticJava);
16287
16288 effect(USE meth);
16289
16290 ins_cost(CALL_COST);
16291
16292 format %{ "call,static $meth \t// ==> " %}
16293
16294 ins_encode(aarch64_enc_java_static_call(meth),
16295 aarch64_enc_call_epilog);
16296
16297 ins_pipe(pipe_class_call);
16298 %}
16299
16300 // TO HERE
16301
16302 // Call Java Dynamic Instruction
16303 instruct CallDynamicJavaDirect(method meth)
16304 %{
16305 match(CallDynamicJava);
16306
16307 effect(USE meth);
16308
16309 ins_cost(CALL_COST);
16310
16311 format %{ "CALL,dynamic $meth \t// ==> " %}
16312
16313 ins_encode(aarch64_enc_java_dynamic_call(meth),
16314 aarch64_enc_call_epilog);
16315
16316 ins_pipe(pipe_class_call);
16317 %}
16318
16319 // Call Runtime Instruction
16320
16321 instruct CallRuntimeDirect(method meth)
16322 %{
16323 match(CallRuntime);
16324
16325 effect(USE meth);
16326
16327 ins_cost(CALL_COST);
16328
16329 format %{ "CALL, runtime $meth" %}
16330
16331 ins_encode( aarch64_enc_java_to_runtime(meth) );
16332
16333 ins_pipe(pipe_class_call);
16334 %}
16335
16336 // Call Runtime Instruction
16337
16338 instruct CallLeafDirect(method meth)
16339 %{
16340 match(CallLeaf);
16341
16342 effect(USE meth);
16343
16344 ins_cost(CALL_COST);
16345
16346 format %{ "CALL, runtime leaf $meth" %}
16347
16348 ins_encode( aarch64_enc_java_to_runtime(meth) );
16349
16350 ins_pipe(pipe_class_call);
16351 %}
16352
16353 // Call Runtime Instruction without safepoint and with vector arguments
16354 instruct CallLeafDirectVector(method meth)
16355 %{
16356 match(CallLeafVector);
16357
16358 effect(USE meth);
16359
16360 ins_cost(CALL_COST);
16361
16362 format %{ "CALL, runtime leaf vector $meth" %}
16363
16364 ins_encode(aarch64_enc_java_to_runtime(meth));
16365
16366 ins_pipe(pipe_class_call);
16367 %}
16368
16369 // Call Runtime Instruction
16370
16371 instruct CallLeafNoFPDirect(method meth)
16372 %{
16373 match(CallLeafNoFP);
16374
16375 effect(USE meth);
16376
16377 ins_cost(CALL_COST);
16378
16379 format %{ "CALL, runtime leaf nofp $meth" %}
16380
16381 ins_encode( aarch64_enc_java_to_runtime(meth) );
16382
16383 ins_pipe(pipe_class_call);
16384 %}
16385
16386 // Tail Call; Jump from runtime stub to Java code.
16387 // Also known as an 'interprocedural jump'.
16388 // Target of jump will eventually return to caller.
16389 // TailJump below removes the return address.
16390 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16391 // emitted just above the TailCall which has reset rfp to the caller state.
16392 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16393 %{
16394 match(TailCall jump_target method_ptr);
16395
16396 ins_cost(CALL_COST);
16397
16398 format %{ "br $jump_target\t# $method_ptr holds method" %}
16399
16400 ins_encode(aarch64_enc_tail_call(jump_target));
16401
16402 ins_pipe(pipe_class_call);
16403 %}
16404
16405 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16406 %{
16407 match(TailJump jump_target ex_oop);
16408
16409 ins_cost(CALL_COST);
16410
16411 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16412
16413 ins_encode(aarch64_enc_tail_jmp(jump_target));
16414
16415 ins_pipe(pipe_class_call);
16416 %}
16417
16418 // Forward exception.
16419 instruct ForwardExceptionjmp()
16420 %{
16421 match(ForwardException);
16422 ins_cost(CALL_COST);
16423
16424 format %{ "b forward_exception_stub" %}
16425 ins_encode %{
16426 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16427 %}
16428 ins_pipe(pipe_class_call);
16429 %}
16430
16431 // Create exception oop: created by stack-crawling runtime code.
16432 // Created exception is now available to this handler, and is setup
16433 // just prior to jumping to this handler. No code emitted.
16434 // TODO check
16435 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16436 instruct CreateException(iRegP_R0 ex_oop)
16437 %{
16438 match(Set ex_oop (CreateEx));
16439
16440 format %{ " -- \t// exception oop; no code emitted" %}
16441
16442 size(0);
16443
16444 ins_encode( /*empty*/ );
16445
16446 ins_pipe(pipe_class_empty);
16447 %}
16448
16449 // Rethrow exception: The exception oop will come in the first
16450 // argument position. Then JUMP (not call) to the rethrow stub code.
16451 instruct RethrowException() %{
16452 match(Rethrow);
16453 ins_cost(CALL_COST);
16454
16455 format %{ "b rethrow_stub" %}
16456
16457 ins_encode( aarch64_enc_rethrow() );
16458
16459 ins_pipe(pipe_class_call);
16460 %}
16461
16462
16463 // Return Instruction
16464 // epilog node loads ret address into lr as part of frame pop
16465 instruct Ret()
16466 %{
16467 match(Return);
16468
16469 format %{ "ret\t// return register" %}
16470
16471 ins_encode( aarch64_enc_ret() );
16472
16473 ins_pipe(pipe_branch);
16474 %}
16475
16476 // Die now.
16477 instruct ShouldNotReachHere() %{
16478 match(Halt);
16479
16480 ins_cost(CALL_COST);
16481 format %{ "ShouldNotReachHere" %}
16482
16483 ins_encode %{
16484 if (is_reachable()) {
16485 const char* str = __ code_string(_halt_reason);
16486 __ stop(str);
16487 }
16488 %}
16489
16490 ins_pipe(pipe_class_default);
16491 %}
16492
16493 // ============================================================================
16494 // Partial Subtype Check
16495 //
16496 // superklass array for an instance of the superklass. Set a hidden
16497 // internal cache on a hit (cache is checked with exposed code in
16498 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16499 // encoding ALSO sets flags.
16500
16501 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16502 %{
16503 match(Set result (PartialSubtypeCheck sub super));
16504 predicate(!UseSecondarySupersTable);
16505 effect(KILL cr, KILL temp);
16506
16507 ins_cost(20 * INSN_COST); // slightly larger than the next version
16508 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16509
16510 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16511
16512 opcode(0x1); // Force zero of result reg on hit
16513
16514 ins_pipe(pipe_class_memory);
16515 %}
16516
16517 // Two versions of partialSubtypeCheck, both used when we need to
16518 // search for a super class in the secondary supers array. The first
16519 // is used when we don't know _a priori_ the class being searched
16520 // for. The second, far more common, is used when we do know: this is
16521 // used for instanceof, checkcast, and any case where C2 can determine
16522 // it by constant propagation.
16523
16524 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16525 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16526 rFlagsReg cr)
16527 %{
16528 match(Set result (PartialSubtypeCheck sub super));
16529 predicate(UseSecondarySupersTable);
16530 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16531
16532 ins_cost(10 * INSN_COST); // slightly larger than the next version
16533 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16534
16535 ins_encode %{
16536 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16537 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16538 $vtemp$$FloatRegister,
16539 $result$$Register, /*L_success*/nullptr);
16540 %}
16541
16542 ins_pipe(pipe_class_memory);
16543 %}
16544
16545 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16546 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16547 rFlagsReg cr)
16548 %{
16549 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16550 predicate(UseSecondarySupersTable);
16551 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16552
16553 ins_cost(5 * INSN_COST); // smaller than the next version
16554 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16555
16556 ins_encode %{
16557 bool success = false;
16558 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16559 if (InlineSecondarySupersTest) {
16560 success =
16561 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16562 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16563 $vtemp$$FloatRegister,
16564 $result$$Register,
16565 super_klass_slot);
16566 } else {
16567 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16568 success = (call != nullptr);
16569 }
16570 if (!success) {
16571 ciEnv::current()->record_failure("CodeCache is full");
16572 return;
16573 }
16574 %}
16575
16576 ins_pipe(pipe_class_memory);
16577 %}
16578
16579 // Intrisics for String.compareTo()
16580
16581 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16582 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16583 %{
16584 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16585 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16586 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16587
16588 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16589 ins_encode %{
16590 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16591 __ string_compare($str1$$Register, $str2$$Register,
16592 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16593 $tmp1$$Register, $tmp2$$Register,
16594 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16595 %}
16596 ins_pipe(pipe_class_memory);
16597 %}
16598
16599 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16600 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16601 %{
16602 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16603 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16604 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16605
16606 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16607 ins_encode %{
16608 __ string_compare($str1$$Register, $str2$$Register,
16609 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16610 $tmp1$$Register, $tmp2$$Register,
16611 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16612 %}
16613 ins_pipe(pipe_class_memory);
16614 %}
16615
16616 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16617 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16618 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16619 %{
16620 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16621 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16622 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16623 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16624
16625 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16626 ins_encode %{
16627 __ string_compare($str1$$Register, $str2$$Register,
16628 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16629 $tmp1$$Register, $tmp2$$Register,
16630 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16631 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16632 %}
16633 ins_pipe(pipe_class_memory);
16634 %}
16635
16636 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16637 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16638 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16639 %{
16640 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16641 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16642 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16643 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16644
16645 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16646 ins_encode %{
16647 __ string_compare($str1$$Register, $str2$$Register,
16648 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16649 $tmp1$$Register, $tmp2$$Register,
16650 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16651 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16652 %}
16653 ins_pipe(pipe_class_memory);
16654 %}
16655
16656 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16657 // these string_compare variants as NEON register type for convenience so that the prototype of
16658 // string_compare can be shared with all variants.
16659
16660 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16661 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16662 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16663 pRegGov_P1 pgtmp2, rFlagsReg cr)
16664 %{
16665 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16666 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16667 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16668 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16669
16670 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16671 ins_encode %{
16672 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16673 __ string_compare($str1$$Register, $str2$$Register,
16674 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16675 $tmp1$$Register, $tmp2$$Register,
16676 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16677 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16678 StrIntrinsicNode::LL);
16679 %}
16680 ins_pipe(pipe_class_memory);
16681 %}
16682
16683 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16684 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16685 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16686 pRegGov_P1 pgtmp2, rFlagsReg cr)
16687 %{
16688 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16689 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16690 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16691 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16692
16693 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16694 ins_encode %{
16695 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16696 __ string_compare($str1$$Register, $str2$$Register,
16697 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16698 $tmp1$$Register, $tmp2$$Register,
16699 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16700 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16701 StrIntrinsicNode::LU);
16702 %}
16703 ins_pipe(pipe_class_memory);
16704 %}
16705
16706 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16707 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16708 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16709 pRegGov_P1 pgtmp2, rFlagsReg cr)
16710 %{
16711 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16712 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16713 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16714 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16715
16716 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16717 ins_encode %{
16718 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16719 __ string_compare($str1$$Register, $str2$$Register,
16720 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16721 $tmp1$$Register, $tmp2$$Register,
16722 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16723 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16724 StrIntrinsicNode::UL);
16725 %}
16726 ins_pipe(pipe_class_memory);
16727 %}
16728
16729 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16730 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16731 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16732 pRegGov_P1 pgtmp2, rFlagsReg cr)
16733 %{
16734 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16735 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16736 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16737 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16738
16739 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16740 ins_encode %{
16741 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16742 __ string_compare($str1$$Register, $str2$$Register,
16743 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16744 $tmp1$$Register, $tmp2$$Register,
16745 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16746 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16747 StrIntrinsicNode::UU);
16748 %}
16749 ins_pipe(pipe_class_memory);
16750 %}
16751
16752 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16753 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16754 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16755 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16756 %{
16757 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16758 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16759 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16760 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16761 TEMP vtmp0, TEMP vtmp1, KILL cr);
16762 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16763 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16764
16765 ins_encode %{
16766 __ string_indexof($str1$$Register, $str2$$Register,
16767 $cnt1$$Register, $cnt2$$Register,
16768 $tmp1$$Register, $tmp2$$Register,
16769 $tmp3$$Register, $tmp4$$Register,
16770 $tmp5$$Register, $tmp6$$Register,
16771 -1, $result$$Register, StrIntrinsicNode::UU);
16772 %}
16773 ins_pipe(pipe_class_memory);
16774 %}
16775
16776 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16777 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16778 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16779 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16780 %{
16781 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16782 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16783 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16784 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16785 TEMP vtmp0, TEMP vtmp1, KILL cr);
16786 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16787 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16788
16789 ins_encode %{
16790 __ string_indexof($str1$$Register, $str2$$Register,
16791 $cnt1$$Register, $cnt2$$Register,
16792 $tmp1$$Register, $tmp2$$Register,
16793 $tmp3$$Register, $tmp4$$Register,
16794 $tmp5$$Register, $tmp6$$Register,
16795 -1, $result$$Register, StrIntrinsicNode::LL);
16796 %}
16797 ins_pipe(pipe_class_memory);
16798 %}
16799
16800 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16801 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16802 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16803 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16804 %{
16805 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16806 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16807 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16808 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16809 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16810 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16811 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16812
16813 ins_encode %{
16814 __ string_indexof($str1$$Register, $str2$$Register,
16815 $cnt1$$Register, $cnt2$$Register,
16816 $tmp1$$Register, $tmp2$$Register,
16817 $tmp3$$Register, $tmp4$$Register,
16818 $tmp5$$Register, $tmp6$$Register,
16819 -1, $result$$Register, StrIntrinsicNode::UL);
16820 %}
16821 ins_pipe(pipe_class_memory);
16822 %}
16823
16824 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16825 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16826 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16827 %{
16828 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16829 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16830 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16831 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16832 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16833 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16834
16835 ins_encode %{
16836 int icnt2 = (int)$int_cnt2$$constant;
16837 __ string_indexof($str1$$Register, $str2$$Register,
16838 $cnt1$$Register, zr,
16839 $tmp1$$Register, $tmp2$$Register,
16840 $tmp3$$Register, $tmp4$$Register, zr, zr,
16841 icnt2, $result$$Register, StrIntrinsicNode::UU);
16842 %}
16843 ins_pipe(pipe_class_memory);
16844 %}
16845
16846 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16847 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16848 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16849 %{
16850 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16851 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16852 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16853 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16854 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16855 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16856
16857 ins_encode %{
16858 int icnt2 = (int)$int_cnt2$$constant;
16859 __ string_indexof($str1$$Register, $str2$$Register,
16860 $cnt1$$Register, zr,
16861 $tmp1$$Register, $tmp2$$Register,
16862 $tmp3$$Register, $tmp4$$Register, zr, zr,
16863 icnt2, $result$$Register, StrIntrinsicNode::LL);
16864 %}
16865 ins_pipe(pipe_class_memory);
16866 %}
16867
16868 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16869 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16870 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16871 %{
16872 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16873 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16874 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16875 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16876 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16877 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16878
16879 ins_encode %{
16880 int icnt2 = (int)$int_cnt2$$constant;
16881 __ string_indexof($str1$$Register, $str2$$Register,
16882 $cnt1$$Register, zr,
16883 $tmp1$$Register, $tmp2$$Register,
16884 $tmp3$$Register, $tmp4$$Register, zr, zr,
16885 icnt2, $result$$Register, StrIntrinsicNode::UL);
16886 %}
16887 ins_pipe(pipe_class_memory);
16888 %}
16889
16890 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16891 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16892 iRegINoSp tmp3, rFlagsReg cr)
16893 %{
16894 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16895 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16896 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16897 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16898
16899 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16900
16901 ins_encode %{
16902 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16903 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16904 $tmp3$$Register);
16905 %}
16906 ins_pipe(pipe_class_memory);
16907 %}
16908
16909 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16910 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16911 iRegINoSp tmp3, rFlagsReg cr)
16912 %{
16913 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16914 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16915 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16916 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16917
16918 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16919
16920 ins_encode %{
16921 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16922 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16923 $tmp3$$Register);
16924 %}
16925 ins_pipe(pipe_class_memory);
16926 %}
16927
16928 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16929 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16930 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16931 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16932 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16933 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16934 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16935 ins_encode %{
16936 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16937 $result$$Register, $ztmp1$$FloatRegister,
16938 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16939 $ptmp$$PRegister, true /* isL */);
16940 %}
16941 ins_pipe(pipe_class_memory);
16942 %}
16943
16944 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16945 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16946 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16947 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16948 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16949 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16950 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16951 ins_encode %{
16952 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16953 $result$$Register, $ztmp1$$FloatRegister,
16954 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16955 $ptmp$$PRegister, false /* isL */);
16956 %}
16957 ins_pipe(pipe_class_memory);
16958 %}
16959
16960 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16961 iRegI_R0 result, rFlagsReg cr)
16962 %{
16963 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16964 match(Set result (StrEquals (Binary str1 str2) cnt));
16965 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16966
16967 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16968 ins_encode %{
16969 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16970 __ string_equals($str1$$Register, $str2$$Register,
16971 $result$$Register, $cnt$$Register);
16972 %}
16973 ins_pipe(pipe_class_memory);
16974 %}
16975
16976 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16977 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16978 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16979 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16980 iRegP_R10 tmp, rFlagsReg cr)
16981 %{
16982 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16983 match(Set result (AryEq ary1 ary2));
16984 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16985 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16986 TEMP vtmp6, TEMP vtmp7, KILL cr);
16987
16988 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16989 ins_encode %{
16990 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16991 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16992 $result$$Register, $tmp$$Register, 1);
16993 if (tpc == nullptr) {
16994 ciEnv::current()->record_failure("CodeCache is full");
16995 return;
16996 }
16997 %}
16998 ins_pipe(pipe_class_memory);
16999 %}
17000
17001 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17002 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17003 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17004 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17005 iRegP_R10 tmp, rFlagsReg cr)
17006 %{
17007 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17008 match(Set result (AryEq ary1 ary2));
17009 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17010 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17011 TEMP vtmp6, TEMP vtmp7, KILL cr);
17012
17013 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17014 ins_encode %{
17015 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17016 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17017 $result$$Register, $tmp$$Register, 2);
17018 if (tpc == nullptr) {
17019 ciEnv::current()->record_failure("CodeCache is full");
17020 return;
17021 }
17022 %}
17023 ins_pipe(pipe_class_memory);
17024 %}
17025
17026 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17027 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17028 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17029 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17030 %{
17031 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17032 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17033 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17034
17035 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17036 ins_encode %{
17037 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17038 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17039 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17040 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17041 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17042 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17043 (BasicType)$basic_type$$constant);
17044 if (tpc == nullptr) {
17045 ciEnv::current()->record_failure("CodeCache is full");
17046 return;
17047 }
17048 %}
17049 ins_pipe(pipe_class_memory);
17050 %}
17051
17052 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17053 %{
17054 match(Set result (CountPositives ary1 len));
17055 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17056 format %{ "count positives byte[] $ary1,$len -> $result" %}
17057 ins_encode %{
17058 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17059 if (tpc == nullptr) {
17060 ciEnv::current()->record_failure("CodeCache is full");
17061 return;
17062 }
17063 %}
17064 ins_pipe( pipe_slow );
17065 %}
17066
17067 // fast char[] to byte[] compression
17068 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17069 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17070 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17071 iRegI_R0 result, rFlagsReg cr)
17072 %{
17073 match(Set result (StrCompressedCopy src (Binary dst len)));
17074 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17075 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17076
17077 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17078 ins_encode %{
17079 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17080 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17081 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17082 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17083 %}
17084 ins_pipe(pipe_slow);
17085 %}
17086
17087 // fast byte[] to char[] inflation
17088 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17089 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17090 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17091 %{
17092 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17093 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17094 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17095 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17096
17097 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17098 ins_encode %{
17099 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17100 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17101 $vtmp2$$FloatRegister, $tmp$$Register);
17102 if (tpc == nullptr) {
17103 ciEnv::current()->record_failure("CodeCache is full");
17104 return;
17105 }
17106 %}
17107 ins_pipe(pipe_class_memory);
17108 %}
17109
17110 // encode char[] to byte[] in ISO_8859_1
17111 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17112 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17113 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17114 iRegI_R0 result, rFlagsReg cr)
17115 %{
17116 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17117 match(Set result (EncodeISOArray src (Binary dst len)));
17118 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17119 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17120
17121 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17122 ins_encode %{
17123 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17124 $result$$Register, false,
17125 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17126 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17127 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17128 %}
17129 ins_pipe(pipe_class_memory);
17130 %}
17131
17132 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17133 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17134 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17135 iRegI_R0 result, rFlagsReg cr)
17136 %{
17137 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17138 match(Set result (EncodeISOArray src (Binary dst len)));
17139 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17140 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17141
17142 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17143 ins_encode %{
17144 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17145 $result$$Register, true,
17146 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17147 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17148 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17149 %}
17150 ins_pipe(pipe_class_memory);
17151 %}
17152
17153 //----------------------------- CompressBits/ExpandBits ------------------------
17154
17155 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17156 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17157 match(Set dst (CompressBits src mask));
17158 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17159 format %{ "mov $tsrc, $src\n\t"
17160 "mov $tmask, $mask\n\t"
17161 "bext $tdst, $tsrc, $tmask\n\t"
17162 "mov $dst, $tdst"
17163 %}
17164 ins_encode %{
17165 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17166 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17167 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17168 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17169 %}
17170 ins_pipe(pipe_slow);
17171 %}
17172
17173 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17174 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17175 match(Set dst (CompressBits (LoadI mem) mask));
17176 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17177 format %{ "ldrs $tsrc, $mem\n\t"
17178 "ldrs $tmask, $mask\n\t"
17179 "bext $tdst, $tsrc, $tmask\n\t"
17180 "mov $dst, $tdst"
17181 %}
17182 ins_encode %{
17183 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17184 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17185 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17186 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17187 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17188 %}
17189 ins_pipe(pipe_slow);
17190 %}
17191
17192 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17193 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17194 match(Set dst (CompressBits src mask));
17195 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17196 format %{ "mov $tsrc, $src\n\t"
17197 "mov $tmask, $mask\n\t"
17198 "bext $tdst, $tsrc, $tmask\n\t"
17199 "mov $dst, $tdst"
17200 %}
17201 ins_encode %{
17202 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17203 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17204 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17205 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17206 %}
17207 ins_pipe(pipe_slow);
17208 %}
17209
17210 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17211 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17212 match(Set dst (CompressBits (LoadL mem) mask));
17213 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17214 format %{ "ldrd $tsrc, $mem\n\t"
17215 "ldrd $tmask, $mask\n\t"
17216 "bext $tdst, $tsrc, $tmask\n\t"
17217 "mov $dst, $tdst"
17218 %}
17219 ins_encode %{
17220 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17221 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17222 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17223 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17224 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17225 %}
17226 ins_pipe(pipe_slow);
17227 %}
17228
17229 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17230 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17231 match(Set dst (ExpandBits src mask));
17232 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17233 format %{ "mov $tsrc, $src\n\t"
17234 "mov $tmask, $mask\n\t"
17235 "bdep $tdst, $tsrc, $tmask\n\t"
17236 "mov $dst, $tdst"
17237 %}
17238 ins_encode %{
17239 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17240 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17241 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17242 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17243 %}
17244 ins_pipe(pipe_slow);
17245 %}
17246
17247 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17248 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17249 match(Set dst (ExpandBits (LoadI mem) mask));
17250 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17251 format %{ "ldrs $tsrc, $mem\n\t"
17252 "ldrs $tmask, $mask\n\t"
17253 "bdep $tdst, $tsrc, $tmask\n\t"
17254 "mov $dst, $tdst"
17255 %}
17256 ins_encode %{
17257 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17258 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17259 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17260 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17261 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17262 %}
17263 ins_pipe(pipe_slow);
17264 %}
17265
17266 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17267 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17268 match(Set dst (ExpandBits src mask));
17269 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17270 format %{ "mov $tsrc, $src\n\t"
17271 "mov $tmask, $mask\n\t"
17272 "bdep $tdst, $tsrc, $tmask\n\t"
17273 "mov $dst, $tdst"
17274 %}
17275 ins_encode %{
17276 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17277 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17278 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17279 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17280 %}
17281 ins_pipe(pipe_slow);
17282 %}
17283
17284
17285 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17286 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17287 match(Set dst (ExpandBits (LoadL mem) mask));
17288 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17289 format %{ "ldrd $tsrc, $mem\n\t"
17290 "ldrd $tmask, $mask\n\t"
17291 "bdep $tdst, $tsrc, $tmask\n\t"
17292 "mov $dst, $tdst"
17293 %}
17294 ins_encode %{
17295 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17296 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17297 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17298 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17299 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17300 %}
17301 ins_pipe(pipe_slow);
17302 %}
17303
17304 //----------------------------- Reinterpret ----------------------------------
17305 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17306 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17307 match(Set dst (ReinterpretHF2S src));
17308 format %{ "reinterpretHF2S $dst, $src" %}
17309 ins_encode %{
17310 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17311 %}
17312 ins_pipe(pipe_slow);
17313 %}
17314
17315 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17316 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17317 match(Set dst (ReinterpretS2HF src));
17318 format %{ "reinterpretS2HF $dst, $src" %}
17319 ins_encode %{
17320 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17321 %}
17322 ins_pipe(pipe_slow);
17323 %}
17324
17325 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17326 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17327 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17328 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17329 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17330 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17331 // can be omitted in this pattern, resulting in -
17332 // fcvt $dst, $src // Convert float to half-precision float
17333 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17334 %{
17335 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17336 format %{ "convF2HFAndS2HF $dst, $src" %}
17337 ins_encode %{
17338 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17339 %}
17340 ins_pipe(pipe_slow);
17341 %}
17342
17343 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17344 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17345 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17346 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17347 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17348 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17349 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17350 // resulting in -
17351 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17352 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17353 %{
17354 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17355 format %{ "convHF2SAndHF2F $dst, $src" %}
17356 ins_encode %{
17357 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17358 %}
17359 ins_pipe(pipe_slow);
17360 %}
17361
17362 // ============================================================================
17363 // This name is KNOWN by the ADLC and cannot be changed.
17364 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17365 // for this guy.
17366 instruct tlsLoadP(thread_RegP dst)
17367 %{
17368 match(Set dst (ThreadLocal));
17369
17370 ins_cost(0);
17371
17372 format %{ " -- \t// $dst=Thread::current(), empty" %}
17373
17374 size(0);
17375
17376 ins_encode( /*empty*/ );
17377
17378 ins_pipe(pipe_class_empty);
17379 %}
17380
17381 //----------PEEPHOLE RULES-----------------------------------------------------
17382 // These must follow all instruction definitions as they use the names
17383 // defined in the instructions definitions.
17384 //
17385 // peepmatch ( root_instr_name [preceding_instruction]* );
17386 //
17387 // peepconstraint %{
17388 // (instruction_number.operand_name relational_op instruction_number.operand_name
17389 // [, ...] );
17390 // // instruction numbers are zero-based using left to right order in peepmatch
17391 //
17392 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17393 // // provide an instruction_number.operand_name for each operand that appears
17394 // // in the replacement instruction's match rule
17395 //
17396 // ---------VM FLAGS---------------------------------------------------------
17397 //
17398 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17399 //
17400 // Each peephole rule is given an identifying number starting with zero and
17401 // increasing by one in the order seen by the parser. An individual peephole
17402 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17403 // on the command-line.
17404 //
17405 // ---------CURRENT LIMITATIONS----------------------------------------------
17406 //
17407 // Only match adjacent instructions in same basic block
17408 // Only equality constraints
17409 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17410 // Only one replacement instruction
17411 //
17412 // ---------EXAMPLE----------------------------------------------------------
17413 //
17414 // // pertinent parts of existing instructions in architecture description
17415 // instruct movI(iRegINoSp dst, iRegI src)
17416 // %{
17417 // match(Set dst (CopyI src));
17418 // %}
17419 //
17420 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17421 // %{
17422 // match(Set dst (AddI dst src));
17423 // effect(KILL cr);
17424 // %}
17425 //
17426 // // Change (inc mov) to lea
17427 // peephole %{
17428 // // increment preceded by register-register move
17429 // peepmatch ( incI_iReg movI );
17430 // // require that the destination register of the increment
17431 // // match the destination register of the move
17432 // peepconstraint ( 0.dst == 1.dst );
17433 // // construct a replacement instruction that sets
17434 // // the destination to ( move's source register + one )
17435 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17436 // %}
17437 //
17438
17439 // Implementation no longer uses movX instructions since
17440 // machine-independent system no longer uses CopyX nodes.
17441 //
17442 // peephole
17443 // %{
17444 // peepmatch (incI_iReg movI);
17445 // peepconstraint (0.dst == 1.dst);
17446 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17447 // %}
17448
17449 // peephole
17450 // %{
17451 // peepmatch (decI_iReg movI);
17452 // peepconstraint (0.dst == 1.dst);
17453 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17454 // %}
17455
17456 // peephole
17457 // %{
17458 // peepmatch (addI_iReg_imm movI);
17459 // peepconstraint (0.dst == 1.dst);
17460 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17461 // %}
17462
17463 // peephole
17464 // %{
17465 // peepmatch (incL_iReg movL);
17466 // peepconstraint (0.dst == 1.dst);
17467 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17468 // %}
17469
17470 // peephole
17471 // %{
17472 // peepmatch (decL_iReg movL);
17473 // peepconstraint (0.dst == 1.dst);
17474 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17475 // %}
17476
17477 // peephole
17478 // %{
17479 // peepmatch (addL_iReg_imm movL);
17480 // peepconstraint (0.dst == 1.dst);
17481 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17482 // %}
17483
17484 // peephole
17485 // %{
17486 // peepmatch (addP_iReg_imm movP);
17487 // peepconstraint (0.dst == 1.dst);
17488 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17489 // %}
17490
17491 // // Change load of spilled value to only a spill
17492 // instruct storeI(memory mem, iRegI src)
17493 // %{
17494 // match(Set mem (StoreI mem src));
17495 // %}
17496 //
17497 // instruct loadI(iRegINoSp dst, memory mem)
17498 // %{
17499 // match(Set dst (LoadI mem));
17500 // %}
17501 //
17502
17503 //----------SMARTSPILL RULES---------------------------------------------------
17504 // These must follow all instruction definitions as they use the names
17505 // defined in the instructions definitions.
17506
17507 // Local Variables:
17508 // mode: c++
17509 // End: