1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 // Dummy labels for just measuring the code size
1796 Label dummy_slow_path;
1797 Label dummy_continuation;
1798 Label dummy_guard;
1799 Label* slow_path = &dummy_slow_path;
1800 Label* continuation = &dummy_continuation;
1801 Label* guard = &dummy_guard;
1802 if (!Compile::current()->output()->in_scratch_emit_size()) {
1803 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1804 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1805 Compile::current()->output()->add_stub(stub);
1806 slow_path = &stub->entry();
1807 continuation = &stub->continuation();
1808 guard = &stub->guard();
1809 }
1810 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1811 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1812 }
1813
1814 if (VerifyStackAtCalls) {
1815 Unimplemented();
1816 }
1817
1818 C->output()->set_frame_complete(__ offset());
1819
1820 if (C->has_mach_constant_base_node()) {
1821 // NOTE: We set the table base offset here because users might be
1822 // emitted before MachConstantBaseNode.
1823 ConstantTable& constant_table = C->output()->constant_table();
1824 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1825 }
1826 }
1827
1828 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1829 {
1830 return MachNode::size(ra_); // too many variables; just compute it
1831 // the hard way
1832 }
1833
1834 int MachPrologNode::reloc() const
1835 {
1836 return 0;
1837 }
1838
1839 //=============================================================================
1840
1841 #ifndef PRODUCT
1842 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1843 Compile* C = ra_->C;
1844 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1845
1846 st->print("# pop frame %d\n\t",framesize);
1847
1848 if (framesize == 0) {
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1851 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1852 st->print("add sp, sp, #%d\n\t", framesize);
1853 } else {
1854 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1855 st->print("add sp, sp, rscratch1\n\t");
1856 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1857 }
1858 if (VM_Version::use_rop_protection()) {
1859 st->print("autiaz\n\t");
1860 st->print("ldr zr, [lr]\n\t");
1861 }
1862
1863 if (do_polling() && C->is_method_compilation()) {
1864 st->print("# test polling word\n\t");
1865 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1866 st->print("cmp sp, rscratch1\n\t");
1867 st->print("bhi #slow_path");
1868 }
1869 }
1870 #endif
1871
1872 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 __ remove_frame(framesize);
1877
1878 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1879 __ reserved_stack_check();
1880 }
1881
1882 if (do_polling() && C->is_method_compilation()) {
1883 Label dummy_label;
1884 Label* code_stub = &dummy_label;
1885 if (!C->output()->in_scratch_emit_size()) {
1886 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1887 C->output()->add_stub(stub);
1888 code_stub = &stub->entry();
1889 }
1890 __ relocate(relocInfo::poll_return_type);
1891 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1892 }
1893 }
1894
1895 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1896 // Variable size. Determine dynamically.
1897 return MachNode::size(ra_);
1898 }
1899
1900 int MachEpilogNode::reloc() const {
1901 // Return number of relocatable values contained in this instruction.
1902 return 1; // 1 for polling page.
1903 }
1904
1905 const Pipeline * MachEpilogNode::pipeline() const {
1906 return MachNode::pipeline_class();
1907 }
1908
1909 //=============================================================================
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && masm) {
1975 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1976 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1977 // stack->stack
1978 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1979 sve_vector_reg_size_in_bytes);
1980 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1981 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1982 sve_vector_reg_size_in_bytes);
1983 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1984 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1985 sve_vector_reg_size_in_bytes);
1986 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1987 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1988 as_FloatRegister(Matcher::_regEncode[src_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]));
1990 } else {
1991 ShouldNotReachHere();
1992 }
1993 } else if (masm) {
1994 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1995 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1996 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1997 // stack->stack
1998 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1999 if (ireg == Op_VecD) {
2000 __ unspill(rscratch1, true, src_offset);
2001 __ spill(rscratch1, true, dst_offset);
2002 } else {
2003 __ spill_copy128(src_offset, dst_offset);
2004 }
2005 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2006 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2007 ireg == Op_VecD ? __ T8B : __ T16B,
2008 as_FloatRegister(Matcher::_regEncode[src_lo]));
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2011 ireg == Op_VecD ? __ D : __ Q,
2012 ra_->reg2offset(dst_lo));
2013 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2014 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2015 ireg == Op_VecD ? __ D : __ Q,
2016 ra_->reg2offset(src_lo));
2017 } else {
2018 ShouldNotReachHere();
2019 }
2020 }
2021 } else if (masm) {
2022 switch (src_lo_rc) {
2023 case rc_int:
2024 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2025 if (is64) {
2026 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2027 as_Register(Matcher::_regEncode[src_lo]));
2028 } else {
2029 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 }
2032 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2033 if (is64) {
2034 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2035 as_Register(Matcher::_regEncode[src_lo]));
2036 } else {
2037 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2038 as_Register(Matcher::_regEncode[src_lo]));
2039 }
2040 } else { // gpr --> stack spill
2041 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2042 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2043 }
2044 break;
2045 case rc_float:
2046 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2047 if (is64) {
2048 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2049 as_FloatRegister(Matcher::_regEncode[src_lo]));
2050 } else {
2051 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2052 as_FloatRegister(Matcher::_regEncode[src_lo]));
2053 }
2054 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2055 if (is64) {
2056 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2057 as_FloatRegister(Matcher::_regEncode[src_lo]));
2058 } else {
2059 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2060 as_FloatRegister(Matcher::_regEncode[src_lo]));
2061 }
2062 } else { // fpr --> stack spill
2063 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2064 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2065 is64 ? __ D : __ S, dst_offset);
2066 }
2067 break;
2068 case rc_stack:
2069 if (dst_lo_rc == rc_int) { // stack --> gpr load
2070 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2071 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2072 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2073 is64 ? __ D : __ S, src_offset);
2074 } else if (dst_lo_rc == rc_predicate) {
2075 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2076 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2077 } else { // stack --> stack copy
2078 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2079 if (ideal_reg() == Op_RegVectMask) {
2080 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2081 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2082 } else {
2083 __ unspill(rscratch1, is64, src_offset);
2084 __ spill(rscratch1, is64, dst_offset);
2085 }
2086 }
2087 break;
2088 case rc_predicate:
2089 if (dst_lo_rc == rc_predicate) {
2090 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2091 } else if (dst_lo_rc == rc_stack) {
2092 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2093 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2094 } else {
2095 assert(false, "bad src and dst rc_class combination.");
2096 ShouldNotReachHere();
2097 }
2098 break;
2099 default:
2100 assert(false, "bad rc_class for spill");
2101 ShouldNotReachHere();
2102 }
2103 }
2104
2105 if (st) {
2106 st->print("spill ");
2107 if (src_lo_rc == rc_stack) {
2108 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2109 } else {
2110 st->print("%s -> ", Matcher::regName[src_lo]);
2111 }
2112 if (dst_lo_rc == rc_stack) {
2113 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2114 } else {
2115 st->print("%s", Matcher::regName[dst_lo]);
2116 }
2117 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2118 int vsize = 0;
2119 switch (ideal_reg()) {
2120 case Op_VecD:
2121 vsize = 64;
2122 break;
2123 case Op_VecX:
2124 vsize = 128;
2125 break;
2126 case Op_VecA:
2127 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2128 break;
2129 default:
2130 assert(false, "bad register type for spill");
2131 ShouldNotReachHere();
2132 }
2133 st->print("\t# vector spill size = %d", vsize);
2134 } else if (ideal_reg() == Op_RegVectMask) {
2135 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2136 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2137 st->print("\t# predicate spill size = %d", vsize);
2138 } else {
2139 st->print("\t# spill size = %d", is64 ? 64 : 32);
2140 }
2141 }
2142
2143 return 0;
2144
2145 }
2146
2147 #ifndef PRODUCT
2148 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2149 if (!ra_)
2150 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2151 else
2152 implementation(nullptr, ra_, false, st);
2153 }
2154 #endif
2155
2156 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2157 implementation(masm, ra_, false, nullptr);
2158 }
2159
2160 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2161 return MachNode::size(ra_);
2162 }
2163
2164 //=============================================================================
2165
2166 #ifndef PRODUCT
2167 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2168 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2169 int reg = ra_->get_reg_first(this);
2170 st->print("add %s, rsp, #%d]\t# box lock",
2171 Matcher::regName[reg], offset);
2172 }
2173 #endif
2174
2175 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177 int reg = ra_->get_encode(this);
2178
2179 // This add will handle any 24-bit signed offset. 24 bits allows an
2180 // 8 megabyte stack frame.
2181 __ add(as_Register(reg), sp, offset);
2182 }
2183
2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2185 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2186 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2187
2188 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2189 return NativeInstruction::instruction_size;
2190 } else {
2191 return 2 * NativeInstruction::instruction_size;
2192 }
2193 }
2194
2195 //=============================================================================
2196
2197 #ifndef PRODUCT
2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2199 {
2200 st->print_cr("# MachUEPNode");
2201 if (UseCompressedClassPointers) {
2202 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2203 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2204 st->print_cr("\tcmpw rscratch1, r10");
2205 } else {
2206 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2207 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2208 st->print_cr("\tcmp rscratch1, r10");
2209 }
2210 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2211 }
2212 #endif
2213
2214 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2215 {
2216 __ ic_check(InteriorEntryAlignment);
2217 }
2218
2219 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2220 {
2221 return MachNode::size(ra_);
2222 }
2223
2224 // REQUIRED EMIT CODE
2225
2226 //=============================================================================
2227
2228 // Emit exception handler code.
2229 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2230 {
2231 // mov rscratch1 #exception_blob_entry_point
2232 // br rscratch1
2233 // Note that the code buffer's insts_mark is always relative to insts.
2234 // That's why we must use the macroassembler to generate a handler.
2235 address base = __ start_a_stub(size_exception_handler());
2236 if (base == nullptr) {
2237 ciEnv::current()->record_failure("CodeCache is full");
2238 return 0; // CodeBuffer::expand failed
2239 }
2240 int offset = __ offset();
2241 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2242 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2243 __ end_a_stub();
2244 return offset;
2245 }
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258
2259 __ adr(lr, __ pc());
2260 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2261
2262 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2263 __ end_a_stub();
2264 return offset;
2265 }
2266
2267 // REQUIRED MATCHER CODE
2268
2269 //=============================================================================
2270
2271 bool Matcher::match_rule_supported(int opcode) {
2272 if (!has_match_rule(opcode))
2273 return false;
2274
2275 switch (opcode) {
2276 case Op_OnSpinWait:
2277 return VM_Version::supports_on_spin_wait();
2278 case Op_CacheWB:
2279 case Op_CacheWBPreSync:
2280 case Op_CacheWBPostSync:
2281 if (!VM_Version::supports_data_cache_line_flush()) {
2282 return false;
2283 }
2284 break;
2285 case Op_ExpandBits:
2286 case Op_CompressBits:
2287 if (!VM_Version::supports_svebitperm()) {
2288 return false;
2289 }
2290 break;
2291 case Op_FmaF:
2292 case Op_FmaD:
2293 case Op_FmaVF:
2294 case Op_FmaVD:
2295 if (!UseFMA) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaHF:
2300 // UseFMA flag also needs to be checked along with FEAT_FP16
2301 if (!UseFMA || !is_feat_fp16_supported()) {
2302 return false;
2303 }
2304 break;
2305 case Op_AddHF:
2306 case Op_SubHF:
2307 case Op_MulHF:
2308 case Op_DivHF:
2309 case Op_MinHF:
2310 case Op_MaxHF:
2311 case Op_SqrtHF:
2312 // Half-precision floating point scalar operations require FEAT_FP16
2313 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2314 // features are supported.
2315 if (!is_feat_fp16_supported()) {
2316 return false;
2317 }
2318 break;
2319 }
2320
2321 return true; // Per default match rules are supported.
2322 }
2323
2324 const RegMask* Matcher::predicate_reg_mask(void) {
2325 return &_PR_REG_mask;
2326 }
2327
2328 bool Matcher::supports_vector_calling_convention(void) {
2329 return EnableVectorSupport;
2330 }
2331
2332 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2333 assert(EnableVectorSupport, "sanity");
2334 int lo = V0_num;
2335 int hi = V0_H_num;
2336 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2337 hi = V0_K_num;
2338 }
2339 return OptoRegPair(hi, lo);
2340 }
2341
2342 // Is this branch offset short enough that a short branch can be used?
2343 //
2344 // NOTE: If the platform does not provide any short branch variants, then
2345 // this method should return false for offset 0.
2346 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2347 // The passed offset is relative to address of the branch.
2348
2349 return (-32768 <= offset && offset < 32768);
2350 }
2351
2352 // Vector width in bytes.
2353 int Matcher::vector_width_in_bytes(BasicType bt) {
2354 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2355 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2356 // Minimum 2 values in vector
2357 if (size < 2*type2aelembytes(bt)) size = 0;
2358 // But never < 4
2359 if (size < 4) size = 0;
2360 return size;
2361 }
2362
2363 // Limits on vector size (number of elements) loaded into vector.
2364 int Matcher::max_vector_size(const BasicType bt) {
2365 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2366 }
2367
2368 int Matcher::min_vector_size(const BasicType bt) {
2369 int max_size = max_vector_size(bt);
2370 // Limit the min vector size to 8 bytes.
2371 int size = 8 / type2aelembytes(bt);
2372 if (bt == T_BYTE) {
2373 // To support vector api shuffle/rearrange.
2374 size = 4;
2375 } else if (bt == T_BOOLEAN) {
2376 // To support vector api load/store mask.
2377 size = 2;
2378 }
2379 if (size < 2) size = 2;
2380 return MIN2(size, max_size);
2381 }
2382
2383 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2384 return Matcher::max_vector_size(bt);
2385 }
2386
2387 // Actual max scalable vector register length.
2388 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2389 return Matcher::max_vector_size(bt);
2390 }
2391
2392 // Vector ideal reg.
2393 uint Matcher::vector_ideal_reg(int len) {
2394 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2395 return Op_VecA;
2396 }
2397 switch(len) {
2398 // For 16-bit/32-bit mask vector, reuse VecD.
2399 case 2:
2400 case 4:
2401 case 8: return Op_VecD;
2402 case 16: return Op_VecX;
2403 }
2404 ShouldNotReachHere();
2405 return 0;
2406 }
2407
2408 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2409 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2410 switch (ideal_reg) {
2411 case Op_VecA: return new vecAOper();
2412 case Op_VecD: return new vecDOper();
2413 case Op_VecX: return new vecXOper();
2414 }
2415 ShouldNotReachHere();
2416 return nullptr;
2417 }
2418
2419 bool Matcher::is_reg2reg_move(MachNode* m) {
2420 return false;
2421 }
2422
2423 bool Matcher::is_generic_vector(MachOper* opnd) {
2424 return opnd->opcode() == VREG;
2425 }
2426
2427 // Return whether or not this register is ever used as an argument.
2428 // This function is used on startup to build the trampoline stubs in
2429 // generateOptoStub. Registers not mentioned will be killed by the VM
2430 // call in the trampoline, and arguments in those registers not be
2431 // available to the callee.
2432 bool Matcher::can_be_java_arg(int reg)
2433 {
2434 return
2435 reg == R0_num || reg == R0_H_num ||
2436 reg == R1_num || reg == R1_H_num ||
2437 reg == R2_num || reg == R2_H_num ||
2438 reg == R3_num || reg == R3_H_num ||
2439 reg == R4_num || reg == R4_H_num ||
2440 reg == R5_num || reg == R5_H_num ||
2441 reg == R6_num || reg == R6_H_num ||
2442 reg == R7_num || reg == R7_H_num ||
2443 reg == V0_num || reg == V0_H_num ||
2444 reg == V1_num || reg == V1_H_num ||
2445 reg == V2_num || reg == V2_H_num ||
2446 reg == V3_num || reg == V3_H_num ||
2447 reg == V4_num || reg == V4_H_num ||
2448 reg == V5_num || reg == V5_H_num ||
2449 reg == V6_num || reg == V6_H_num ||
2450 reg == V7_num || reg == V7_H_num;
2451 }
2452
2453 bool Matcher::is_spillable_arg(int reg)
2454 {
2455 return can_be_java_arg(reg);
2456 }
2457
2458 uint Matcher::int_pressure_limit()
2459 {
2460 // JDK-8183543: When taking the number of available registers as int
2461 // register pressure threshold, the jtreg test:
2462 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2463 // failed due to C2 compilation failure with
2464 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2465 //
2466 // A derived pointer is live at CallNode and then is flagged by RA
2467 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2468 // derived pointers and lastly fail to spill after reaching maximum
2469 // number of iterations. Lowering the default pressure threshold to
2470 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2471 // a high register pressure area of the code so that split_DEF can
2472 // generate DefinitionSpillCopy for the derived pointer.
2473 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2474 if (!PreserveFramePointer) {
2475 // When PreserveFramePointer is off, frame pointer is allocatable,
2476 // but different from other SOC registers, it is excluded from
2477 // fatproj's mask because its save type is No-Save. Decrease 1 to
2478 // ensure high pressure at fatproj when PreserveFramePointer is off.
2479 // See check_pressure_at_fatproj().
2480 default_int_pressure_threshold--;
2481 }
2482 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2483 }
2484
2485 uint Matcher::float_pressure_limit()
2486 {
2487 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2488 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2489 }
2490
2491 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2492 return false;
2493 }
2494
2495 RegMask Matcher::divI_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 // Register for MODI projection of divmodI.
2501 RegMask Matcher::modI_proj_mask() {
2502 ShouldNotReachHere();
2503 return RegMask();
2504 }
2505
2506 // Register for DIVL projection of divmodL.
2507 RegMask Matcher::divL_proj_mask() {
2508 ShouldNotReachHere();
2509 return RegMask();
2510 }
2511
2512 // Register for MODL projection of divmodL.
2513 RegMask Matcher::modL_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask();
2516 }
2517
2518 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2519 return FP_REG_mask();
2520 }
2521
2522 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2523 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2524 Node* u = addp->fast_out(i);
2525 if (u->is_LoadStore()) {
2526 // On AArch64, LoadStoreNodes (i.e. compare and swap
2527 // instructions) only take register indirect as an operand, so
2528 // any attempt to use an AddPNode as an input to a LoadStoreNode
2529 // must fail.
2530 return false;
2531 }
2532 if (u->is_Mem()) {
2533 int opsize = u->as_Mem()->memory_size();
2534 assert(opsize > 0, "unexpected memory operand size");
2535 if (u->as_Mem()->memory_size() != (1<<shift)) {
2536 return false;
2537 }
2538 }
2539 }
2540 return true;
2541 }
2542
2543 // Convert BootTest condition to Assembler condition.
2544 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2545 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2546 Assembler::Condition result;
2547 switch(cond) {
2548 case BoolTest::eq:
2549 result = Assembler::EQ; break;
2550 case BoolTest::ne:
2551 result = Assembler::NE; break;
2552 case BoolTest::le:
2553 result = Assembler::LE; break;
2554 case BoolTest::ge:
2555 result = Assembler::GE; break;
2556 case BoolTest::lt:
2557 result = Assembler::LT; break;
2558 case BoolTest::gt:
2559 result = Assembler::GT; break;
2560 case BoolTest::ule:
2561 result = Assembler::LS; break;
2562 case BoolTest::uge:
2563 result = Assembler::HS; break;
2564 case BoolTest::ult:
2565 result = Assembler::LO; break;
2566 case BoolTest::ugt:
2567 result = Assembler::HI; break;
2568 case BoolTest::overflow:
2569 result = Assembler::VS; break;
2570 case BoolTest::no_overflow:
2571 result = Assembler::VC; break;
2572 default:
2573 ShouldNotReachHere();
2574 return Assembler::Condition(-1);
2575 }
2576
2577 // Check conversion
2578 if (cond & BoolTest::unsigned_compare) {
2579 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2580 } else {
2581 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2582 }
2583
2584 return result;
2585 }
2586
2587 // Binary src (Replicate con)
2588 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2589 if (n == nullptr || m == nullptr) {
2590 return false;
2591 }
2592
2593 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2594 return false;
2595 }
2596
2597 Node* imm_node = m->in(1);
2598 if (!imm_node->is_Con()) {
2599 return false;
2600 }
2601
2602 const Type* t = imm_node->bottom_type();
2603 if (!(t->isa_int() || t->isa_long())) {
2604 return false;
2605 }
2606
2607 switch (n->Opcode()) {
2608 case Op_AndV:
2609 case Op_OrV:
2610 case Op_XorV: {
2611 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2612 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2613 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2614 }
2615 case Op_AddVB:
2616 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2617 case Op_AddVS:
2618 case Op_AddVI:
2619 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2620 case Op_AddVL:
2621 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2622 default:
2623 return false;
2624 }
2625 }
2626
2627 // (XorV src (Replicate m1))
2628 // (XorVMask src (MaskAll m1))
2629 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2630 if (n != nullptr && m != nullptr) {
2631 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2632 VectorNode::is_all_ones_vector(m);
2633 }
2634 return false;
2635 }
2636
2637 // Should the matcher clone input 'm' of node 'n'?
2638 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2639 if (is_vshift_con_pattern(n, m) ||
2640 is_vector_bitwise_not_pattern(n, m) ||
2641 is_valid_sve_arith_imm_pattern(n, m) ||
2642 is_encode_and_store_pattern(n, m)) {
2643 mstack.push(m, Visit);
2644 return true;
2645 }
2646 return false;
2647 }
2648
2649 // Should the Matcher clone shifts on addressing modes, expecting them
2650 // to be subsumed into complex addressing expressions or compute them
2651 // into registers?
2652 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2653
2654 // Loads and stores with indirect memory input (e.g., volatile loads and
2655 // stores) do not subsume the input into complex addressing expressions. If
2656 // the addressing expression is input to at least one such load or store, do
2657 // not clone the addressing expression. Query needs_acquiring_load and
2658 // needs_releasing_store as a proxy for indirect memory input, as it is not
2659 // possible to directly query for indirect memory input at this stage.
2660 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2661 Node* n = m->fast_out(i);
2662 if (n->is_Load() && needs_acquiring_load(n)) {
2663 return false;
2664 }
2665 if (n->is_Store() && needs_releasing_store(n)) {
2666 return false;
2667 }
2668 }
2669
2670 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2671 return true;
2672 }
2673
2674 Node *off = m->in(AddPNode::Offset);
2675 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2676 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.set(off->_idx); // Flag as address_visited
2680 mstack.push(off->in(2), Visit);
2681 Node *conv = off->in(1);
2682 if (conv->Opcode() == Op_ConvI2L &&
2683 // Are there other uses besides address expressions?
2684 !is_visited(conv)) {
2685 address_visited.set(conv->_idx); // Flag as address_visited
2686 mstack.push(conv->in(1), Pre_Visit);
2687 } else {
2688 mstack.push(conv, Pre_Visit);
2689 }
2690 address_visited.test_set(m->_idx); // Flag as address_visited
2691 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2692 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2693 return true;
2694 } else if (off->Opcode() == Op_ConvI2L &&
2695 // Are there other uses besides address expressions?
2696 !is_visited(off)) {
2697 address_visited.test_set(m->_idx); // Flag as address_visited
2698 address_visited.set(off->_idx); // Flag as address_visited
2699 mstack.push(off->in(1), Pre_Visit);
2700 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2701 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2702 return true;
2703 }
2704 return false;
2705 }
2706
2707 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2708 { \
2709 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2710 guarantee(DISP == 0, "mode not permitted for volatile"); \
2711 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2712 __ INSN(REG, as_Register(BASE)); \
2713 }
2714
2715
2716 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2717 {
2718 Address::extend scale;
2719
2720 // Hooboy, this is fugly. We need a way to communicate to the
2721 // encoder that the index needs to be sign extended, so we have to
2722 // enumerate all the cases.
2723 switch (opcode) {
2724 case INDINDEXSCALEDI2L:
2725 case INDINDEXSCALEDI2LN:
2726 case INDINDEXI2L:
2727 case INDINDEXI2LN:
2728 scale = Address::sxtw(size);
2729 break;
2730 default:
2731 scale = Address::lsl(size);
2732 }
2733
2734 if (index == -1) {
2735 return Address(base, disp);
2736 } else {
2737 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2738 return Address(base, as_Register(index), scale);
2739 }
2740 }
2741
2742
2743 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2744 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2745 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2746 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2747 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2748
2749 // Used for all non-volatile memory accesses. The use of
2750 // $mem->opcode() to discover whether this pattern uses sign-extended
2751 // offsets is something of a kludge.
2752 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2753 Register reg, int opcode,
2754 Register base, int index, int scale, int disp,
2755 int size_in_memory)
2756 {
2757 Address addr = mem2address(opcode, base, index, scale, disp);
2758 if (addr.getMode() == Address::base_plus_offset) {
2759 /* Fix up any out-of-range offsets. */
2760 assert_different_registers(rscratch1, base);
2761 assert_different_registers(rscratch1, reg);
2762 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2763 }
2764 (masm->*insn)(reg, addr);
2765 }
2766
2767 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2768 FloatRegister reg, int opcode,
2769 Register base, int index, int size, int disp,
2770 int size_in_memory)
2771 {
2772 Address::extend scale;
2773
2774 switch (opcode) {
2775 case INDINDEXSCALEDI2L:
2776 case INDINDEXSCALEDI2LN:
2777 scale = Address::sxtw(size);
2778 break;
2779 default:
2780 scale = Address::lsl(size);
2781 }
2782
2783 if (index == -1) {
2784 // Fix up any out-of-range offsets.
2785 assert_different_registers(rscratch1, base);
2786 Address addr = Address(base, disp);
2787 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2788 (masm->*insn)(reg, addr);
2789 } else {
2790 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2791 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2792 }
2793 }
2794
2795 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2796 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2797 int opcode, Register base, int index, int size, int disp)
2798 {
2799 if (index == -1) {
2800 (masm->*insn)(reg, T, Address(base, disp));
2801 } else {
2802 assert(disp == 0, "unsupported address mode");
2803 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2804 }
2805 }
2806
2807 %}
2808
2809
2810
2811 //----------ENCODING BLOCK-----------------------------------------------------
2812 // This block specifies the encoding classes used by the compiler to
2813 // output byte streams. Encoding classes are parameterized macros
2814 // used by Machine Instruction Nodes in order to generate the bit
2815 // encoding of the instruction. Operands specify their base encoding
2816 // interface with the interface keyword. There are currently
2817 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2818 // COND_INTER. REG_INTER causes an operand to generate a function
2819 // which returns its register number when queried. CONST_INTER causes
2820 // an operand to generate a function which returns the value of the
2821 // constant when queried. MEMORY_INTER causes an operand to generate
2822 // four functions which return the Base Register, the Index Register,
2823 // the Scale Value, and the Offset Value of the operand when queried.
2824 // COND_INTER causes an operand to generate six functions which return
2825 // the encoding code (ie - encoding bits for the instruction)
2826 // associated with each basic boolean condition for a conditional
2827 // instruction.
2828 //
2829 // Instructions specify two basic values for encoding. Again, a
2830 // function is available to check if the constant displacement is an
2831 // oop. They use the ins_encode keyword to specify their encoding
2832 // classes (which must be a sequence of enc_class names, and their
2833 // parameters, specified in the encoding block), and they use the
2834 // opcode keyword to specify, in order, their primary, secondary, and
2835 // tertiary opcode. Only the opcode sections which a particular
2836 // instruction needs for encoding need to be specified.
2837 encode %{
2838 // Build emit functions for each basic byte or larger field in the
2839 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2840 // from C++ code in the enc_class source block. Emit functions will
2841 // live in the main source block for now. In future, we can
2842 // generalize this by adding a syntax that specifies the sizes of
2843 // fields in an order, so that the adlc can build the emit functions
2844 // automagically
2845
2846 // catch all for unimplemented encodings
2847 enc_class enc_unimplemented %{
2848 __ unimplemented("C2 catch all");
2849 %}
2850
2851 // BEGIN Non-volatile memory access
2852
2853 // This encoding class is generated automatically from ad_encode.m4.
2854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2855 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2856 Register dst_reg = as_Register($dst$$reg);
2857 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2859 %}
2860
2861 // This encoding class is generated automatically from ad_encode.m4.
2862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2863 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2864 Register dst_reg = as_Register($dst$$reg);
2865 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2867 %}
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2880 Register dst_reg = as_Register($dst$$reg);
2881 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2952 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2960 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2968 Register src_reg = as_Register($src$$reg);
2969 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_strb0(memory1 mem) %{
2976 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2977 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2978 %}
2979
2980 // This encoding class is generated automatically from ad_encode.m4.
2981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2982 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2983 Register src_reg = as_Register($src$$reg);
2984 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2985 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2986 %}
2987
2988 // This encoding class is generated automatically from ad_encode.m4.
2989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2990 enc_class aarch64_enc_strh0(memory2 mem) %{
2991 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strw0(memory4 mem) %{
3006 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 // we sometimes get asked to store the stack pointer into the
3015 // current thread -- we cannot do that directly on AArch64
3016 if (src_reg == r31_sp) {
3017 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3018 __ mov(rscratch2, sp);
3019 src_reg = rscratch2;
3020 }
3021 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_str0(memory8 mem) %{
3028 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3029 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3030 %}
3031
3032 // This encoding class is generated automatically from ad_encode.m4.
3033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3034 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3035 FloatRegister src_reg = as_FloatRegister($src$$reg);
3036 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3043 FloatRegister src_reg = as_FloatRegister($src$$reg);
3044 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3051 __ membar(Assembler::StoreStore);
3052 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3054 %}
3055
3056 // END Non-volatile memory access
3057
3058 // Vector loads and stores
3059 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3078 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3079 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3102 FloatRegister src_reg = as_FloatRegister($src$$reg);
3103 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 // volatile loads and stores
3108
3109 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3110 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3111 rscratch1, stlrb);
3112 %}
3113
3114 enc_class aarch64_enc_stlrb0(memory mem) %{
3115 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, stlrb);
3117 %}
3118
3119 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3120 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3121 rscratch1, stlrh);
3122 %}
3123
3124 enc_class aarch64_enc_stlrh0(memory mem) %{
3125 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3126 rscratch1, stlrh);
3127 %}
3128
3129 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3130 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, stlrw);
3132 %}
3133
3134 enc_class aarch64_enc_stlrw0(memory mem) %{
3135 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, stlrw);
3137 %}
3138
3139 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3140 Register dst_reg = as_Register($dst$$reg);
3141 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 __ sxtbw(dst_reg, dst_reg);
3144 %}
3145
3146 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3147 Register dst_reg = as_Register($dst$$reg);
3148 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 __ sxtb(dst_reg, dst_reg);
3151 %}
3152
3153 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3154 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarb);
3156 %}
3157
3158 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3159 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarb);
3161 %}
3162
3163 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3164 Register dst_reg = as_Register($dst$$reg);
3165 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 __ sxthw(dst_reg, dst_reg);
3168 %}
3169
3170 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3171 Register dst_reg = as_Register($dst$$reg);
3172 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 __ sxth(dst_reg, dst_reg);
3175 %}
3176
3177 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3178 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarh);
3180 %}
3181
3182 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3183 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarh);
3185 %}
3186
3187 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3188 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarw);
3190 %}
3191
3192 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3193 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarw);
3195 %}
3196
3197 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3198 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, ldar);
3200 %}
3201
3202 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldarw);
3205 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3209 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldar);
3211 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3212 %}
3213
3214 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3215 Register src_reg = as_Register($src$$reg);
3216 // we sometimes get asked to store the stack pointer into the
3217 // current thread -- we cannot do that directly on AArch64
3218 if (src_reg == r31_sp) {
3219 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3220 __ mov(rscratch2, sp);
3221 src_reg = rscratch2;
3222 }
3223 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, stlr);
3225 %}
3226
3227 enc_class aarch64_enc_stlr0(memory mem) %{
3228 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3233 {
3234 FloatRegister src_reg = as_FloatRegister($src$$reg);
3235 __ fmovs(rscratch2, src_reg);
3236 }
3237 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, stlrw);
3239 %}
3240
3241 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3242 {
3243 FloatRegister src_reg = as_FloatRegister($src$$reg);
3244 __ fmovd(rscratch2, src_reg);
3245 }
3246 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, stlr);
3248 %}
3249
3250 // synchronized read/update encodings
3251
3252 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3253 Register dst_reg = as_Register($dst$$reg);
3254 Register base = as_Register($mem$$base);
3255 int index = $mem$$index;
3256 int scale = $mem$$scale;
3257 int disp = $mem$$disp;
3258 if (index == -1) {
3259 if (disp != 0) {
3260 __ lea(rscratch1, Address(base, disp));
3261 __ ldaxr(dst_reg, rscratch1);
3262 } else {
3263 // TODO
3264 // should we ever get anything other than this case?
3265 __ ldaxr(dst_reg, base);
3266 }
3267 } else {
3268 Register index_reg = as_Register(index);
3269 if (disp == 0) {
3270 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3271 __ ldaxr(dst_reg, rscratch1);
3272 } else {
3273 __ lea(rscratch1, Address(base, disp));
3274 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3275 __ ldaxr(dst_reg, rscratch1);
3276 }
3277 }
3278 %}
3279
3280 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3281 Register src_reg = as_Register($src$$reg);
3282 Register base = as_Register($mem$$base);
3283 int index = $mem$$index;
3284 int scale = $mem$$scale;
3285 int disp = $mem$$disp;
3286 if (index == -1) {
3287 if (disp != 0) {
3288 __ lea(rscratch2, Address(base, disp));
3289 __ stlxr(rscratch1, src_reg, rscratch2);
3290 } else {
3291 // TODO
3292 // should we ever get anything other than this case?
3293 __ stlxr(rscratch1, src_reg, base);
3294 }
3295 } else {
3296 Register index_reg = as_Register(index);
3297 if (disp == 0) {
3298 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3299 __ stlxr(rscratch1, src_reg, rscratch2);
3300 } else {
3301 __ lea(rscratch2, Address(base, disp));
3302 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3303 __ stlxr(rscratch1, src_reg, rscratch2);
3304 }
3305 }
3306 __ cmpw(rscratch1, zr);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ false, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::word, /*acquire*/ false, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::byte, /*acquire*/ false, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337
3338 // The only difference between aarch64_enc_cmpxchg and
3339 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3340 // CompareAndSwap sequence to serve as a barrier on acquiring a
3341 // lock.
3342 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3343 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3344 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3345 Assembler::xword, /*acquire*/ true, /*release*/ true,
3346 /*weak*/ false, noreg);
3347 %}
3348
3349 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3350 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3351 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3352 Assembler::word, /*acquire*/ true, /*release*/ true,
3353 /*weak*/ false, noreg);
3354 %}
3355
3356 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3357 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3358 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3359 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3360 /*weak*/ false, noreg);
3361 %}
3362
3363 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 Register res_reg = as_Register($res$$reg);
3373 __ cset(res_reg, Assembler::EQ);
3374 %}
3375
3376 // prefetch encodings
3377
3378 enc_class aarch64_enc_prefetchw(memory mem) %{
3379 Register base = as_Register($mem$$base);
3380 int index = $mem$$index;
3381 int scale = $mem$$scale;
3382 int disp = $mem$$disp;
3383 if (index == -1) {
3384 // Fix up any out-of-range offsets.
3385 assert_different_registers(rscratch1, base);
3386 Address addr = Address(base, disp);
3387 addr = __ legitimize_address(addr, 8, rscratch1);
3388 __ prfm(addr, PSTL1KEEP);
3389 } else {
3390 Register index_reg = as_Register(index);
3391 if (disp == 0) {
3392 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3393 } else {
3394 __ lea(rscratch1, Address(base, disp));
3395 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3396 }
3397 }
3398 %}
3399
3400 // mov encodings
3401
3402 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3403 uint32_t con = (uint32_t)$src$$constant;
3404 Register dst_reg = as_Register($dst$$reg);
3405 if (con == 0) {
3406 __ movw(dst_reg, zr);
3407 } else {
3408 __ movw(dst_reg, con);
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr || con == (address)1) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 if (rtype == relocInfo::oop_type) {
3430 __ movoop(dst_reg, (jobject)con);
3431 } else if (rtype == relocInfo::metadata_type) {
3432 __ mov_metadata(dst_reg, (Metadata*)con);
3433 } else {
3434 assert(rtype == relocInfo::none, "unexpected reloc type");
3435 if (! __ is_valid_AArch64_address(con) ||
3436 con < (address)(uintptr_t)os::vm_page_size()) {
3437 __ mov(dst_reg, con);
3438 } else {
3439 uint64_t offset;
3440 __ adrp(dst_reg, con, offset);
3441 __ add(dst_reg, dst_reg, offset);
3442 }
3443 }
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 __ mov(dst_reg, (uint64_t)1);
3455 %}
3456
3457 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3458 __ load_byte_map_base($dst$$Register);
3459 %}
3460
3461 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3462 Register dst_reg = as_Register($dst$$reg);
3463 address con = (address)$src$$constant;
3464 if (con == nullptr) {
3465 ShouldNotReachHere();
3466 } else {
3467 relocInfo::relocType rtype = $src->constant_reloc();
3468 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3469 __ set_narrow_oop(dst_reg, (jobject)con);
3470 }
3471 %}
3472
3473 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3474 Register dst_reg = as_Register($dst$$reg);
3475 __ mov(dst_reg, zr);
3476 %}
3477
3478 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 address con = (address)$src$$constant;
3481 if (con == nullptr) {
3482 ShouldNotReachHere();
3483 } else {
3484 relocInfo::relocType rtype = $src->constant_reloc();
3485 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3486 __ set_narrow_klass(dst_reg, (Klass *)con);
3487 }
3488 %}
3489
3490 // arithmetic encodings
3491
3492 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src_reg = as_Register($src1$$reg);
3495 int32_t con = (int32_t)$src2$$constant;
3496 // add has primary == 0, subtract has primary == 1
3497 if ($primary) { con = -con; }
3498 if (con < 0) {
3499 __ subw(dst_reg, src_reg, -con);
3500 } else {
3501 __ addw(dst_reg, src_reg, con);
3502 }
3503 %}
3504
3505 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 Register src_reg = as_Register($src1$$reg);
3508 int32_t con = (int32_t)$src2$$constant;
3509 // add has primary == 0, subtract has primary == 1
3510 if ($primary) { con = -con; }
3511 if (con < 0) {
3512 __ sub(dst_reg, src_reg, -con);
3513 } else {
3514 __ add(dst_reg, src_reg, con);
3515 }
3516 %}
3517
3518 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3519 Register dst_reg = as_Register($dst$$reg);
3520 Register src1_reg = as_Register($src1$$reg);
3521 Register src2_reg = as_Register($src2$$reg);
3522 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3523 %}
3524
3525 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3526 Register dst_reg = as_Register($dst$$reg);
3527 Register src1_reg = as_Register($src1$$reg);
3528 Register src2_reg = as_Register($src2$$reg);
3529 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3530 %}
3531
3532 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3533 Register dst_reg = as_Register($dst$$reg);
3534 Register src1_reg = as_Register($src1$$reg);
3535 Register src2_reg = as_Register($src2$$reg);
3536 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3537 %}
3538
3539 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3540 Register dst_reg = as_Register($dst$$reg);
3541 Register src1_reg = as_Register($src1$$reg);
3542 Register src2_reg = as_Register($src2$$reg);
3543 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3544 %}
3545
3546 // compare instruction encodings
3547
3548 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3549 Register reg1 = as_Register($src1$$reg);
3550 Register reg2 = as_Register($src2$$reg);
3551 __ cmpw(reg1, reg2);
3552 %}
3553
3554 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3555 Register reg = as_Register($src1$$reg);
3556 int32_t val = $src2$$constant;
3557 if (val >= 0) {
3558 __ subsw(zr, reg, val);
3559 } else {
3560 __ addsw(zr, reg, -val);
3561 }
3562 %}
3563
3564 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3565 Register reg1 = as_Register($src1$$reg);
3566 uint32_t val = (uint32_t)$src2$$constant;
3567 __ movw(rscratch1, val);
3568 __ cmpw(reg1, rscratch1);
3569 %}
3570
3571 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3572 Register reg1 = as_Register($src1$$reg);
3573 Register reg2 = as_Register($src2$$reg);
3574 __ cmp(reg1, reg2);
3575 %}
3576
3577 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3578 Register reg = as_Register($src1$$reg);
3579 int64_t val = $src2$$constant;
3580 if (val >= 0) {
3581 __ subs(zr, reg, val);
3582 } else if (val != -val) {
3583 __ adds(zr, reg, -val);
3584 } else {
3585 // aargh, Long.MIN_VALUE is a special case
3586 __ orr(rscratch1, zr, (uint64_t)val);
3587 __ subs(zr, reg, rscratch1);
3588 }
3589 %}
3590
3591 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3592 Register reg1 = as_Register($src1$$reg);
3593 uint64_t val = (uint64_t)$src2$$constant;
3594 __ mov(rscratch1, val);
3595 __ cmp(reg1, rscratch1);
3596 %}
3597
3598 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3599 Register reg1 = as_Register($src1$$reg);
3600 Register reg2 = as_Register($src2$$reg);
3601 __ cmp(reg1, reg2);
3602 %}
3603
3604 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3605 Register reg1 = as_Register($src1$$reg);
3606 Register reg2 = as_Register($src2$$reg);
3607 __ cmpw(reg1, reg2);
3608 %}
3609
3610 enc_class aarch64_enc_testp(iRegP src) %{
3611 Register reg = as_Register($src$$reg);
3612 __ cmp(reg, zr);
3613 %}
3614
3615 enc_class aarch64_enc_testn(iRegN src) %{
3616 Register reg = as_Register($src$$reg);
3617 __ cmpw(reg, zr);
3618 %}
3619
3620 enc_class aarch64_enc_b(label lbl) %{
3621 Label *L = $lbl$$label;
3622 __ b(*L);
3623 %}
3624
3625 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3626 Label *L = $lbl$$label;
3627 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3628 %}
3629
3630 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3631 Label *L = $lbl$$label;
3632 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3633 %}
3634
3635 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3636 %{
3637 Register sub_reg = as_Register($sub$$reg);
3638 Register super_reg = as_Register($super$$reg);
3639 Register temp_reg = as_Register($temp$$reg);
3640 Register result_reg = as_Register($result$$reg);
3641
3642 Label miss;
3643 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3644 nullptr, &miss,
3645 /*set_cond_codes:*/ true);
3646 if ($primary) {
3647 __ mov(result_reg, zr);
3648 }
3649 __ bind(miss);
3650 %}
3651
3652 enc_class aarch64_enc_java_static_call(method meth) %{
3653 address addr = (address)$meth$$method;
3654 address call;
3655 if (!_method) {
3656 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3657 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3658 if (call == nullptr) {
3659 ciEnv::current()->record_failure("CodeCache is full");
3660 return;
3661 }
3662 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3663 // The NOP here is purely to ensure that eliding a call to
3664 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3665 __ nop();
3666 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3667 } else {
3668 int method_index = resolved_method_index(masm);
3669 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3670 : static_call_Relocation::spec(method_index);
3671 call = __ trampoline_call(Address(addr, rspec));
3672 if (call == nullptr) {
3673 ciEnv::current()->record_failure("CodeCache is full");
3674 return;
3675 }
3676 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3677 // Calls of the same statically bound method can share
3678 // a stub to the interpreter.
3679 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3680 } else {
3681 // Emit stub for static call
3682 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3683 if (stub == nullptr) {
3684 ciEnv::current()->record_failure("CodeCache is full");
3685 return;
3686 }
3687 }
3688 }
3689
3690 __ post_call_nop();
3691
3692 // Only non uncommon_trap calls need to reinitialize ptrue.
3693 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3694 __ reinitialize_ptrue();
3695 }
3696 %}
3697
3698 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3699 int method_index = resolved_method_index(masm);
3700 address call = __ ic_call((address)$meth$$method, method_index);
3701 if (call == nullptr) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 __ post_call_nop();
3706 if (Compile::current()->max_vector_size() > 0) {
3707 __ reinitialize_ptrue();
3708 }
3709 %}
3710
3711 enc_class aarch64_enc_call_epilog() %{
3712 if (VerifyStackAtCalls) {
3713 // Check that stack depth is unchanged: find majik cookie on stack
3714 __ call_Unimplemented();
3715 }
3716 %}
3717
3718 enc_class aarch64_enc_java_to_runtime(method meth) %{
3719 // some calls to generated routines (arraycopy code) are scheduled
3720 // by C2 as runtime calls. if so we can call them using a br (they
3721 // will be in a reachable segment) otherwise we have to use a blr
3722 // which loads the absolute address into a register.
3723 address entry = (address)$meth$$method;
3724 CodeBlob *cb = CodeCache::find_blob(entry);
3725 if (cb) {
3726 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3727 if (call == nullptr) {
3728 ciEnv::current()->record_failure("CodeCache is full");
3729 return;
3730 }
3731 __ post_call_nop();
3732 } else {
3733 Label retaddr;
3734 // Make the anchor frame walkable
3735 __ adr(rscratch2, retaddr);
3736 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3737 __ lea(rscratch1, RuntimeAddress(entry));
3738 __ blr(rscratch1);
3739 __ bind(retaddr);
3740 __ post_call_nop();
3741 }
3742 if (Compile::current()->max_vector_size() > 0) {
3743 __ reinitialize_ptrue();
3744 }
3745 %}
3746
3747 enc_class aarch64_enc_rethrow() %{
3748 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3749 %}
3750
3751 enc_class aarch64_enc_ret() %{
3752 #ifdef ASSERT
3753 if (Compile::current()->max_vector_size() > 0) {
3754 __ verify_ptrue();
3755 }
3756 #endif
3757 __ ret(lr);
3758 %}
3759
3760 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3761 Register target_reg = as_Register($jump_target$$reg);
3762 __ br(target_reg);
3763 %}
3764
3765 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3766 Register target_reg = as_Register($jump_target$$reg);
3767 // exception oop should be in r0
3768 // ret addr has been popped into lr
3769 // callee expects it in r3
3770 __ mov(r3, lr);
3771 __ br(target_reg);
3772 %}
3773
3774 %}
3775
3776 //----------FRAME--------------------------------------------------------------
3777 // Definition of frame structure and management information.
3778 //
3779 // S T A C K L A Y O U T Allocators stack-slot number
3780 // | (to get allocators register number
3781 // G Owned by | | v add OptoReg::stack0())
3782 // r CALLER | |
3783 // o | +--------+ pad to even-align allocators stack-slot
3784 // w V | pad0 | numbers; owned by CALLER
3785 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3786 // h ^ | in | 5
3787 // | | args | 4 Holes in incoming args owned by SELF
3788 // | | | | 3
3789 // | | +--------+
3790 // V | | old out| Empty on Intel, window on Sparc
3791 // | old |preserve| Must be even aligned.
3792 // | SP-+--------+----> Matcher::_old_SP, even aligned
3793 // | | in | 3 area for Intel ret address
3794 // Owned by |preserve| Empty on Sparc.
3795 // SELF +--------+
3796 // | | pad2 | 2 pad to align old SP
3797 // | +--------+ 1
3798 // | | locks | 0
3799 // | +--------+----> OptoReg::stack0(), even aligned
3800 // | | pad1 | 11 pad to align new SP
3801 // | +--------+
3802 // | | | 10
3803 // | | spills | 9 spills
3804 // V | | 8 (pad0 slot for callee)
3805 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3806 // ^ | out | 7
3807 // | | args | 6 Holes in outgoing args owned by CALLEE
3808 // Owned by +--------+
3809 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3810 // | new |preserve| Must be even-aligned.
3811 // | SP-+--------+----> Matcher::_new_SP, even aligned
3812 // | | |
3813 //
3814 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3815 // known from SELF's arguments and the Java calling convention.
3816 // Region 6-7 is determined per call site.
3817 // Note 2: If the calling convention leaves holes in the incoming argument
3818 // area, those holes are owned by SELF. Holes in the outgoing area
3819 // are owned by the CALLEE. Holes should not be necessary in the
3820 // incoming area, as the Java calling convention is completely under
3821 // the control of the AD file. Doubles can be sorted and packed to
3822 // avoid holes. Holes in the outgoing arguments may be necessary for
3823 // varargs C calling conventions.
3824 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3825 // even aligned with pad0 as needed.
3826 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3827 // (the latter is true on Intel but is it false on AArch64?)
3828 // region 6-11 is even aligned; it may be padded out more so that
3829 // the region from SP to FP meets the minimum stack alignment.
3830 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3831 // alignment. Region 11, pad1, may be dynamically extended so that
3832 // SP meets the minimum alignment.
3833
3834 frame %{
3835 // These three registers define part of the calling convention
3836 // between compiled code and the interpreter.
3837
3838 // Inline Cache Register or Method for I2C.
3839 inline_cache_reg(R12);
3840
3841 // Number of stack slots consumed by locking an object
3842 sync_stack_slots(2);
3843
3844 // Compiled code's Frame Pointer
3845 frame_pointer(R31);
3846
3847 // Interpreter stores its frame pointer in a register which is
3848 // stored to the stack by I2CAdaptors.
3849 // I2CAdaptors convert from interpreted java to compiled java.
3850 interpreter_frame_pointer(R29);
3851
3852 // Stack alignment requirement
3853 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3854
3855 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3856 // for calls to C. Supports the var-args backing area for register parms.
3857 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3858
3859 // The after-PROLOG location of the return address. Location of
3860 // return address specifies a type (REG or STACK) and a number
3861 // representing the register number (i.e. - use a register name) or
3862 // stack slot.
3863 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3864 // Otherwise, it is above the locks and verification slot and alignment word
3865 // TODO this may well be correct but need to check why that - 2 is there
3866 // ppc port uses 0 but we definitely need to allow for fixed_slots
3867 // which folds in the space used for monitors
3868 return_addr(STACK - 2 +
3869 align_up((Compile::current()->in_preserve_stack_slots() +
3870 Compile::current()->fixed_slots()),
3871 stack_alignment_in_slots()));
3872
3873 // Location of compiled Java return values. Same as C for now.
3874 return_value
3875 %{
3876 // TODO do we allow ideal_reg == Op_RegN???
3877 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3878 "only return normal values");
3879
3880 static const int lo[Op_RegL + 1] = { // enum name
3881 0, // Op_Node
3882 0, // Op_Set
3883 R0_num, // Op_RegN
3884 R0_num, // Op_RegI
3885 R0_num, // Op_RegP
3886 V0_num, // Op_RegF
3887 V0_num, // Op_RegD
3888 R0_num // Op_RegL
3889 };
3890
3891 static const int hi[Op_RegL + 1] = { // enum name
3892 0, // Op_Node
3893 0, // Op_Set
3894 OptoReg::Bad, // Op_RegN
3895 OptoReg::Bad, // Op_RegI
3896 R0_H_num, // Op_RegP
3897 OptoReg::Bad, // Op_RegF
3898 V0_H_num, // Op_RegD
3899 R0_H_num // Op_RegL
3900 };
3901
3902 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3903 %}
3904 %}
3905
3906 //----------ATTRIBUTES---------------------------------------------------------
3907 //----------Operand Attributes-------------------------------------------------
3908 op_attrib op_cost(1); // Required cost attribute
3909
3910 //----------Instruction Attributes---------------------------------------------
3911 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3912 ins_attrib ins_size(32); // Required size attribute (in bits)
3913 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3914 // a non-matching short branch variant
3915 // of some long branch?
3916 ins_attrib ins_alignment(4); // Required alignment attribute (must
3917 // be a power of 2) specifies the
3918 // alignment that some part of the
3919 // instruction (not necessarily the
3920 // start) requires. If > 1, a
3921 // compute_padding() function must be
3922 // provided for the instruction
3923
3924 //----------OPERANDS-----------------------------------------------------------
3925 // Operand definitions must precede instruction definitions for correct parsing
3926 // in the ADLC because operands constitute user defined types which are used in
3927 // instruction definitions.
3928
3929 //----------Simple Operands----------------------------------------------------
3930
3931 // Integer operands 32 bit
3932 // 32 bit immediate
3933 operand immI()
3934 %{
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 // 32 bit zero
3943 operand immI0()
3944 %{
3945 predicate(n->get_int() == 0);
3946 match(ConI);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 // 32 bit unit increment
3954 operand immI_1()
3955 %{
3956 predicate(n->get_int() == 1);
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 // 32 bit unit decrement
3965 operand immI_M1()
3966 %{
3967 predicate(n->get_int() == -1);
3968 match(ConI);
3969
3970 op_cost(0);
3971 format %{ %}
3972 interface(CONST_INTER);
3973 %}
3974
3975 // Shift values for add/sub extension shift
3976 operand immIExt()
3977 %{
3978 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3979 match(ConI);
3980
3981 op_cost(0);
3982 format %{ %}
3983 interface(CONST_INTER);
3984 %}
3985
3986 operand immI_gt_1()
3987 %{
3988 predicate(n->get_int() > 1);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 operand immI_le_4()
3997 %{
3998 predicate(n->get_int() <= 4);
3999 match(ConI);
4000
4001 op_cost(0);
4002 format %{ %}
4003 interface(CONST_INTER);
4004 %}
4005
4006 operand immI_16()
4007 %{
4008 predicate(n->get_int() == 16);
4009 match(ConI);
4010
4011 op_cost(0);
4012 format %{ %}
4013 interface(CONST_INTER);
4014 %}
4015
4016 operand immI_24()
4017 %{
4018 predicate(n->get_int() == 24);
4019 match(ConI);
4020
4021 op_cost(0);
4022 format %{ %}
4023 interface(CONST_INTER);
4024 %}
4025
4026 operand immI_32()
4027 %{
4028 predicate(n->get_int() == 32);
4029 match(ConI);
4030
4031 op_cost(0);
4032 format %{ %}
4033 interface(CONST_INTER);
4034 %}
4035
4036 operand immI_48()
4037 %{
4038 predicate(n->get_int() == 48);
4039 match(ConI);
4040
4041 op_cost(0);
4042 format %{ %}
4043 interface(CONST_INTER);
4044 %}
4045
4046 operand immI_56()
4047 %{
4048 predicate(n->get_int() == 56);
4049 match(ConI);
4050
4051 op_cost(0);
4052 format %{ %}
4053 interface(CONST_INTER);
4054 %}
4055
4056 operand immI_255()
4057 %{
4058 predicate(n->get_int() == 255);
4059 match(ConI);
4060
4061 op_cost(0);
4062 format %{ %}
4063 interface(CONST_INTER);
4064 %}
4065
4066 operand immI_65535()
4067 %{
4068 predicate(n->get_int() == 65535);
4069 match(ConI);
4070
4071 op_cost(0);
4072 format %{ %}
4073 interface(CONST_INTER);
4074 %}
4075
4076 operand immI_positive()
4077 %{
4078 predicate(n->get_int() > 0);
4079 match(ConI);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 // BoolTest condition for signed compare
4087 operand immI_cmp_cond()
4088 %{
4089 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4090 match(ConI);
4091
4092 op_cost(0);
4093 format %{ %}
4094 interface(CONST_INTER);
4095 %}
4096
4097 // BoolTest condition for unsigned compare
4098 operand immI_cmpU_cond()
4099 %{
4100 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immL_255()
4109 %{
4110 predicate(n->get_long() == 255L);
4111 match(ConL);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 operand immL_65535()
4119 %{
4120 predicate(n->get_long() == 65535L);
4121 match(ConL);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immL_4294967295()
4129 %{
4130 predicate(n->get_long() == 4294967295L);
4131 match(ConL);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 operand immL_bitmask()
4139 %{
4140 predicate((n->get_long() != 0)
4141 && ((n->get_long() & 0xc000000000000000l) == 0)
4142 && is_power_of_2(n->get_long() + 1));
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immI_bitmask()
4151 %{
4152 predicate((n->get_int() != 0)
4153 && ((n->get_int() & 0xc0000000) == 0)
4154 && is_power_of_2(n->get_int() + 1));
4155 match(ConI);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 operand immL_positive_bitmaskI()
4163 %{
4164 predicate((n->get_long() != 0)
4165 && ((julong)n->get_long() < 0x80000000ULL)
4166 && is_power_of_2(n->get_long() + 1));
4167 match(ConL);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 // Scale values for scaled offset addressing modes (up to long but not quad)
4175 operand immIScale()
4176 %{
4177 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4178 match(ConI);
4179
4180 op_cost(0);
4181 format %{ %}
4182 interface(CONST_INTER);
4183 %}
4184
4185 // 5 bit signed integer
4186 operand immI5()
4187 %{
4188 predicate(Assembler::is_simm(n->get_int(), 5));
4189 match(ConI);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 // 7 bit unsigned integer
4197 operand immIU7()
4198 %{
4199 predicate(Assembler::is_uimm(n->get_int(), 7));
4200 match(ConI);
4201
4202 op_cost(0);
4203 format %{ %}
4204 interface(CONST_INTER);
4205 %}
4206
4207 // Offset for scaled or unscaled immediate loads and stores
4208 operand immIOffset()
4209 %{
4210 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4211 match(ConI);
4212
4213 op_cost(0);
4214 format %{ %}
4215 interface(CONST_INTER);
4216 %}
4217
4218 operand immIOffset1()
4219 %{
4220 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 operand immIOffset2()
4229 %{
4230 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 operand immIOffset4()
4239 %{
4240 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 operand immIOffset8()
4249 %{
4250 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 operand immIOffset16()
4259 %{
4260 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4261 match(ConI);
4262
4263 op_cost(0);
4264 format %{ %}
4265 interface(CONST_INTER);
4266 %}
4267
4268 operand immLOffset()
4269 %{
4270 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4271 match(ConL);
4272
4273 op_cost(0);
4274 format %{ %}
4275 interface(CONST_INTER);
4276 %}
4277
4278 operand immLoffset1()
4279 %{
4280 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4281 match(ConL);
4282
4283 op_cost(0);
4284 format %{ %}
4285 interface(CONST_INTER);
4286 %}
4287
4288 operand immLoffset2()
4289 %{
4290 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4291 match(ConL);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immLoffset4()
4299 %{
4300 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 operand immLoffset8()
4309 %{
4310 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 operand immLoffset16()
4319 %{
4320 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 // 5 bit signed long integer
4329 operand immL5()
4330 %{
4331 predicate(Assembler::is_simm(n->get_long(), 5));
4332 match(ConL);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 // 7 bit unsigned long integer
4340 operand immLU7()
4341 %{
4342 predicate(Assembler::is_uimm(n->get_long(), 7));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 // 8 bit signed value.
4351 operand immI8()
4352 %{
4353 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4354 match(ConI);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 8 bit signed value (simm8), or #simm8 LSL 8.
4362 operand immI8_shift8()
4363 %{
4364 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4365 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4366 match(ConI);
4367
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // 8 bit signed value (simm8), or #simm8 LSL 8.
4374 operand immL8_shift8()
4375 %{
4376 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4377 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4378 match(ConL);
4379
4380 op_cost(0);
4381 format %{ %}
4382 interface(CONST_INTER);
4383 %}
4384
4385 // 8 bit integer valid for vector add sub immediate
4386 operand immBAddSubV()
4387 %{
4388 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4389 match(ConI);
4390
4391 op_cost(0);
4392 format %{ %}
4393 interface(CONST_INTER);
4394 %}
4395
4396 // 32 bit integer valid for add sub immediate
4397 operand immIAddSub()
4398 %{
4399 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4400 match(ConI);
4401 op_cost(0);
4402 format %{ %}
4403 interface(CONST_INTER);
4404 %}
4405
4406 // 32 bit integer valid for vector add sub immediate
4407 operand immIAddSubV()
4408 %{
4409 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4410 match(ConI);
4411
4412 op_cost(0);
4413 format %{ %}
4414 interface(CONST_INTER);
4415 %}
4416
4417 // 32 bit unsigned integer valid for logical immediate
4418
4419 operand immBLog()
4420 %{
4421 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4422 match(ConI);
4423
4424 op_cost(0);
4425 format %{ %}
4426 interface(CONST_INTER);
4427 %}
4428
4429 operand immSLog()
4430 %{
4431 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 operand immILog()
4440 %{
4441 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4442 match(ConI);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // Integer operands 64 bit
4450 // 64 bit immediate
4451 operand immL()
4452 %{
4453 match(ConL);
4454
4455 op_cost(0);
4456 format %{ %}
4457 interface(CONST_INTER);
4458 %}
4459
4460 // 64 bit zero
4461 operand immL0()
4462 %{
4463 predicate(n->get_long() == 0);
4464 match(ConL);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 // 64 bit unit decrement
4472 operand immL_M1()
4473 %{
4474 predicate(n->get_long() == -1);
4475 match(ConL);
4476
4477 op_cost(0);
4478 format %{ %}
4479 interface(CONST_INTER);
4480 %}
4481
4482 // 64 bit integer valid for add sub immediate
4483 operand immLAddSub()
4484 %{
4485 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4486 match(ConL);
4487 op_cost(0);
4488 format %{ %}
4489 interface(CONST_INTER);
4490 %}
4491
4492 // 64 bit integer valid for addv subv immediate
4493 operand immLAddSubV()
4494 %{
4495 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4496 match(ConL);
4497
4498 op_cost(0);
4499 format %{ %}
4500 interface(CONST_INTER);
4501 %}
4502
4503 // 64 bit integer valid for logical immediate
4504 operand immLLog()
4505 %{
4506 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4507 match(ConL);
4508 op_cost(0);
4509 format %{ %}
4510 interface(CONST_INTER);
4511 %}
4512
4513 // Long Immediate: low 32-bit mask
4514 operand immL_32bits()
4515 %{
4516 predicate(n->get_long() == 0xFFFFFFFFL);
4517 match(ConL);
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // Pointer operands
4524 // Pointer Immediate
4525 operand immP()
4526 %{
4527 match(ConP);
4528
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // nullptr Pointer Immediate
4535 operand immP0()
4536 %{
4537 predicate(n->get_ptr() == 0);
4538 match(ConP);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // Pointer Immediate One
4546 // this is used in object initialization (initial object header)
4547 operand immP_1()
4548 %{
4549 predicate(n->get_ptr() == 1);
4550 match(ConP);
4551
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // Card Table Byte Map Base
4558 operand immByteMapBase()
4559 %{
4560 // Get base of card map
4561 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4562 SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
4563 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4564 match(ConP);
4565
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // Float and Double operands
4572 // Double Immediate
4573 operand immD()
4574 %{
4575 match(ConD);
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Double Immediate: +0.0d
4582 operand immD0()
4583 %{
4584 predicate(jlong_cast(n->getd()) == 0);
4585 match(ConD);
4586
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 // constant 'double +0.0'.
4593 operand immDPacked()
4594 %{
4595 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4596 match(ConD);
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 // Float Immediate
4603 operand immF()
4604 %{
4605 match(ConF);
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Float Immediate: +0.0f.
4612 operand immF0()
4613 %{
4614 predicate(jint_cast(n->getf()) == 0);
4615 match(ConF);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 // Half Float (FP16) Immediate
4623 operand immH()
4624 %{
4625 match(ConH);
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 //
4632 operand immFPacked()
4633 %{
4634 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4635 match(ConF);
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Narrow pointer operands
4642 // Narrow Pointer Immediate
4643 operand immN()
4644 %{
4645 match(ConN);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 // Narrow nullptr Pointer Immediate
4653 operand immN0()
4654 %{
4655 predicate(n->get_narrowcon() == 0);
4656 match(ConN);
4657
4658 op_cost(0);
4659 format %{ %}
4660 interface(CONST_INTER);
4661 %}
4662
4663 operand immNKlass()
4664 %{
4665 match(ConNKlass);
4666
4667 op_cost(0);
4668 format %{ %}
4669 interface(CONST_INTER);
4670 %}
4671
4672 // Integer 32 bit Register Operands
4673 // Integer 32 bitRegister (excludes SP)
4674 operand iRegI()
4675 %{
4676 constraint(ALLOC_IN_RC(any_reg32));
4677 match(RegI);
4678 match(iRegINoSp);
4679 op_cost(0);
4680 format %{ %}
4681 interface(REG_INTER);
4682 %}
4683
4684 // Integer 32 bit Register not Special
4685 operand iRegINoSp()
4686 %{
4687 constraint(ALLOC_IN_RC(no_special_reg32));
4688 match(RegI);
4689 op_cost(0);
4690 format %{ %}
4691 interface(REG_INTER);
4692 %}
4693
4694 // Integer 64 bit Register Operands
4695 // Integer 64 bit Register (includes SP)
4696 operand iRegL()
4697 %{
4698 constraint(ALLOC_IN_RC(any_reg));
4699 match(RegL);
4700 match(iRegLNoSp);
4701 op_cost(0);
4702 format %{ %}
4703 interface(REG_INTER);
4704 %}
4705
4706 // Integer 64 bit Register not Special
4707 operand iRegLNoSp()
4708 %{
4709 constraint(ALLOC_IN_RC(no_special_reg));
4710 match(RegL);
4711 match(iRegL_R0);
4712 format %{ %}
4713 interface(REG_INTER);
4714 %}
4715
4716 // Pointer Register Operands
4717 // Pointer Register
4718 operand iRegP()
4719 %{
4720 constraint(ALLOC_IN_RC(ptr_reg));
4721 match(RegP);
4722 match(iRegPNoSp);
4723 match(iRegP_R0);
4724 //match(iRegP_R2);
4725 //match(iRegP_R4);
4726 match(iRegP_R5);
4727 match(thread_RegP);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Pointer 64 bit Register not Special
4734 operand iRegPNoSp()
4735 %{
4736 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4737 match(RegP);
4738 // match(iRegP);
4739 // match(iRegP_R0);
4740 // match(iRegP_R2);
4741 // match(iRegP_R4);
4742 // match(iRegP_R5);
4743 // match(thread_RegP);
4744 op_cost(0);
4745 format %{ %}
4746 interface(REG_INTER);
4747 %}
4748
4749 // This operand is not allowed to use rfp even if
4750 // rfp is not used to hold the frame pointer.
4751 operand iRegPNoSpNoRfp()
4752 %{
4753 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4754 match(RegP);
4755 match(iRegPNoSp);
4756 op_cost(0);
4757 format %{ %}
4758 interface(REG_INTER);
4759 %}
4760
4761 // Pointer 64 bit Register R0 only
4762 operand iRegP_R0()
4763 %{
4764 constraint(ALLOC_IN_RC(r0_reg));
4765 match(RegP);
4766 // match(iRegP);
4767 match(iRegPNoSp);
4768 op_cost(0);
4769 format %{ %}
4770 interface(REG_INTER);
4771 %}
4772
4773 // Pointer 64 bit Register R1 only
4774 operand iRegP_R1()
4775 %{
4776 constraint(ALLOC_IN_RC(r1_reg));
4777 match(RegP);
4778 // match(iRegP);
4779 match(iRegPNoSp);
4780 op_cost(0);
4781 format %{ %}
4782 interface(REG_INTER);
4783 %}
4784
4785 // Pointer 64 bit Register R2 only
4786 operand iRegP_R2()
4787 %{
4788 constraint(ALLOC_IN_RC(r2_reg));
4789 match(RegP);
4790 // match(iRegP);
4791 match(iRegPNoSp);
4792 op_cost(0);
4793 format %{ %}
4794 interface(REG_INTER);
4795 %}
4796
4797 // Pointer 64 bit Register R3 only
4798 operand iRegP_R3()
4799 %{
4800 constraint(ALLOC_IN_RC(r3_reg));
4801 match(RegP);
4802 // match(iRegP);
4803 match(iRegPNoSp);
4804 op_cost(0);
4805 format %{ %}
4806 interface(REG_INTER);
4807 %}
4808
4809 // Pointer 64 bit Register R4 only
4810 operand iRegP_R4()
4811 %{
4812 constraint(ALLOC_IN_RC(r4_reg));
4813 match(RegP);
4814 // match(iRegP);
4815 match(iRegPNoSp);
4816 op_cost(0);
4817 format %{ %}
4818 interface(REG_INTER);
4819 %}
4820
4821 // Pointer 64 bit Register R5 only
4822 operand iRegP_R5()
4823 %{
4824 constraint(ALLOC_IN_RC(r5_reg));
4825 match(RegP);
4826 // match(iRegP);
4827 match(iRegPNoSp);
4828 op_cost(0);
4829 format %{ %}
4830 interface(REG_INTER);
4831 %}
4832
4833 // Pointer 64 bit Register R10 only
4834 operand iRegP_R10()
4835 %{
4836 constraint(ALLOC_IN_RC(r10_reg));
4837 match(RegP);
4838 // match(iRegP);
4839 match(iRegPNoSp);
4840 op_cost(0);
4841 format %{ %}
4842 interface(REG_INTER);
4843 %}
4844
4845 // Long 64 bit Register R0 only
4846 operand iRegL_R0()
4847 %{
4848 constraint(ALLOC_IN_RC(r0_reg));
4849 match(RegL);
4850 match(iRegLNoSp);
4851 op_cost(0);
4852 format %{ %}
4853 interface(REG_INTER);
4854 %}
4855
4856 // Long 64 bit Register R11 only
4857 operand iRegL_R11()
4858 %{
4859 constraint(ALLOC_IN_RC(r11_reg));
4860 match(RegL);
4861 match(iRegLNoSp);
4862 op_cost(0);
4863 format %{ %}
4864 interface(REG_INTER);
4865 %}
4866
4867 // Register R0 only
4868 operand iRegI_R0()
4869 %{
4870 constraint(ALLOC_IN_RC(int_r0_reg));
4871 match(RegI);
4872 match(iRegINoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Register R2 only
4879 operand iRegI_R2()
4880 %{
4881 constraint(ALLOC_IN_RC(int_r2_reg));
4882 match(RegI);
4883 match(iRegINoSp);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Register R3 only
4890 operand iRegI_R3()
4891 %{
4892 constraint(ALLOC_IN_RC(int_r3_reg));
4893 match(RegI);
4894 match(iRegINoSp);
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900
4901 // Register R4 only
4902 operand iRegI_R4()
4903 %{
4904 constraint(ALLOC_IN_RC(int_r4_reg));
4905 match(RegI);
4906 match(iRegINoSp);
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912
4913 // Pointer Register Operands
4914 // Narrow Pointer Register
4915 operand iRegN()
4916 %{
4917 constraint(ALLOC_IN_RC(any_reg32));
4918 match(RegN);
4919 match(iRegNNoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925 // Integer 64 bit Register not Special
4926 operand iRegNNoSp()
4927 %{
4928 constraint(ALLOC_IN_RC(no_special_reg32));
4929 match(RegN);
4930 op_cost(0);
4931 format %{ %}
4932 interface(REG_INTER);
4933 %}
4934
4935 // Float Register
4936 // Float register operands
4937 operand vRegF()
4938 %{
4939 constraint(ALLOC_IN_RC(float_reg));
4940 match(RegF);
4941
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947 // Double Register
4948 // Double register operands
4949 operand vRegD()
4950 %{
4951 constraint(ALLOC_IN_RC(double_reg));
4952 match(RegD);
4953
4954 op_cost(0);
4955 format %{ %}
4956 interface(REG_INTER);
4957 %}
4958
4959 // Generic vector class. This will be used for
4960 // all vector operands, including NEON and SVE.
4961 operand vReg()
4962 %{
4963 constraint(ALLOC_IN_RC(dynamic));
4964 match(VecA);
4965 match(VecD);
4966 match(VecX);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vecA()
4974 %{
4975 constraint(ALLOC_IN_RC(vectora_reg));
4976 match(VecA);
4977
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 operand vecD()
4984 %{
4985 constraint(ALLOC_IN_RC(vectord_reg));
4986 match(VecD);
4987
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vecX()
4994 %{
4995 constraint(ALLOC_IN_RC(vectorx_reg));
4996 match(VecX);
4997
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vRegD_V0()
5004 %{
5005 constraint(ALLOC_IN_RC(v0_reg));
5006 match(RegD);
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vRegD_V1()
5013 %{
5014 constraint(ALLOC_IN_RC(v1_reg));
5015 match(RegD);
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 operand vRegD_V2()
5022 %{
5023 constraint(ALLOC_IN_RC(v2_reg));
5024 match(RegD);
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vRegD_V3()
5031 %{
5032 constraint(ALLOC_IN_RC(v3_reg));
5033 match(RegD);
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand vRegD_V4()
5040 %{
5041 constraint(ALLOC_IN_RC(v4_reg));
5042 match(RegD);
5043 op_cost(0);
5044 format %{ %}
5045 interface(REG_INTER);
5046 %}
5047
5048 operand vRegD_V5()
5049 %{
5050 constraint(ALLOC_IN_RC(v5_reg));
5051 match(RegD);
5052 op_cost(0);
5053 format %{ %}
5054 interface(REG_INTER);
5055 %}
5056
5057 operand vRegD_V6()
5058 %{
5059 constraint(ALLOC_IN_RC(v6_reg));
5060 match(RegD);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 operand vRegD_V7()
5067 %{
5068 constraint(ALLOC_IN_RC(v7_reg));
5069 match(RegD);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 operand vRegD_V12()
5076 %{
5077 constraint(ALLOC_IN_RC(v12_reg));
5078 match(RegD);
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 operand vRegD_V13()
5085 %{
5086 constraint(ALLOC_IN_RC(v13_reg));
5087 match(RegD);
5088 op_cost(0);
5089 format %{ %}
5090 interface(REG_INTER);
5091 %}
5092
5093 operand pReg()
5094 %{
5095 constraint(ALLOC_IN_RC(pr_reg));
5096 match(RegVectMask);
5097 match(pRegGov);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 operand pRegGov()
5104 %{
5105 constraint(ALLOC_IN_RC(gov_pr));
5106 match(RegVectMask);
5107 match(pReg);
5108 op_cost(0);
5109 format %{ %}
5110 interface(REG_INTER);
5111 %}
5112
5113 operand pRegGov_P0()
5114 %{
5115 constraint(ALLOC_IN_RC(p0_reg));
5116 match(RegVectMask);
5117 op_cost(0);
5118 format %{ %}
5119 interface(REG_INTER);
5120 %}
5121
5122 operand pRegGov_P1()
5123 %{
5124 constraint(ALLOC_IN_RC(p1_reg));
5125 match(RegVectMask);
5126 op_cost(0);
5127 format %{ %}
5128 interface(REG_INTER);
5129 %}
5130
5131 // Flags register, used as output of signed compare instructions
5132
5133 // note that on AArch64 we also use this register as the output for
5134 // for floating point compare instructions (CmpF CmpD). this ensures
5135 // that ordered inequality tests use GT, GE, LT or LE none of which
5136 // pass through cases where the result is unordered i.e. one or both
5137 // inputs to the compare is a NaN. this means that the ideal code can
5138 // replace e.g. a GT with an LE and not end up capturing the NaN case
5139 // (where the comparison should always fail). EQ and NE tests are
5140 // always generated in ideal code so that unordered folds into the NE
5141 // case, matching the behaviour of AArch64 NE.
5142 //
5143 // This differs from x86 where the outputs of FP compares use a
5144 // special FP flags registers and where compares based on this
5145 // register are distinguished into ordered inequalities (cmpOpUCF) and
5146 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5147 // to explicitly handle the unordered case in branches. x86 also has
5148 // to include extra CMoveX rules to accept a cmpOpUCF input.
5149
5150 operand rFlagsReg()
5151 %{
5152 constraint(ALLOC_IN_RC(int_flags));
5153 match(RegFlags);
5154
5155 op_cost(0);
5156 format %{ "RFLAGS" %}
5157 interface(REG_INTER);
5158 %}
5159
5160 // Flags register, used as output of unsigned compare instructions
5161 operand rFlagsRegU()
5162 %{
5163 constraint(ALLOC_IN_RC(int_flags));
5164 match(RegFlags);
5165
5166 op_cost(0);
5167 format %{ "RFLAGSU" %}
5168 interface(REG_INTER);
5169 %}
5170
5171 // Special Registers
5172
5173 // Method Register
5174 operand inline_cache_RegP(iRegP reg)
5175 %{
5176 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5177 match(reg);
5178 match(iRegPNoSp);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 // Thread Register
5185 operand thread_RegP(iRegP reg)
5186 %{
5187 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5188 match(reg);
5189 op_cost(0);
5190 format %{ %}
5191 interface(REG_INTER);
5192 %}
5193
5194 //----------Memory Operands----------------------------------------------------
5195
5196 operand indirect(iRegP reg)
5197 %{
5198 constraint(ALLOC_IN_RC(ptr_reg));
5199 match(reg);
5200 op_cost(0);
5201 format %{ "[$reg]" %}
5202 interface(MEMORY_INTER) %{
5203 base($reg);
5204 index(0xffffffff);
5205 scale(0x0);
5206 disp(0x0);
5207 %}
5208 %}
5209
5210 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5211 %{
5212 constraint(ALLOC_IN_RC(ptr_reg));
5213 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5214 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5215 op_cost(0);
5216 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5217 interface(MEMORY_INTER) %{
5218 base($reg);
5219 index($ireg);
5220 scale($scale);
5221 disp(0x0);
5222 %}
5223 %}
5224
5225 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5226 %{
5227 constraint(ALLOC_IN_RC(ptr_reg));
5228 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5229 match(AddP reg (LShiftL lreg scale));
5230 op_cost(0);
5231 format %{ "$reg, $lreg lsl($scale)" %}
5232 interface(MEMORY_INTER) %{
5233 base($reg);
5234 index($lreg);
5235 scale($scale);
5236 disp(0x0);
5237 %}
5238 %}
5239
5240 operand indIndexI2L(iRegP reg, iRegI ireg)
5241 %{
5242 constraint(ALLOC_IN_RC(ptr_reg));
5243 match(AddP reg (ConvI2L ireg));
5244 op_cost(0);
5245 format %{ "$reg, $ireg, 0, I2L" %}
5246 interface(MEMORY_INTER) %{
5247 base($reg);
5248 index($ireg);
5249 scale(0x0);
5250 disp(0x0);
5251 %}
5252 %}
5253
5254 operand indIndex(iRegP reg, iRegL lreg)
5255 %{
5256 constraint(ALLOC_IN_RC(ptr_reg));
5257 match(AddP reg lreg);
5258 op_cost(0);
5259 format %{ "$reg, $lreg" %}
5260 interface(MEMORY_INTER) %{
5261 base($reg);
5262 index($lreg);
5263 scale(0x0);
5264 disp(0x0);
5265 %}
5266 %}
5267
5268 operand indOffI1(iRegP reg, immIOffset1 off)
5269 %{
5270 constraint(ALLOC_IN_RC(ptr_reg));
5271 match(AddP reg off);
5272 op_cost(0);
5273 format %{ "[$reg, $off]" %}
5274 interface(MEMORY_INTER) %{
5275 base($reg);
5276 index(0xffffffff);
5277 scale(0x0);
5278 disp($off);
5279 %}
5280 %}
5281
5282 operand indOffI2(iRegP reg, immIOffset2 off)
5283 %{
5284 constraint(ALLOC_IN_RC(ptr_reg));
5285 match(AddP reg off);
5286 op_cost(0);
5287 format %{ "[$reg, $off]" %}
5288 interface(MEMORY_INTER) %{
5289 base($reg);
5290 index(0xffffffff);
5291 scale(0x0);
5292 disp($off);
5293 %}
5294 %}
5295
5296 operand indOffI4(iRegP reg, immIOffset4 off)
5297 %{
5298 constraint(ALLOC_IN_RC(ptr_reg));
5299 match(AddP reg off);
5300 op_cost(0);
5301 format %{ "[$reg, $off]" %}
5302 interface(MEMORY_INTER) %{
5303 base($reg);
5304 index(0xffffffff);
5305 scale(0x0);
5306 disp($off);
5307 %}
5308 %}
5309
5310 operand indOffI8(iRegP reg, immIOffset8 off)
5311 %{
5312 constraint(ALLOC_IN_RC(ptr_reg));
5313 match(AddP reg off);
5314 op_cost(0);
5315 format %{ "[$reg, $off]" %}
5316 interface(MEMORY_INTER) %{
5317 base($reg);
5318 index(0xffffffff);
5319 scale(0x0);
5320 disp($off);
5321 %}
5322 %}
5323
5324 operand indOffI16(iRegP reg, immIOffset16 off)
5325 %{
5326 constraint(ALLOC_IN_RC(ptr_reg));
5327 match(AddP reg off);
5328 op_cost(0);
5329 format %{ "[$reg, $off]" %}
5330 interface(MEMORY_INTER) %{
5331 base($reg);
5332 index(0xffffffff);
5333 scale(0x0);
5334 disp($off);
5335 %}
5336 %}
5337
5338 operand indOffL1(iRegP reg, immLoffset1 off)
5339 %{
5340 constraint(ALLOC_IN_RC(ptr_reg));
5341 match(AddP reg off);
5342 op_cost(0);
5343 format %{ "[$reg, $off]" %}
5344 interface(MEMORY_INTER) %{
5345 base($reg);
5346 index(0xffffffff);
5347 scale(0x0);
5348 disp($off);
5349 %}
5350 %}
5351
5352 operand indOffL2(iRegP reg, immLoffset2 off)
5353 %{
5354 constraint(ALLOC_IN_RC(ptr_reg));
5355 match(AddP reg off);
5356 op_cost(0);
5357 format %{ "[$reg, $off]" %}
5358 interface(MEMORY_INTER) %{
5359 base($reg);
5360 index(0xffffffff);
5361 scale(0x0);
5362 disp($off);
5363 %}
5364 %}
5365
5366 operand indOffL4(iRegP reg, immLoffset4 off)
5367 %{
5368 constraint(ALLOC_IN_RC(ptr_reg));
5369 match(AddP reg off);
5370 op_cost(0);
5371 format %{ "[$reg, $off]" %}
5372 interface(MEMORY_INTER) %{
5373 base($reg);
5374 index(0xffffffff);
5375 scale(0x0);
5376 disp($off);
5377 %}
5378 %}
5379
5380 operand indOffL8(iRegP reg, immLoffset8 off)
5381 %{
5382 constraint(ALLOC_IN_RC(ptr_reg));
5383 match(AddP reg off);
5384 op_cost(0);
5385 format %{ "[$reg, $off]" %}
5386 interface(MEMORY_INTER) %{
5387 base($reg);
5388 index(0xffffffff);
5389 scale(0x0);
5390 disp($off);
5391 %}
5392 %}
5393
5394 operand indOffL16(iRegP reg, immLoffset16 off)
5395 %{
5396 constraint(ALLOC_IN_RC(ptr_reg));
5397 match(AddP reg off);
5398 op_cost(0);
5399 format %{ "[$reg, $off]" %}
5400 interface(MEMORY_INTER) %{
5401 base($reg);
5402 index(0xffffffff);
5403 scale(0x0);
5404 disp($off);
5405 %}
5406 %}
5407
5408 operand indirectX2P(iRegL reg)
5409 %{
5410 constraint(ALLOC_IN_RC(ptr_reg));
5411 match(CastX2P reg);
5412 op_cost(0);
5413 format %{ "[$reg]\t# long -> ptr" %}
5414 interface(MEMORY_INTER) %{
5415 base($reg);
5416 index(0xffffffff);
5417 scale(0x0);
5418 disp(0x0);
5419 %}
5420 %}
5421
5422 operand indOffX2P(iRegL reg, immLOffset off)
5423 %{
5424 constraint(ALLOC_IN_RC(ptr_reg));
5425 match(AddP (CastX2P reg) off);
5426 op_cost(0);
5427 format %{ "[$reg, $off]\t# long -> ptr" %}
5428 interface(MEMORY_INTER) %{
5429 base($reg);
5430 index(0xffffffff);
5431 scale(0x0);
5432 disp($off);
5433 %}
5434 %}
5435
5436 operand indirectN(iRegN reg)
5437 %{
5438 predicate(CompressedOops::shift() == 0);
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(DecodeN reg);
5441 op_cost(0);
5442 format %{ "[$reg]\t# narrow" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp(0x0);
5448 %}
5449 %}
5450
5451 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5452 %{
5453 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5454 constraint(ALLOC_IN_RC(ptr_reg));
5455 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5456 op_cost(0);
5457 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5458 interface(MEMORY_INTER) %{
5459 base($reg);
5460 index($ireg);
5461 scale($scale);
5462 disp(0x0);
5463 %}
5464 %}
5465
5466 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5467 %{
5468 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5469 constraint(ALLOC_IN_RC(ptr_reg));
5470 match(AddP (DecodeN reg) (LShiftL lreg scale));
5471 op_cost(0);
5472 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5473 interface(MEMORY_INTER) %{
5474 base($reg);
5475 index($lreg);
5476 scale($scale);
5477 disp(0x0);
5478 %}
5479 %}
5480
5481 operand indIndexI2LN(iRegN reg, iRegI ireg)
5482 %{
5483 predicate(CompressedOops::shift() == 0);
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(AddP (DecodeN reg) (ConvI2L ireg));
5486 op_cost(0);
5487 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index($ireg);
5491 scale(0x0);
5492 disp(0x0);
5493 %}
5494 %}
5495
5496 operand indIndexN(iRegN reg, iRegL lreg)
5497 %{
5498 predicate(CompressedOops::shift() == 0);
5499 constraint(ALLOC_IN_RC(ptr_reg));
5500 match(AddP (DecodeN reg) lreg);
5501 op_cost(0);
5502 format %{ "$reg, $lreg\t# narrow" %}
5503 interface(MEMORY_INTER) %{
5504 base($reg);
5505 index($lreg);
5506 scale(0x0);
5507 disp(0x0);
5508 %}
5509 %}
5510
5511 operand indOffIN(iRegN reg, immIOffset off)
5512 %{
5513 predicate(CompressedOops::shift() == 0);
5514 constraint(ALLOC_IN_RC(ptr_reg));
5515 match(AddP (DecodeN reg) off);
5516 op_cost(0);
5517 format %{ "[$reg, $off]\t# narrow" %}
5518 interface(MEMORY_INTER) %{
5519 base($reg);
5520 index(0xffffffff);
5521 scale(0x0);
5522 disp($off);
5523 %}
5524 %}
5525
5526 operand indOffLN(iRegN reg, immLOffset off)
5527 %{
5528 predicate(CompressedOops::shift() == 0);
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(AddP (DecodeN reg) off);
5531 op_cost(0);
5532 format %{ "[$reg, $off]\t# narrow" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index(0xffffffff);
5536 scale(0x0);
5537 disp($off);
5538 %}
5539 %}
5540
5541
5542 //----------Special Memory Operands--------------------------------------------
5543 // Stack Slot Operand - This operand is used for loading and storing temporary
5544 // values on the stack where a match requires a value to
5545 // flow through memory.
5546 operand stackSlotP(sRegP reg)
5547 %{
5548 constraint(ALLOC_IN_RC(stack_slots));
5549 op_cost(100);
5550 // No match rule because this operand is only generated in matching
5551 // match(RegP);
5552 format %{ "[$reg]" %}
5553 interface(MEMORY_INTER) %{
5554 base(0x1e); // RSP
5555 index(0x0); // No Index
5556 scale(0x0); // No Scale
5557 disp($reg); // Stack Offset
5558 %}
5559 %}
5560
5561 operand stackSlotI(sRegI reg)
5562 %{
5563 constraint(ALLOC_IN_RC(stack_slots));
5564 // No match rule because this operand is only generated in matching
5565 // match(RegI);
5566 format %{ "[$reg]" %}
5567 interface(MEMORY_INTER) %{
5568 base(0x1e); // RSP
5569 index(0x0); // No Index
5570 scale(0x0); // No Scale
5571 disp($reg); // Stack Offset
5572 %}
5573 %}
5574
5575 operand stackSlotF(sRegF reg)
5576 %{
5577 constraint(ALLOC_IN_RC(stack_slots));
5578 // No match rule because this operand is only generated in matching
5579 // match(RegF);
5580 format %{ "[$reg]" %}
5581 interface(MEMORY_INTER) %{
5582 base(0x1e); // RSP
5583 index(0x0); // No Index
5584 scale(0x0); // No Scale
5585 disp($reg); // Stack Offset
5586 %}
5587 %}
5588
5589 operand stackSlotD(sRegD reg)
5590 %{
5591 constraint(ALLOC_IN_RC(stack_slots));
5592 // No match rule because this operand is only generated in matching
5593 // match(RegD);
5594 format %{ "[$reg]" %}
5595 interface(MEMORY_INTER) %{
5596 base(0x1e); // RSP
5597 index(0x0); // No Index
5598 scale(0x0); // No Scale
5599 disp($reg); // Stack Offset
5600 %}
5601 %}
5602
5603 operand stackSlotL(sRegL reg)
5604 %{
5605 constraint(ALLOC_IN_RC(stack_slots));
5606 // No match rule because this operand is only generated in matching
5607 // match(RegL);
5608 format %{ "[$reg]" %}
5609 interface(MEMORY_INTER) %{
5610 base(0x1e); // RSP
5611 index(0x0); // No Index
5612 scale(0x0); // No Scale
5613 disp($reg); // Stack Offset
5614 %}
5615 %}
5616
5617 // Operands for expressing Control Flow
5618 // NOTE: Label is a predefined operand which should not be redefined in
5619 // the AD file. It is generically handled within the ADLC.
5620
5621 //----------Conditional Branch Operands----------------------------------------
5622 // Comparison Op - This is the operation of the comparison, and is limited to
5623 // the following set of codes:
5624 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5625 //
5626 // Other attributes of the comparison, such as unsignedness, are specified
5627 // by the comparison instruction that sets a condition code flags register.
5628 // That result is represented by a flags operand whose subtype is appropriate
5629 // to the unsignedness (etc.) of the comparison.
5630 //
5631 // Later, the instruction which matches both the Comparison Op (a Bool) and
5632 // the flags (produced by the Cmp) specifies the coding of the comparison op
5633 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5634
5635 // used for signed integral comparisons and fp comparisons
5636
5637 operand cmpOp()
5638 %{
5639 match(Bool);
5640
5641 format %{ "" %}
5642 interface(COND_INTER) %{
5643 equal(0x0, "eq");
5644 not_equal(0x1, "ne");
5645 less(0xb, "lt");
5646 greater_equal(0xa, "ge");
5647 less_equal(0xd, "le");
5648 greater(0xc, "gt");
5649 overflow(0x6, "vs");
5650 no_overflow(0x7, "vc");
5651 %}
5652 %}
5653
5654 // used for unsigned integral comparisons
5655
5656 operand cmpOpU()
5657 %{
5658 match(Bool);
5659
5660 format %{ "" %}
5661 interface(COND_INTER) %{
5662 equal(0x0, "eq");
5663 not_equal(0x1, "ne");
5664 less(0x3, "lo");
5665 greater_equal(0x2, "hs");
5666 less_equal(0x9, "ls");
5667 greater(0x8, "hi");
5668 overflow(0x6, "vs");
5669 no_overflow(0x7, "vc");
5670 %}
5671 %}
5672
5673 // used for certain integral comparisons which can be
5674 // converted to cbxx or tbxx instructions
5675
5676 operand cmpOpEqNe()
5677 %{
5678 match(Bool);
5679 op_cost(0);
5680 predicate(n->as_Bool()->_test._test == BoolTest::ne
5681 || n->as_Bool()->_test._test == BoolTest::eq);
5682
5683 format %{ "" %}
5684 interface(COND_INTER) %{
5685 equal(0x0, "eq");
5686 not_equal(0x1, "ne");
5687 less(0xb, "lt");
5688 greater_equal(0xa, "ge");
5689 less_equal(0xd, "le");
5690 greater(0xc, "gt");
5691 overflow(0x6, "vs");
5692 no_overflow(0x7, "vc");
5693 %}
5694 %}
5695
5696 // used for certain integral comparisons which can be
5697 // converted to cbxx or tbxx instructions
5698
5699 operand cmpOpLtGe()
5700 %{
5701 match(Bool);
5702 op_cost(0);
5703
5704 predicate(n->as_Bool()->_test._test == BoolTest::lt
5705 || n->as_Bool()->_test._test == BoolTest::ge);
5706
5707 format %{ "" %}
5708 interface(COND_INTER) %{
5709 equal(0x0, "eq");
5710 not_equal(0x1, "ne");
5711 less(0xb, "lt");
5712 greater_equal(0xa, "ge");
5713 less_equal(0xd, "le");
5714 greater(0xc, "gt");
5715 overflow(0x6, "vs");
5716 no_overflow(0x7, "vc");
5717 %}
5718 %}
5719
5720 // used for certain unsigned integral comparisons which can be
5721 // converted to cbxx or tbxx instructions
5722
5723 operand cmpOpUEqNeLeGt()
5724 %{
5725 match(Bool);
5726 op_cost(0);
5727
5728 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5729 n->as_Bool()->_test._test == BoolTest::ne ||
5730 n->as_Bool()->_test._test == BoolTest::le ||
5731 n->as_Bool()->_test._test == BoolTest::gt);
5732
5733 format %{ "" %}
5734 interface(COND_INTER) %{
5735 equal(0x0, "eq");
5736 not_equal(0x1, "ne");
5737 less(0x3, "lo");
5738 greater_equal(0x2, "hs");
5739 less_equal(0x9, "ls");
5740 greater(0x8, "hi");
5741 overflow(0x6, "vs");
5742 no_overflow(0x7, "vc");
5743 %}
5744 %}
5745
5746 // Special operand allowing long args to int ops to be truncated for free
5747
5748 operand iRegL2I(iRegL reg) %{
5749
5750 op_cost(0);
5751
5752 match(ConvL2I reg);
5753
5754 format %{ "l2i($reg)" %}
5755
5756 interface(REG_INTER)
5757 %}
5758
5759 operand iRegL2P(iRegL reg) %{
5760
5761 op_cost(0);
5762
5763 match(CastX2P reg);
5764
5765 format %{ "l2p($reg)" %}
5766
5767 interface(REG_INTER)
5768 %}
5769
5770 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5771 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5772 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5773 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5774
5775 //----------OPERAND CLASSES----------------------------------------------------
5776 // Operand Classes are groups of operands that are used as to simplify
5777 // instruction definitions by not requiring the AD writer to specify
5778 // separate instructions for every form of operand when the
5779 // instruction accepts multiple operand types with the same basic
5780 // encoding and format. The classic case of this is memory operands.
5781
5782 // memory is used to define read/write location for load/store
5783 // instruction defs. we can turn a memory op into an Address
5784
5785 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5786 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5787
5788 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5789 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5790
5791 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5792 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5793
5794 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5795 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5796
5797 // All of the memory operands. For the pipeline description.
5798 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5799 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5800 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5801
5802
5803 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5804 // operations. it allows the src to be either an iRegI or a (ConvL2I
5805 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5806 // can be elided because the 32-bit instruction will just employ the
5807 // lower 32 bits anyway.
5808 //
5809 // n.b. this does not elide all L2I conversions. if the truncated
5810 // value is consumed by more than one operation then the ConvL2I
5811 // cannot be bundled into the consuming nodes so an l2i gets planted
5812 // (actually a movw $dst $src) and the downstream instructions consume
5813 // the result of the l2i as an iRegI input. That's a shame since the
5814 // movw is actually redundant but its not too costly.
5815
5816 opclass iRegIorL2I(iRegI, iRegL2I);
5817 opclass iRegPorL2P(iRegP, iRegL2P);
5818
5819 //----------PIPELINE-----------------------------------------------------------
5820 // Rules which define the behavior of the target architectures pipeline.
5821
5822 // For specific pipelines, eg A53, define the stages of that pipeline
5823 //pipe_desc(ISS, EX1, EX2, WR);
5824 #define ISS S0
5825 #define EX1 S1
5826 #define EX2 S2
5827 #define WR S3
5828
5829 // Integer ALU reg operation
5830 pipeline %{
5831
5832 attributes %{
5833 // ARM instructions are of fixed length
5834 fixed_size_instructions; // Fixed size instructions TODO does
5835 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5836 // ARM instructions come in 32-bit word units
5837 instruction_unit_size = 4; // An instruction is 4 bytes long
5838 instruction_fetch_unit_size = 64; // The processor fetches one line
5839 instruction_fetch_units = 1; // of 64 bytes
5840
5841 // List of nop instructions
5842 nops( MachNop );
5843 %}
5844
5845 // We don't use an actual pipeline model so don't care about resources
5846 // or description. we do use pipeline classes to introduce fixed
5847 // latencies
5848
5849 //----------RESOURCES----------------------------------------------------------
5850 // Resources are the functional units available to the machine
5851
5852 resources( INS0, INS1, INS01 = INS0 | INS1,
5853 ALU0, ALU1, ALU = ALU0 | ALU1,
5854 MAC,
5855 DIV,
5856 BRANCH,
5857 LDST,
5858 NEON_FP);
5859
5860 //----------PIPELINE DESCRIPTION-----------------------------------------------
5861 // Pipeline Description specifies the stages in the machine's pipeline
5862
5863 // Define the pipeline as a generic 6 stage pipeline
5864 pipe_desc(S0, S1, S2, S3, S4, S5);
5865
5866 //----------PIPELINE CLASSES---------------------------------------------------
5867 // Pipeline Classes describe the stages in which input and output are
5868 // referenced by the hardware pipeline.
5869
5870 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5871 %{
5872 single_instruction;
5873 src1 : S1(read);
5874 src2 : S2(read);
5875 dst : S5(write);
5876 INS01 : ISS;
5877 NEON_FP : S5;
5878 %}
5879
5880 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5881 %{
5882 single_instruction;
5883 src1 : S1(read);
5884 src2 : S2(read);
5885 dst : S5(write);
5886 INS01 : ISS;
5887 NEON_FP : S5;
5888 %}
5889
5890 pipe_class fp_uop_s(vRegF dst, vRegF src)
5891 %{
5892 single_instruction;
5893 src : S1(read);
5894 dst : S5(write);
5895 INS01 : ISS;
5896 NEON_FP : S5;
5897 %}
5898
5899 pipe_class fp_uop_d(vRegD dst, vRegD src)
5900 %{
5901 single_instruction;
5902 src : S1(read);
5903 dst : S5(write);
5904 INS01 : ISS;
5905 NEON_FP : S5;
5906 %}
5907
5908 pipe_class fp_d2f(vRegF dst, vRegD src)
5909 %{
5910 single_instruction;
5911 src : S1(read);
5912 dst : S5(write);
5913 INS01 : ISS;
5914 NEON_FP : S5;
5915 %}
5916
5917 pipe_class fp_f2d(vRegD dst, vRegF src)
5918 %{
5919 single_instruction;
5920 src : S1(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5927 %{
5928 single_instruction;
5929 src : S1(read);
5930 dst : S5(write);
5931 INS01 : ISS;
5932 NEON_FP : S5;
5933 %}
5934
5935 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5936 %{
5937 single_instruction;
5938 src : S1(read);
5939 dst : S5(write);
5940 INS01 : ISS;
5941 NEON_FP : S5;
5942 %}
5943
5944 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5945 %{
5946 single_instruction;
5947 src : S1(read);
5948 dst : S5(write);
5949 INS01 : ISS;
5950 NEON_FP : S5;
5951 %}
5952
5953 pipe_class fp_l2f(vRegF dst, iRegL src)
5954 %{
5955 single_instruction;
5956 src : S1(read);
5957 dst : S5(write);
5958 INS01 : ISS;
5959 NEON_FP : S5;
5960 %}
5961
5962 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5963 %{
5964 single_instruction;
5965 src : S1(read);
5966 dst : S5(write);
5967 INS01 : ISS;
5968 NEON_FP : S5;
5969 %}
5970
5971 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5972 %{
5973 single_instruction;
5974 src : S1(read);
5975 dst : S5(write);
5976 INS01 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5981 %{
5982 single_instruction;
5983 src : S1(read);
5984 dst : S5(write);
5985 INS01 : ISS;
5986 NEON_FP : S5;
5987 %}
5988
5989 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5990 %{
5991 single_instruction;
5992 src : S1(read);
5993 dst : S5(write);
5994 INS01 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5999 %{
6000 single_instruction;
6001 src1 : S1(read);
6002 src2 : S2(read);
6003 dst : S5(write);
6004 INS0 : ISS;
6005 NEON_FP : S5;
6006 %}
6007
6008 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6009 %{
6010 single_instruction;
6011 src1 : S1(read);
6012 src2 : S2(read);
6013 dst : S5(write);
6014 INS0 : ISS;
6015 NEON_FP : S5;
6016 %}
6017
6018 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6019 %{
6020 single_instruction;
6021 cr : S1(read);
6022 src1 : S1(read);
6023 src2 : S1(read);
6024 dst : S3(write);
6025 INS01 : ISS;
6026 NEON_FP : S3;
6027 %}
6028
6029 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6030 %{
6031 single_instruction;
6032 cr : S1(read);
6033 src1 : S1(read);
6034 src2 : S1(read);
6035 dst : S3(write);
6036 INS01 : ISS;
6037 NEON_FP : S3;
6038 %}
6039
6040 pipe_class fp_imm_s(vRegF dst)
6041 %{
6042 single_instruction;
6043 dst : S3(write);
6044 INS01 : ISS;
6045 NEON_FP : S3;
6046 %}
6047
6048 pipe_class fp_imm_d(vRegD dst)
6049 %{
6050 single_instruction;
6051 dst : S3(write);
6052 INS01 : ISS;
6053 NEON_FP : S3;
6054 %}
6055
6056 pipe_class fp_load_constant_s(vRegF dst)
6057 %{
6058 single_instruction;
6059 dst : S4(write);
6060 INS01 : ISS;
6061 NEON_FP : S4;
6062 %}
6063
6064 pipe_class fp_load_constant_d(vRegD dst)
6065 %{
6066 single_instruction;
6067 dst : S4(write);
6068 INS01 : ISS;
6069 NEON_FP : S4;
6070 %}
6071
6072 //------- Integer ALU operations --------------------------
6073
6074 // Integer ALU reg-reg operation
6075 // Operands needed in EX1, result generated in EX2
6076 // Eg. ADD x0, x1, x2
6077 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6078 %{
6079 single_instruction;
6080 dst : EX2(write);
6081 src1 : EX1(read);
6082 src2 : EX1(read);
6083 INS01 : ISS; // Dual issue as instruction 0 or 1
6084 ALU : EX2;
6085 %}
6086
6087 // Integer ALU reg-reg operation with constant shift
6088 // Shifted register must be available in LATE_ISS instead of EX1
6089 // Eg. ADD x0, x1, x2, LSL #2
6090 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6091 %{
6092 single_instruction;
6093 dst : EX2(write);
6094 src1 : EX1(read);
6095 src2 : ISS(read);
6096 INS01 : ISS;
6097 ALU : EX2;
6098 %}
6099
6100 // Integer ALU reg operation with constant shift
6101 // Eg. LSL x0, x1, #shift
6102 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6103 %{
6104 single_instruction;
6105 dst : EX2(write);
6106 src1 : ISS(read);
6107 INS01 : ISS;
6108 ALU : EX2;
6109 %}
6110
6111 // Integer ALU reg-reg operation with variable shift
6112 // Both operands must be available in LATE_ISS instead of EX1
6113 // Result is available in EX1 instead of EX2
6114 // Eg. LSLV x0, x1, x2
6115 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6116 %{
6117 single_instruction;
6118 dst : EX1(write);
6119 src1 : ISS(read);
6120 src2 : ISS(read);
6121 INS01 : ISS;
6122 ALU : EX1;
6123 %}
6124
6125 // Integer ALU reg-reg operation with extract
6126 // As for _vshift above, but result generated in EX2
6127 // Eg. EXTR x0, x1, x2, #N
6128 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6129 %{
6130 single_instruction;
6131 dst : EX2(write);
6132 src1 : ISS(read);
6133 src2 : ISS(read);
6134 INS1 : ISS; // Can only dual issue as Instruction 1
6135 ALU : EX1;
6136 %}
6137
6138 // Integer ALU reg operation
6139 // Eg. NEG x0, x1
6140 pipe_class ialu_reg(iRegI dst, iRegI src)
6141 %{
6142 single_instruction;
6143 dst : EX2(write);
6144 src : EX1(read);
6145 INS01 : ISS;
6146 ALU : EX2;
6147 %}
6148
6149 // Integer ALU reg mmediate operation
6150 // Eg. ADD x0, x1, #N
6151 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6152 %{
6153 single_instruction;
6154 dst : EX2(write);
6155 src1 : EX1(read);
6156 INS01 : ISS;
6157 ALU : EX2;
6158 %}
6159
6160 // Integer ALU immediate operation (no source operands)
6161 // Eg. MOV x0, #N
6162 pipe_class ialu_imm(iRegI dst)
6163 %{
6164 single_instruction;
6165 dst : EX1(write);
6166 INS01 : ISS;
6167 ALU : EX1;
6168 %}
6169
6170 //------- Compare operation -------------------------------
6171
6172 // Compare reg-reg
6173 // Eg. CMP x0, x1
6174 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6175 %{
6176 single_instruction;
6177 // fixed_latency(16);
6178 cr : EX2(write);
6179 op1 : EX1(read);
6180 op2 : EX1(read);
6181 INS01 : ISS;
6182 ALU : EX2;
6183 %}
6184
6185 // Compare reg-reg
6186 // Eg. CMP x0, #N
6187 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6188 %{
6189 single_instruction;
6190 // fixed_latency(16);
6191 cr : EX2(write);
6192 op1 : EX1(read);
6193 INS01 : ISS;
6194 ALU : EX2;
6195 %}
6196
6197 //------- Conditional instructions ------------------------
6198
6199 // Conditional no operands
6200 // Eg. CSINC x0, zr, zr, <cond>
6201 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6202 %{
6203 single_instruction;
6204 cr : EX1(read);
6205 dst : EX2(write);
6206 INS01 : ISS;
6207 ALU : EX2;
6208 %}
6209
6210 // Conditional 2 operand
6211 // EG. CSEL X0, X1, X2, <cond>
6212 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6213 %{
6214 single_instruction;
6215 cr : EX1(read);
6216 src1 : EX1(read);
6217 src2 : EX1(read);
6218 dst : EX2(write);
6219 INS01 : ISS;
6220 ALU : EX2;
6221 %}
6222
6223 // Conditional 2 operand
6224 // EG. CSEL X0, X1, X2, <cond>
6225 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6226 %{
6227 single_instruction;
6228 cr : EX1(read);
6229 src : EX1(read);
6230 dst : EX2(write);
6231 INS01 : ISS;
6232 ALU : EX2;
6233 %}
6234
6235 //------- Multiply pipeline operations --------------------
6236
6237 // Multiply reg-reg
6238 // Eg. MUL w0, w1, w2
6239 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6240 %{
6241 single_instruction;
6242 dst : WR(write);
6243 src1 : ISS(read);
6244 src2 : ISS(read);
6245 INS01 : ISS;
6246 MAC : WR;
6247 %}
6248
6249 // Multiply accumulate
6250 // Eg. MADD w0, w1, w2, w3
6251 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6252 %{
6253 single_instruction;
6254 dst : WR(write);
6255 src1 : ISS(read);
6256 src2 : ISS(read);
6257 src3 : ISS(read);
6258 INS01 : ISS;
6259 MAC : WR;
6260 %}
6261
6262 // Eg. MUL w0, w1, w2
6263 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6264 %{
6265 single_instruction;
6266 fixed_latency(3); // Maximum latency for 64 bit mul
6267 dst : WR(write);
6268 src1 : ISS(read);
6269 src2 : ISS(read);
6270 INS01 : ISS;
6271 MAC : WR;
6272 %}
6273
6274 // Multiply accumulate
6275 // Eg. MADD w0, w1, w2, w3
6276 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6277 %{
6278 single_instruction;
6279 fixed_latency(3); // Maximum latency for 64 bit mul
6280 dst : WR(write);
6281 src1 : ISS(read);
6282 src2 : ISS(read);
6283 src3 : ISS(read);
6284 INS01 : ISS;
6285 MAC : WR;
6286 %}
6287
6288 //------- Divide pipeline operations --------------------
6289
6290 // Eg. SDIV w0, w1, w2
6291 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6292 %{
6293 single_instruction;
6294 fixed_latency(8); // Maximum latency for 32 bit divide
6295 dst : WR(write);
6296 src1 : ISS(read);
6297 src2 : ISS(read);
6298 INS0 : ISS; // Can only dual issue as instruction 0
6299 DIV : WR;
6300 %}
6301
6302 // Eg. SDIV x0, x1, x2
6303 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6304 %{
6305 single_instruction;
6306 fixed_latency(16); // Maximum latency for 64 bit divide
6307 dst : WR(write);
6308 src1 : ISS(read);
6309 src2 : ISS(read);
6310 INS0 : ISS; // Can only dual issue as instruction 0
6311 DIV : WR;
6312 %}
6313
6314 //------- Load pipeline operations ------------------------
6315
6316 // Load - prefetch
6317 // Eg. PFRM <mem>
6318 pipe_class iload_prefetch(memory mem)
6319 %{
6320 single_instruction;
6321 mem : ISS(read);
6322 INS01 : ISS;
6323 LDST : WR;
6324 %}
6325
6326 // Load - reg, mem
6327 // Eg. LDR x0, <mem>
6328 pipe_class iload_reg_mem(iRegI dst, memory mem)
6329 %{
6330 single_instruction;
6331 dst : WR(write);
6332 mem : ISS(read);
6333 INS01 : ISS;
6334 LDST : WR;
6335 %}
6336
6337 // Load - reg, reg
6338 // Eg. LDR x0, [sp, x1]
6339 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6340 %{
6341 single_instruction;
6342 dst : WR(write);
6343 src : ISS(read);
6344 INS01 : ISS;
6345 LDST : WR;
6346 %}
6347
6348 //------- Store pipeline operations -----------------------
6349
6350 // Store - zr, mem
6351 // Eg. STR zr, <mem>
6352 pipe_class istore_mem(memory mem)
6353 %{
6354 single_instruction;
6355 mem : ISS(read);
6356 INS01 : ISS;
6357 LDST : WR;
6358 %}
6359
6360 // Store - reg, mem
6361 // Eg. STR x0, <mem>
6362 pipe_class istore_reg_mem(iRegI src, memory mem)
6363 %{
6364 single_instruction;
6365 mem : ISS(read);
6366 src : EX2(read);
6367 INS01 : ISS;
6368 LDST : WR;
6369 %}
6370
6371 // Store - reg, reg
6372 // Eg. STR x0, [sp, x1]
6373 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6374 %{
6375 single_instruction;
6376 dst : ISS(read);
6377 src : EX2(read);
6378 INS01 : ISS;
6379 LDST : WR;
6380 %}
6381
6382 //------- Store pipeline operations -----------------------
6383
6384 // Branch
6385 pipe_class pipe_branch()
6386 %{
6387 single_instruction;
6388 INS01 : ISS;
6389 BRANCH : EX1;
6390 %}
6391
6392 // Conditional branch
6393 pipe_class pipe_branch_cond(rFlagsReg cr)
6394 %{
6395 single_instruction;
6396 cr : EX1(read);
6397 INS01 : ISS;
6398 BRANCH : EX1;
6399 %}
6400
6401 // Compare & Branch
6402 // EG. CBZ/CBNZ
6403 pipe_class pipe_cmp_branch(iRegI op1)
6404 %{
6405 single_instruction;
6406 op1 : EX1(read);
6407 INS01 : ISS;
6408 BRANCH : EX1;
6409 %}
6410
6411 //------- Synchronisation operations ----------------------
6412
6413 // Any operation requiring serialization.
6414 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6415 pipe_class pipe_serial()
6416 %{
6417 single_instruction;
6418 force_serialization;
6419 fixed_latency(16);
6420 INS01 : ISS(2); // Cannot dual issue with any other instruction
6421 LDST : WR;
6422 %}
6423
6424 // Generic big/slow expanded idiom - also serialized
6425 pipe_class pipe_slow()
6426 %{
6427 instruction_count(10);
6428 multiple_bundles;
6429 force_serialization;
6430 fixed_latency(16);
6431 INS01 : ISS(2); // Cannot dual issue with any other instruction
6432 LDST : WR;
6433 %}
6434
6435 // Empty pipeline class
6436 pipe_class pipe_class_empty()
6437 %{
6438 single_instruction;
6439 fixed_latency(0);
6440 %}
6441
6442 // Default pipeline class.
6443 pipe_class pipe_class_default()
6444 %{
6445 single_instruction;
6446 fixed_latency(2);
6447 %}
6448
6449 // Pipeline class for compares.
6450 pipe_class pipe_class_compare()
6451 %{
6452 single_instruction;
6453 fixed_latency(16);
6454 %}
6455
6456 // Pipeline class for memory operations.
6457 pipe_class pipe_class_memory()
6458 %{
6459 single_instruction;
6460 fixed_latency(16);
6461 %}
6462
6463 // Pipeline class for call.
6464 pipe_class pipe_class_call()
6465 %{
6466 single_instruction;
6467 fixed_latency(100);
6468 %}
6469
6470 // Define the class for the Nop node.
6471 define %{
6472 MachNop = pipe_class_empty;
6473 %}
6474
6475 %}
6476 //----------INSTRUCTIONS-------------------------------------------------------
6477 //
6478 // match -- States which machine-independent subtree may be replaced
6479 // by this instruction.
6480 // ins_cost -- The estimated cost of this instruction is used by instruction
6481 // selection to identify a minimum cost tree of machine
6482 // instructions that matches a tree of machine-independent
6483 // instructions.
6484 // format -- A string providing the disassembly for this instruction.
6485 // The value of an instruction's operand may be inserted
6486 // by referring to it with a '$' prefix.
6487 // opcode -- Three instruction opcodes may be provided. These are referred
6488 // to within an encode class as $primary, $secondary, and $tertiary
6489 // rrspectively. The primary opcode is commonly used to
6490 // indicate the type of machine instruction, while secondary
6491 // and tertiary are often used for prefix options or addressing
6492 // modes.
6493 // ins_encode -- A list of encode classes with parameters. The encode class
6494 // name must have been defined in an 'enc_class' specification
6495 // in the encode section of the architecture description.
6496
6497 // ============================================================================
6498 // Memory (Load/Store) Instructions
6499
6500 // Load Instructions
6501
6502 // Load Byte (8 bit signed)
6503 instruct loadB(iRegINoSp dst, memory1 mem)
6504 %{
6505 match(Set dst (LoadB mem));
6506 predicate(!needs_acquiring_load(n));
6507
6508 ins_cost(4 * INSN_COST);
6509 format %{ "ldrsbw $dst, $mem\t# byte" %}
6510
6511 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6512
6513 ins_pipe(iload_reg_mem);
6514 %}
6515
6516 // Load Byte (8 bit signed) into long
6517 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6518 %{
6519 match(Set dst (ConvI2L (LoadB mem)));
6520 predicate(!needs_acquiring_load(n->in(1)));
6521
6522 ins_cost(4 * INSN_COST);
6523 format %{ "ldrsb $dst, $mem\t# byte" %}
6524
6525 ins_encode(aarch64_enc_ldrsb(dst, mem));
6526
6527 ins_pipe(iload_reg_mem);
6528 %}
6529
6530 // Load Byte (8 bit unsigned)
6531 instruct loadUB(iRegINoSp dst, memory1 mem)
6532 %{
6533 match(Set dst (LoadUB mem));
6534 predicate(!needs_acquiring_load(n));
6535
6536 ins_cost(4 * INSN_COST);
6537 format %{ "ldrbw $dst, $mem\t# byte" %}
6538
6539 ins_encode(aarch64_enc_ldrb(dst, mem));
6540
6541 ins_pipe(iload_reg_mem);
6542 %}
6543
6544 // Load Byte (8 bit unsigned) into long
6545 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6546 %{
6547 match(Set dst (ConvI2L (LoadUB mem)));
6548 predicate(!needs_acquiring_load(n->in(1)));
6549
6550 ins_cost(4 * INSN_COST);
6551 format %{ "ldrb $dst, $mem\t# byte" %}
6552
6553 ins_encode(aarch64_enc_ldrb(dst, mem));
6554
6555 ins_pipe(iload_reg_mem);
6556 %}
6557
6558 // Load Short (16 bit signed)
6559 instruct loadS(iRegINoSp dst, memory2 mem)
6560 %{
6561 match(Set dst (LoadS mem));
6562 predicate(!needs_acquiring_load(n));
6563
6564 ins_cost(4 * INSN_COST);
6565 format %{ "ldrshw $dst, $mem\t# short" %}
6566
6567 ins_encode(aarch64_enc_ldrshw(dst, mem));
6568
6569 ins_pipe(iload_reg_mem);
6570 %}
6571
6572 // Load Short (16 bit signed) into long
6573 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6574 %{
6575 match(Set dst (ConvI2L (LoadS mem)));
6576 predicate(!needs_acquiring_load(n->in(1)));
6577
6578 ins_cost(4 * INSN_COST);
6579 format %{ "ldrsh $dst, $mem\t# short" %}
6580
6581 ins_encode(aarch64_enc_ldrsh(dst, mem));
6582
6583 ins_pipe(iload_reg_mem);
6584 %}
6585
6586 // Load Char (16 bit unsigned)
6587 instruct loadUS(iRegINoSp dst, memory2 mem)
6588 %{
6589 match(Set dst (LoadUS mem));
6590 predicate(!needs_acquiring_load(n));
6591
6592 ins_cost(4 * INSN_COST);
6593 format %{ "ldrh $dst, $mem\t# short" %}
6594
6595 ins_encode(aarch64_enc_ldrh(dst, mem));
6596
6597 ins_pipe(iload_reg_mem);
6598 %}
6599
6600 // Load Short/Char (16 bit unsigned) into long
6601 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6602 %{
6603 match(Set dst (ConvI2L (LoadUS mem)));
6604 predicate(!needs_acquiring_load(n->in(1)));
6605
6606 ins_cost(4 * INSN_COST);
6607 format %{ "ldrh $dst, $mem\t# short" %}
6608
6609 ins_encode(aarch64_enc_ldrh(dst, mem));
6610
6611 ins_pipe(iload_reg_mem);
6612 %}
6613
6614 // Load Integer (32 bit signed)
6615 instruct loadI(iRegINoSp dst, memory4 mem)
6616 %{
6617 match(Set dst (LoadI mem));
6618 predicate(!needs_acquiring_load(n));
6619
6620 ins_cost(4 * INSN_COST);
6621 format %{ "ldrw $dst, $mem\t# int" %}
6622
6623 ins_encode(aarch64_enc_ldrw(dst, mem));
6624
6625 ins_pipe(iload_reg_mem);
6626 %}
6627
6628 // Load Integer (32 bit signed) into long
6629 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6630 %{
6631 match(Set dst (ConvI2L (LoadI mem)));
6632 predicate(!needs_acquiring_load(n->in(1)));
6633
6634 ins_cost(4 * INSN_COST);
6635 format %{ "ldrsw $dst, $mem\t# int" %}
6636
6637 ins_encode(aarch64_enc_ldrsw(dst, mem));
6638
6639 ins_pipe(iload_reg_mem);
6640 %}
6641
6642 // Load Integer (32 bit unsigned) into long
6643 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6644 %{
6645 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6646 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6647
6648 ins_cost(4 * INSN_COST);
6649 format %{ "ldrw $dst, $mem\t# int" %}
6650
6651 ins_encode(aarch64_enc_ldrw(dst, mem));
6652
6653 ins_pipe(iload_reg_mem);
6654 %}
6655
6656 // Load Long (64 bit signed)
6657 instruct loadL(iRegLNoSp dst, memory8 mem)
6658 %{
6659 match(Set dst (LoadL mem));
6660 predicate(!needs_acquiring_load(n));
6661
6662 ins_cost(4 * INSN_COST);
6663 format %{ "ldr $dst, $mem\t# int" %}
6664
6665 ins_encode(aarch64_enc_ldr(dst, mem));
6666
6667 ins_pipe(iload_reg_mem);
6668 %}
6669
6670 // Load Range
6671 instruct loadRange(iRegINoSp dst, memory4 mem)
6672 %{
6673 match(Set dst (LoadRange mem));
6674
6675 ins_cost(4 * INSN_COST);
6676 format %{ "ldrw $dst, $mem\t# range" %}
6677
6678 ins_encode(aarch64_enc_ldrw(dst, mem));
6679
6680 ins_pipe(iload_reg_mem);
6681 %}
6682
6683 // Load Pointer
6684 instruct loadP(iRegPNoSp dst, memory8 mem)
6685 %{
6686 match(Set dst (LoadP mem));
6687 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6688
6689 ins_cost(4 * INSN_COST);
6690 format %{ "ldr $dst, $mem\t# ptr" %}
6691
6692 ins_encode(aarch64_enc_ldr(dst, mem));
6693
6694 ins_pipe(iload_reg_mem);
6695 %}
6696
6697 // Load Compressed Pointer
6698 instruct loadN(iRegNNoSp dst, memory4 mem)
6699 %{
6700 match(Set dst (LoadN mem));
6701 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6702
6703 ins_cost(4 * INSN_COST);
6704 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6705
6706 ins_encode(aarch64_enc_ldrw(dst, mem));
6707
6708 ins_pipe(iload_reg_mem);
6709 %}
6710
6711 // Load Klass Pointer
6712 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6713 %{
6714 match(Set dst (LoadKlass mem));
6715 predicate(!needs_acquiring_load(n));
6716
6717 ins_cost(4 * INSN_COST);
6718 format %{ "ldr $dst, $mem\t# class" %}
6719
6720 ins_encode(aarch64_enc_ldr(dst, mem));
6721
6722 ins_pipe(iload_reg_mem);
6723 %}
6724
6725 // Load Narrow Klass Pointer
6726 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6727 %{
6728 match(Set dst (LoadNKlass mem));
6729 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6730
6731 ins_cost(4 * INSN_COST);
6732 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6733
6734 ins_encode(aarch64_enc_ldrw(dst, mem));
6735
6736 ins_pipe(iload_reg_mem);
6737 %}
6738
6739 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6740 %{
6741 match(Set dst (LoadNKlass mem));
6742 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6743
6744 ins_cost(4 * INSN_COST);
6745 format %{
6746 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6747 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6748 %}
6749 ins_encode %{
6750 // inlined aarch64_enc_ldrw
6751 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6752 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6753 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6754 %}
6755 ins_pipe(iload_reg_mem);
6756 %}
6757
6758 // Load Float
6759 instruct loadF(vRegF dst, memory4 mem)
6760 %{
6761 match(Set dst (LoadF mem));
6762 predicate(!needs_acquiring_load(n));
6763
6764 ins_cost(4 * INSN_COST);
6765 format %{ "ldrs $dst, $mem\t# float" %}
6766
6767 ins_encode( aarch64_enc_ldrs(dst, mem) );
6768
6769 ins_pipe(pipe_class_memory);
6770 %}
6771
6772 // Load Double
6773 instruct loadD(vRegD dst, memory8 mem)
6774 %{
6775 match(Set dst (LoadD mem));
6776 predicate(!needs_acquiring_load(n));
6777
6778 ins_cost(4 * INSN_COST);
6779 format %{ "ldrd $dst, $mem\t# double" %}
6780
6781 ins_encode( aarch64_enc_ldrd(dst, mem) );
6782
6783 ins_pipe(pipe_class_memory);
6784 %}
6785
6786
6787 // Load Int Constant
6788 instruct loadConI(iRegINoSp dst, immI src)
6789 %{
6790 match(Set dst src);
6791
6792 ins_cost(INSN_COST);
6793 format %{ "mov $dst, $src\t# int" %}
6794
6795 ins_encode( aarch64_enc_movw_imm(dst, src) );
6796
6797 ins_pipe(ialu_imm);
6798 %}
6799
6800 // Load Long Constant
6801 instruct loadConL(iRegLNoSp dst, immL src)
6802 %{
6803 match(Set dst src);
6804
6805 ins_cost(INSN_COST);
6806 format %{ "mov $dst, $src\t# long" %}
6807
6808 ins_encode( aarch64_enc_mov_imm(dst, src) );
6809
6810 ins_pipe(ialu_imm);
6811 %}
6812
6813 // Load Pointer Constant
6814
6815 instruct loadConP(iRegPNoSp dst, immP con)
6816 %{
6817 match(Set dst con);
6818
6819 ins_cost(INSN_COST * 4);
6820 format %{
6821 "mov $dst, $con\t# ptr\n\t"
6822 %}
6823
6824 ins_encode(aarch64_enc_mov_p(dst, con));
6825
6826 ins_pipe(ialu_imm);
6827 %}
6828
6829 // Load Null Pointer Constant
6830
6831 instruct loadConP0(iRegPNoSp dst, immP0 con)
6832 %{
6833 match(Set dst con);
6834
6835 ins_cost(INSN_COST);
6836 format %{ "mov $dst, $con\t# nullptr ptr" %}
6837
6838 ins_encode(aarch64_enc_mov_p0(dst, con));
6839
6840 ins_pipe(ialu_imm);
6841 %}
6842
6843 // Load Pointer Constant One
6844
6845 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6846 %{
6847 match(Set dst con);
6848
6849 ins_cost(INSN_COST);
6850 format %{ "mov $dst, $con\t# nullptr ptr" %}
6851
6852 ins_encode(aarch64_enc_mov_p1(dst, con));
6853
6854 ins_pipe(ialu_imm);
6855 %}
6856
6857 // Load Byte Map Base Constant
6858
6859 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6860 %{
6861 match(Set dst con);
6862
6863 ins_cost(INSN_COST);
6864 format %{ "adr $dst, $con\t# Byte Map Base" %}
6865
6866 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6867
6868 ins_pipe(ialu_imm);
6869 %}
6870
6871 // Load Narrow Pointer Constant
6872
6873 instruct loadConN(iRegNNoSp dst, immN con)
6874 %{
6875 match(Set dst con);
6876
6877 ins_cost(INSN_COST * 4);
6878 format %{ "mov $dst, $con\t# compressed ptr" %}
6879
6880 ins_encode(aarch64_enc_mov_n(dst, con));
6881
6882 ins_pipe(ialu_imm);
6883 %}
6884
6885 // Load Narrow Null Pointer Constant
6886
6887 instruct loadConN0(iRegNNoSp dst, immN0 con)
6888 %{
6889 match(Set dst con);
6890
6891 ins_cost(INSN_COST);
6892 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6893
6894 ins_encode(aarch64_enc_mov_n0(dst, con));
6895
6896 ins_pipe(ialu_imm);
6897 %}
6898
6899 // Load Narrow Klass Constant
6900
6901 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6902 %{
6903 match(Set dst con);
6904
6905 ins_cost(INSN_COST);
6906 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6907
6908 ins_encode(aarch64_enc_mov_nk(dst, con));
6909
6910 ins_pipe(ialu_imm);
6911 %}
6912
6913 // Load Packed Float Constant
6914
6915 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6916 match(Set dst con);
6917 ins_cost(INSN_COST * 4);
6918 format %{ "fmovs $dst, $con"%}
6919 ins_encode %{
6920 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6921 %}
6922
6923 ins_pipe(fp_imm_s);
6924 %}
6925
6926 // Load Float Constant
6927
6928 instruct loadConF(vRegF dst, immF con) %{
6929 match(Set dst con);
6930
6931 ins_cost(INSN_COST * 4);
6932
6933 format %{
6934 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6935 %}
6936
6937 ins_encode %{
6938 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6939 %}
6940
6941 ins_pipe(fp_load_constant_s);
6942 %}
6943
6944 // Load Packed Double Constant
6945
6946 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6947 match(Set dst con);
6948 ins_cost(INSN_COST);
6949 format %{ "fmovd $dst, $con"%}
6950 ins_encode %{
6951 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6952 %}
6953
6954 ins_pipe(fp_imm_d);
6955 %}
6956
6957 // Load Double Constant
6958
6959 instruct loadConD(vRegD dst, immD con) %{
6960 match(Set dst con);
6961
6962 ins_cost(INSN_COST * 5);
6963 format %{
6964 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6965 %}
6966
6967 ins_encode %{
6968 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6969 %}
6970
6971 ins_pipe(fp_load_constant_d);
6972 %}
6973
6974 // Load Half Float Constant
6975 // The "ldr" instruction loads a 32-bit word from the constant pool into a
6976 // 32-bit register but only the bottom half will be populated and the top
6977 // 16 bits are zero.
6978 instruct loadConH(vRegF dst, immH con) %{
6979 match(Set dst con);
6980 format %{
6981 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
6982 %}
6983 ins_encode %{
6984 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6985 %}
6986 ins_pipe(fp_load_constant_s);
6987 %}
6988
6989 // Store Instructions
6990
6991 // Store Byte
6992 instruct storeB(iRegIorL2I src, memory1 mem)
6993 %{
6994 match(Set mem (StoreB mem src));
6995 predicate(!needs_releasing_store(n));
6996
6997 ins_cost(INSN_COST);
6998 format %{ "strb $src, $mem\t# byte" %}
6999
7000 ins_encode(aarch64_enc_strb(src, mem));
7001
7002 ins_pipe(istore_reg_mem);
7003 %}
7004
7005
7006 instruct storeimmB0(immI0 zero, memory1 mem)
7007 %{
7008 match(Set mem (StoreB mem zero));
7009 predicate(!needs_releasing_store(n));
7010
7011 ins_cost(INSN_COST);
7012 format %{ "strb rscractch2, $mem\t# byte" %}
7013
7014 ins_encode(aarch64_enc_strb0(mem));
7015
7016 ins_pipe(istore_mem);
7017 %}
7018
7019 // Store Char/Short
7020 instruct storeC(iRegIorL2I src, memory2 mem)
7021 %{
7022 match(Set mem (StoreC mem src));
7023 predicate(!needs_releasing_store(n));
7024
7025 ins_cost(INSN_COST);
7026 format %{ "strh $src, $mem\t# short" %}
7027
7028 ins_encode(aarch64_enc_strh(src, mem));
7029
7030 ins_pipe(istore_reg_mem);
7031 %}
7032
7033 instruct storeimmC0(immI0 zero, memory2 mem)
7034 %{
7035 match(Set mem (StoreC mem zero));
7036 predicate(!needs_releasing_store(n));
7037
7038 ins_cost(INSN_COST);
7039 format %{ "strh zr, $mem\t# short" %}
7040
7041 ins_encode(aarch64_enc_strh0(mem));
7042
7043 ins_pipe(istore_mem);
7044 %}
7045
7046 // Store Integer
7047
7048 instruct storeI(iRegIorL2I src, memory4 mem)
7049 %{
7050 match(Set mem(StoreI mem src));
7051 predicate(!needs_releasing_store(n));
7052
7053 ins_cost(INSN_COST);
7054 format %{ "strw $src, $mem\t# int" %}
7055
7056 ins_encode(aarch64_enc_strw(src, mem));
7057
7058 ins_pipe(istore_reg_mem);
7059 %}
7060
7061 instruct storeimmI0(immI0 zero, memory4 mem)
7062 %{
7063 match(Set mem(StoreI mem zero));
7064 predicate(!needs_releasing_store(n));
7065
7066 ins_cost(INSN_COST);
7067 format %{ "strw zr, $mem\t# int" %}
7068
7069 ins_encode(aarch64_enc_strw0(mem));
7070
7071 ins_pipe(istore_mem);
7072 %}
7073
7074 // Store Long (64 bit signed)
7075 instruct storeL(iRegL src, memory8 mem)
7076 %{
7077 match(Set mem (StoreL mem src));
7078 predicate(!needs_releasing_store(n));
7079
7080 ins_cost(INSN_COST);
7081 format %{ "str $src, $mem\t# int" %}
7082
7083 ins_encode(aarch64_enc_str(src, mem));
7084
7085 ins_pipe(istore_reg_mem);
7086 %}
7087
7088 // Store Long (64 bit signed)
7089 instruct storeimmL0(immL0 zero, memory8 mem)
7090 %{
7091 match(Set mem (StoreL mem zero));
7092 predicate(!needs_releasing_store(n));
7093
7094 ins_cost(INSN_COST);
7095 format %{ "str zr, $mem\t# int" %}
7096
7097 ins_encode(aarch64_enc_str0(mem));
7098
7099 ins_pipe(istore_mem);
7100 %}
7101
7102 // Store Pointer
7103 instruct storeP(iRegP src, memory8 mem)
7104 %{
7105 match(Set mem (StoreP mem src));
7106 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7107
7108 ins_cost(INSN_COST);
7109 format %{ "str $src, $mem\t# ptr" %}
7110
7111 ins_encode(aarch64_enc_str(src, mem));
7112
7113 ins_pipe(istore_reg_mem);
7114 %}
7115
7116 // Store Pointer
7117 instruct storeimmP0(immP0 zero, memory8 mem)
7118 %{
7119 match(Set mem (StoreP mem zero));
7120 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7121
7122 ins_cost(INSN_COST);
7123 format %{ "str zr, $mem\t# ptr" %}
7124
7125 ins_encode(aarch64_enc_str0(mem));
7126
7127 ins_pipe(istore_mem);
7128 %}
7129
7130 // Store Compressed Pointer
7131 instruct storeN(iRegN src, memory4 mem)
7132 %{
7133 match(Set mem (StoreN mem src));
7134 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7135
7136 ins_cost(INSN_COST);
7137 format %{ "strw $src, $mem\t# compressed ptr" %}
7138
7139 ins_encode(aarch64_enc_strw(src, mem));
7140
7141 ins_pipe(istore_reg_mem);
7142 %}
7143
7144 instruct storeImmN0(immN0 zero, memory4 mem)
7145 %{
7146 match(Set mem (StoreN mem zero));
7147 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7148
7149 ins_cost(INSN_COST);
7150 format %{ "strw zr, $mem\t# compressed ptr" %}
7151
7152 ins_encode(aarch64_enc_strw0(mem));
7153
7154 ins_pipe(istore_mem);
7155 %}
7156
7157 // Store Float
7158 instruct storeF(vRegF src, memory4 mem)
7159 %{
7160 match(Set mem (StoreF mem src));
7161 predicate(!needs_releasing_store(n));
7162
7163 ins_cost(INSN_COST);
7164 format %{ "strs $src, $mem\t# float" %}
7165
7166 ins_encode( aarch64_enc_strs(src, mem) );
7167
7168 ins_pipe(pipe_class_memory);
7169 %}
7170
7171 // TODO
7172 // implement storeImmF0 and storeFImmPacked
7173
7174 // Store Double
7175 instruct storeD(vRegD src, memory8 mem)
7176 %{
7177 match(Set mem (StoreD mem src));
7178 predicate(!needs_releasing_store(n));
7179
7180 ins_cost(INSN_COST);
7181 format %{ "strd $src, $mem\t# double" %}
7182
7183 ins_encode( aarch64_enc_strd(src, mem) );
7184
7185 ins_pipe(pipe_class_memory);
7186 %}
7187
7188 // Store Compressed Klass Pointer
7189 instruct storeNKlass(iRegN src, memory4 mem)
7190 %{
7191 predicate(!needs_releasing_store(n));
7192 match(Set mem (StoreNKlass mem src));
7193
7194 ins_cost(INSN_COST);
7195 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7196
7197 ins_encode(aarch64_enc_strw(src, mem));
7198
7199 ins_pipe(istore_reg_mem);
7200 %}
7201
7202 // TODO
7203 // implement storeImmD0 and storeDImmPacked
7204
7205 // prefetch instructions
7206 // Must be safe to execute with invalid address (cannot fault).
7207
7208 instruct prefetchalloc( memory8 mem ) %{
7209 match(PrefetchAllocation mem);
7210
7211 ins_cost(INSN_COST);
7212 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7213
7214 ins_encode( aarch64_enc_prefetchw(mem) );
7215
7216 ins_pipe(iload_prefetch);
7217 %}
7218
7219 // ---------------- volatile loads and stores ----------------
7220
7221 // Load Byte (8 bit signed)
7222 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7223 %{
7224 match(Set dst (LoadB mem));
7225
7226 ins_cost(VOLATILE_REF_COST);
7227 format %{ "ldarsb $dst, $mem\t# byte" %}
7228
7229 ins_encode(aarch64_enc_ldarsb(dst, mem));
7230
7231 ins_pipe(pipe_serial);
7232 %}
7233
7234 // Load Byte (8 bit signed) into long
7235 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7236 %{
7237 match(Set dst (ConvI2L (LoadB mem)));
7238
7239 ins_cost(VOLATILE_REF_COST);
7240 format %{ "ldarsb $dst, $mem\t# byte" %}
7241
7242 ins_encode(aarch64_enc_ldarsb(dst, mem));
7243
7244 ins_pipe(pipe_serial);
7245 %}
7246
7247 // Load Byte (8 bit unsigned)
7248 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7249 %{
7250 match(Set dst (LoadUB mem));
7251
7252 ins_cost(VOLATILE_REF_COST);
7253 format %{ "ldarb $dst, $mem\t# byte" %}
7254
7255 ins_encode(aarch64_enc_ldarb(dst, mem));
7256
7257 ins_pipe(pipe_serial);
7258 %}
7259
7260 // Load Byte (8 bit unsigned) into long
7261 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7262 %{
7263 match(Set dst (ConvI2L (LoadUB mem)));
7264
7265 ins_cost(VOLATILE_REF_COST);
7266 format %{ "ldarb $dst, $mem\t# byte" %}
7267
7268 ins_encode(aarch64_enc_ldarb(dst, mem));
7269
7270 ins_pipe(pipe_serial);
7271 %}
7272
7273 // Load Short (16 bit signed)
7274 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7275 %{
7276 match(Set dst (LoadS mem));
7277
7278 ins_cost(VOLATILE_REF_COST);
7279 format %{ "ldarshw $dst, $mem\t# short" %}
7280
7281 ins_encode(aarch64_enc_ldarshw(dst, mem));
7282
7283 ins_pipe(pipe_serial);
7284 %}
7285
7286 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7287 %{
7288 match(Set dst (LoadUS mem));
7289
7290 ins_cost(VOLATILE_REF_COST);
7291 format %{ "ldarhw $dst, $mem\t# short" %}
7292
7293 ins_encode(aarch64_enc_ldarhw(dst, mem));
7294
7295 ins_pipe(pipe_serial);
7296 %}
7297
7298 // Load Short/Char (16 bit unsigned) into long
7299 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7300 %{
7301 match(Set dst (ConvI2L (LoadUS mem)));
7302
7303 ins_cost(VOLATILE_REF_COST);
7304 format %{ "ldarh $dst, $mem\t# short" %}
7305
7306 ins_encode(aarch64_enc_ldarh(dst, mem));
7307
7308 ins_pipe(pipe_serial);
7309 %}
7310
7311 // Load Short/Char (16 bit signed) into long
7312 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7313 %{
7314 match(Set dst (ConvI2L (LoadS mem)));
7315
7316 ins_cost(VOLATILE_REF_COST);
7317 format %{ "ldarh $dst, $mem\t# short" %}
7318
7319 ins_encode(aarch64_enc_ldarsh(dst, mem));
7320
7321 ins_pipe(pipe_serial);
7322 %}
7323
7324 // Load Integer (32 bit signed)
7325 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7326 %{
7327 match(Set dst (LoadI mem));
7328
7329 ins_cost(VOLATILE_REF_COST);
7330 format %{ "ldarw $dst, $mem\t# int" %}
7331
7332 ins_encode(aarch64_enc_ldarw(dst, mem));
7333
7334 ins_pipe(pipe_serial);
7335 %}
7336
7337 // Load Integer (32 bit unsigned) into long
7338 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7339 %{
7340 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7341
7342 ins_cost(VOLATILE_REF_COST);
7343 format %{ "ldarw $dst, $mem\t# int" %}
7344
7345 ins_encode(aarch64_enc_ldarw(dst, mem));
7346
7347 ins_pipe(pipe_serial);
7348 %}
7349
7350 // Load Long (64 bit signed)
7351 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7352 %{
7353 match(Set dst (LoadL mem));
7354
7355 ins_cost(VOLATILE_REF_COST);
7356 format %{ "ldar $dst, $mem\t# int" %}
7357
7358 ins_encode(aarch64_enc_ldar(dst, mem));
7359
7360 ins_pipe(pipe_serial);
7361 %}
7362
7363 // Load Pointer
7364 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7365 %{
7366 match(Set dst (LoadP mem));
7367 predicate(n->as_Load()->barrier_data() == 0);
7368
7369 ins_cost(VOLATILE_REF_COST);
7370 format %{ "ldar $dst, $mem\t# ptr" %}
7371
7372 ins_encode(aarch64_enc_ldar(dst, mem));
7373
7374 ins_pipe(pipe_serial);
7375 %}
7376
7377 // Load Compressed Pointer
7378 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7379 %{
7380 match(Set dst (LoadN mem));
7381 predicate(n->as_Load()->barrier_data() == 0);
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7385
7386 ins_encode(aarch64_enc_ldarw(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Float
7392 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set dst (LoadF mem));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "ldars $dst, $mem\t# float" %}
7398
7399 ins_encode( aarch64_enc_fldars(dst, mem) );
7400
7401 ins_pipe(pipe_serial);
7402 %}
7403
7404 // Load Double
7405 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7406 %{
7407 match(Set dst (LoadD mem));
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "ldard $dst, $mem\t# double" %}
7411
7412 ins_encode( aarch64_enc_fldard(dst, mem) );
7413
7414 ins_pipe(pipe_serial);
7415 %}
7416
7417 // Store Byte
7418 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7419 %{
7420 match(Set mem (StoreB mem src));
7421
7422 ins_cost(VOLATILE_REF_COST);
7423 format %{ "stlrb $src, $mem\t# byte" %}
7424
7425 ins_encode(aarch64_enc_stlrb(src, mem));
7426
7427 ins_pipe(pipe_class_memory);
7428 %}
7429
7430 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set mem (StoreB mem zero));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "stlrb zr, $mem\t# byte" %}
7436
7437 ins_encode(aarch64_enc_stlrb0(mem));
7438
7439 ins_pipe(pipe_class_memory);
7440 %}
7441
7442 // Store Char/Short
7443 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set mem (StoreC mem src));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "stlrh $src, $mem\t# short" %}
7449
7450 ins_encode(aarch64_enc_stlrh(src, mem));
7451
7452 ins_pipe(pipe_class_memory);
7453 %}
7454
7455 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set mem (StoreC mem zero));
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "stlrh zr, $mem\t# short" %}
7461
7462 ins_encode(aarch64_enc_stlrh0(mem));
7463
7464 ins_pipe(pipe_class_memory);
7465 %}
7466
7467 // Store Integer
7468
7469 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7470 %{
7471 match(Set mem(StoreI mem src));
7472
7473 ins_cost(VOLATILE_REF_COST);
7474 format %{ "stlrw $src, $mem\t# int" %}
7475
7476 ins_encode(aarch64_enc_stlrw(src, mem));
7477
7478 ins_pipe(pipe_class_memory);
7479 %}
7480
7481 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7482 %{
7483 match(Set mem(StoreI mem zero));
7484
7485 ins_cost(VOLATILE_REF_COST);
7486 format %{ "stlrw zr, $mem\t# int" %}
7487
7488 ins_encode(aarch64_enc_stlrw0(mem));
7489
7490 ins_pipe(pipe_class_memory);
7491 %}
7492
7493 // Store Long (64 bit signed)
7494 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7495 %{
7496 match(Set mem (StoreL mem src));
7497
7498 ins_cost(VOLATILE_REF_COST);
7499 format %{ "stlr $src, $mem\t# int" %}
7500
7501 ins_encode(aarch64_enc_stlr(src, mem));
7502
7503 ins_pipe(pipe_class_memory);
7504 %}
7505
7506 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem (StoreL mem zero));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlr zr, $mem\t# int" %}
7512
7513 ins_encode(aarch64_enc_stlr0(mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 // Store Pointer
7519 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem (StoreP mem src));
7522 predicate(n->as_Store()->barrier_data() == 0);
7523
7524 ins_cost(VOLATILE_REF_COST);
7525 format %{ "stlr $src, $mem\t# ptr" %}
7526
7527 ins_encode(aarch64_enc_stlr(src, mem));
7528
7529 ins_pipe(pipe_class_memory);
7530 %}
7531
7532 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7533 %{
7534 match(Set mem (StoreP mem zero));
7535 predicate(n->as_Store()->barrier_data() == 0);
7536
7537 ins_cost(VOLATILE_REF_COST);
7538 format %{ "stlr zr, $mem\t# ptr" %}
7539
7540 ins_encode(aarch64_enc_stlr0(mem));
7541
7542 ins_pipe(pipe_class_memory);
7543 %}
7544
7545 // Store Compressed Pointer
7546 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7547 %{
7548 match(Set mem (StoreN mem src));
7549 predicate(n->as_Store()->barrier_data() == 0);
7550
7551 ins_cost(VOLATILE_REF_COST);
7552 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7553
7554 ins_encode(aarch64_enc_stlrw(src, mem));
7555
7556 ins_pipe(pipe_class_memory);
7557 %}
7558
7559 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7560 %{
7561 match(Set mem (StoreN mem zero));
7562 predicate(n->as_Store()->barrier_data() == 0);
7563
7564 ins_cost(VOLATILE_REF_COST);
7565 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7566
7567 ins_encode(aarch64_enc_stlrw0(mem));
7568
7569 ins_pipe(pipe_class_memory);
7570 %}
7571
7572 // Store Float
7573 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7574 %{
7575 match(Set mem (StoreF mem src));
7576
7577 ins_cost(VOLATILE_REF_COST);
7578 format %{ "stlrs $src, $mem\t# float" %}
7579
7580 ins_encode( aarch64_enc_fstlrs(src, mem) );
7581
7582 ins_pipe(pipe_class_memory);
7583 %}
7584
7585 // TODO
7586 // implement storeImmF0 and storeFImmPacked
7587
7588 // Store Double
7589 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7590 %{
7591 match(Set mem (StoreD mem src));
7592
7593 ins_cost(VOLATILE_REF_COST);
7594 format %{ "stlrd $src, $mem\t# double" %}
7595
7596 ins_encode( aarch64_enc_fstlrd(src, mem) );
7597
7598 ins_pipe(pipe_class_memory);
7599 %}
7600
7601 // ---------------- end of volatile loads and stores ----------------
7602
7603 instruct cacheWB(indirect addr)
7604 %{
7605 predicate(VM_Version::supports_data_cache_line_flush());
7606 match(CacheWB addr);
7607
7608 ins_cost(100);
7609 format %{"cache wb $addr" %}
7610 ins_encode %{
7611 assert($addr->index_position() < 0, "should be");
7612 assert($addr$$disp == 0, "should be");
7613 __ cache_wb(Address($addr$$base$$Register, 0));
7614 %}
7615 ins_pipe(pipe_slow); // XXX
7616 %}
7617
7618 instruct cacheWBPreSync()
7619 %{
7620 predicate(VM_Version::supports_data_cache_line_flush());
7621 match(CacheWBPreSync);
7622
7623 ins_cost(100);
7624 format %{"cache wb presync" %}
7625 ins_encode %{
7626 __ cache_wbsync(true);
7627 %}
7628 ins_pipe(pipe_slow); // XXX
7629 %}
7630
7631 instruct cacheWBPostSync()
7632 %{
7633 predicate(VM_Version::supports_data_cache_line_flush());
7634 match(CacheWBPostSync);
7635
7636 ins_cost(100);
7637 format %{"cache wb postsync" %}
7638 ins_encode %{
7639 __ cache_wbsync(false);
7640 %}
7641 ins_pipe(pipe_slow); // XXX
7642 %}
7643
7644 // ============================================================================
7645 // BSWAP Instructions
7646
7647 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7648 match(Set dst (ReverseBytesI src));
7649
7650 ins_cost(INSN_COST);
7651 format %{ "revw $dst, $src" %}
7652
7653 ins_encode %{
7654 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7655 %}
7656
7657 ins_pipe(ialu_reg);
7658 %}
7659
7660 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7661 match(Set dst (ReverseBytesL src));
7662
7663 ins_cost(INSN_COST);
7664 format %{ "rev $dst, $src" %}
7665
7666 ins_encode %{
7667 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7668 %}
7669
7670 ins_pipe(ialu_reg);
7671 %}
7672
7673 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7674 match(Set dst (ReverseBytesUS src));
7675
7676 ins_cost(INSN_COST);
7677 format %{ "rev16w $dst, $src" %}
7678
7679 ins_encode %{
7680 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7681 %}
7682
7683 ins_pipe(ialu_reg);
7684 %}
7685
7686 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7687 match(Set dst (ReverseBytesS src));
7688
7689 ins_cost(INSN_COST);
7690 format %{ "rev16w $dst, $src\n\t"
7691 "sbfmw $dst, $dst, #0, #15" %}
7692
7693 ins_encode %{
7694 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7695 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7696 %}
7697
7698 ins_pipe(ialu_reg);
7699 %}
7700
7701 // ============================================================================
7702 // Zero Count Instructions
7703
7704 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7705 match(Set dst (CountLeadingZerosI src));
7706
7707 ins_cost(INSN_COST);
7708 format %{ "clzw $dst, $src" %}
7709 ins_encode %{
7710 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7711 %}
7712
7713 ins_pipe(ialu_reg);
7714 %}
7715
7716 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7717 match(Set dst (CountLeadingZerosL src));
7718
7719 ins_cost(INSN_COST);
7720 format %{ "clz $dst, $src" %}
7721 ins_encode %{
7722 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7723 %}
7724
7725 ins_pipe(ialu_reg);
7726 %}
7727
7728 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7729 match(Set dst (CountTrailingZerosI src));
7730
7731 ins_cost(INSN_COST * 2);
7732 format %{ "rbitw $dst, $src\n\t"
7733 "clzw $dst, $dst" %}
7734 ins_encode %{
7735 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7736 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7737 %}
7738
7739 ins_pipe(ialu_reg);
7740 %}
7741
7742 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7743 match(Set dst (CountTrailingZerosL src));
7744
7745 ins_cost(INSN_COST * 2);
7746 format %{ "rbit $dst, $src\n\t"
7747 "clz $dst, $dst" %}
7748 ins_encode %{
7749 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7750 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7751 %}
7752
7753 ins_pipe(ialu_reg);
7754 %}
7755
7756 //---------- Population Count Instructions -------------------------------------
7757 //
7758
7759 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7760 match(Set dst (PopCountI src));
7761 effect(TEMP tmp);
7762 ins_cost(INSN_COST * 13);
7763
7764 format %{ "movw $src, $src\n\t"
7765 "mov $tmp, $src\t# vector (1D)\n\t"
7766 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7767 "addv $tmp, $tmp\t# vector (8B)\n\t"
7768 "mov $dst, $tmp\t# vector (1D)" %}
7769 ins_encode %{
7770 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7771 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7772 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7773 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7774 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7775 %}
7776
7777 ins_pipe(pipe_class_default);
7778 %}
7779
7780 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7781 match(Set dst (PopCountI (LoadI mem)));
7782 effect(TEMP tmp);
7783 ins_cost(INSN_COST * 13);
7784
7785 format %{ "ldrs $tmp, $mem\n\t"
7786 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7787 "addv $tmp, $tmp\t# vector (8B)\n\t"
7788 "mov $dst, $tmp\t# vector (1D)" %}
7789 ins_encode %{
7790 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7791 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7792 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7793 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7795 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7796 %}
7797
7798 ins_pipe(pipe_class_default);
7799 %}
7800
7801 // Note: Long.bitCount(long) returns an int.
7802 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7803 match(Set dst (PopCountL src));
7804 effect(TEMP tmp);
7805 ins_cost(INSN_COST * 13);
7806
7807 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7808 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7809 "addv $tmp, $tmp\t# vector (8B)\n\t"
7810 "mov $dst, $tmp\t# vector (1D)" %}
7811 ins_encode %{
7812 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7813 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7814 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7816 %}
7817
7818 ins_pipe(pipe_class_default);
7819 %}
7820
7821 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7822 match(Set dst (PopCountL (LoadL mem)));
7823 effect(TEMP tmp);
7824 ins_cost(INSN_COST * 13);
7825
7826 format %{ "ldrd $tmp, $mem\n\t"
7827 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7828 "addv $tmp, $tmp\t# vector (8B)\n\t"
7829 "mov $dst, $tmp\t# vector (1D)" %}
7830 ins_encode %{
7831 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7832 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7833 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7834 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7835 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7836 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7837 %}
7838
7839 ins_pipe(pipe_class_default);
7840 %}
7841
7842 // ============================================================================
7843 // VerifyVectorAlignment Instruction
7844
7845 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7846 match(Set addr (VerifyVectorAlignment addr mask));
7847 effect(KILL cr);
7848 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7849 ins_encode %{
7850 Label Lskip;
7851 // check if masked bits of addr are zero
7852 __ tst($addr$$Register, $mask$$constant);
7853 __ br(Assembler::EQ, Lskip);
7854 __ stop("verify_vector_alignment found a misaligned vector memory access");
7855 __ bind(Lskip);
7856 %}
7857 ins_pipe(pipe_slow);
7858 %}
7859
7860 // ============================================================================
7861 // MemBar Instruction
7862
7863 instruct load_fence() %{
7864 match(LoadFence);
7865 ins_cost(VOLATILE_REF_COST);
7866
7867 format %{ "load_fence" %}
7868
7869 ins_encode %{
7870 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7871 %}
7872 ins_pipe(pipe_serial);
7873 %}
7874
7875 instruct unnecessary_membar_acquire() %{
7876 predicate(unnecessary_acquire(n));
7877 match(MemBarAcquire);
7878 ins_cost(0);
7879
7880 format %{ "membar_acquire (elided)" %}
7881
7882 ins_encode %{
7883 __ block_comment("membar_acquire (elided)");
7884 %}
7885
7886 ins_pipe(pipe_class_empty);
7887 %}
7888
7889 instruct membar_acquire() %{
7890 match(MemBarAcquire);
7891 ins_cost(VOLATILE_REF_COST);
7892
7893 format %{ "membar_acquire\n\t"
7894 "dmb ishld" %}
7895
7896 ins_encode %{
7897 __ block_comment("membar_acquire");
7898 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7899 %}
7900
7901 ins_pipe(pipe_serial);
7902 %}
7903
7904
7905 instruct membar_acquire_lock() %{
7906 match(MemBarAcquireLock);
7907 ins_cost(VOLATILE_REF_COST);
7908
7909 format %{ "membar_acquire_lock (elided)" %}
7910
7911 ins_encode %{
7912 __ block_comment("membar_acquire_lock (elided)");
7913 %}
7914
7915 ins_pipe(pipe_serial);
7916 %}
7917
7918 instruct store_fence() %{
7919 match(StoreFence);
7920 ins_cost(VOLATILE_REF_COST);
7921
7922 format %{ "store_fence" %}
7923
7924 ins_encode %{
7925 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7926 %}
7927 ins_pipe(pipe_serial);
7928 %}
7929
7930 instruct unnecessary_membar_release() %{
7931 predicate(unnecessary_release(n));
7932 match(MemBarRelease);
7933 ins_cost(0);
7934
7935 format %{ "membar_release (elided)" %}
7936
7937 ins_encode %{
7938 __ block_comment("membar_release (elided)");
7939 %}
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 instruct membar_release() %{
7944 match(MemBarRelease);
7945 ins_cost(VOLATILE_REF_COST);
7946
7947 format %{ "membar_release\n\t"
7948 "dmb ishst\n\tdmb ishld" %}
7949
7950 ins_encode %{
7951 __ block_comment("membar_release");
7952 // These will be merged if AlwaysMergeDMB is enabled.
7953 __ membar(Assembler::StoreStore);
7954 __ membar(Assembler::LoadStore);
7955 %}
7956 ins_pipe(pipe_serial);
7957 %}
7958
7959 instruct membar_storestore() %{
7960 match(MemBarStoreStore);
7961 match(StoreStoreFence);
7962 ins_cost(VOLATILE_REF_COST);
7963
7964 format %{ "MEMBAR-store-store" %}
7965
7966 ins_encode %{
7967 __ membar(Assembler::StoreStore);
7968 %}
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 instruct membar_release_lock() %{
7973 match(MemBarReleaseLock);
7974 ins_cost(VOLATILE_REF_COST);
7975
7976 format %{ "membar_release_lock (elided)" %}
7977
7978 ins_encode %{
7979 __ block_comment("membar_release_lock (elided)");
7980 %}
7981
7982 ins_pipe(pipe_serial);
7983 %}
7984
7985 instruct unnecessary_membar_volatile() %{
7986 predicate(unnecessary_volatile(n));
7987 match(MemBarVolatile);
7988 ins_cost(0);
7989
7990 format %{ "membar_volatile (elided)" %}
7991
7992 ins_encode %{
7993 __ block_comment("membar_volatile (elided)");
7994 %}
7995
7996 ins_pipe(pipe_serial);
7997 %}
7998
7999 instruct membar_volatile() %{
8000 match(MemBarVolatile);
8001 ins_cost(VOLATILE_REF_COST*100);
8002
8003 format %{ "membar_volatile\n\t"
8004 "dmb ish"%}
8005
8006 ins_encode %{
8007 __ block_comment("membar_volatile");
8008 __ membar(Assembler::StoreLoad);
8009 %}
8010
8011 ins_pipe(pipe_serial);
8012 %}
8013
8014 // ============================================================================
8015 // Cast/Convert Instructions
8016
8017 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8018 match(Set dst (CastX2P src));
8019
8020 ins_cost(INSN_COST);
8021 format %{ "mov $dst, $src\t# long -> ptr" %}
8022
8023 ins_encode %{
8024 if ($dst$$reg != $src$$reg) {
8025 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8026 }
8027 %}
8028
8029 ins_pipe(ialu_reg);
8030 %}
8031
8032 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8033 match(Set dst (CastP2X src));
8034
8035 ins_cost(INSN_COST);
8036 format %{ "mov $dst, $src\t# ptr -> long" %}
8037
8038 ins_encode %{
8039 if ($dst$$reg != $src$$reg) {
8040 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8041 }
8042 %}
8043
8044 ins_pipe(ialu_reg);
8045 %}
8046
8047 // Convert oop into int for vectors alignment masking
8048 instruct convP2I(iRegINoSp dst, iRegP src) %{
8049 match(Set dst (ConvL2I (CastP2X src)));
8050
8051 ins_cost(INSN_COST);
8052 format %{ "movw $dst, $src\t# ptr -> int" %}
8053 ins_encode %{
8054 __ movw($dst$$Register, $src$$Register);
8055 %}
8056
8057 ins_pipe(ialu_reg);
8058 %}
8059
8060 // Convert compressed oop into int for vectors alignment masking
8061 // in case of 32bit oops (heap < 4Gb).
8062 instruct convN2I(iRegINoSp dst, iRegN src)
8063 %{
8064 predicate(CompressedOops::shift() == 0);
8065 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8066
8067 ins_cost(INSN_COST);
8068 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8069 ins_encode %{
8070 __ movw($dst$$Register, $src$$Register);
8071 %}
8072
8073 ins_pipe(ialu_reg);
8074 %}
8075
8076
8077 // Convert oop pointer into compressed form
8078 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8079 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8080 match(Set dst (EncodeP src));
8081 effect(KILL cr);
8082 ins_cost(INSN_COST * 3);
8083 format %{ "encode_heap_oop $dst, $src" %}
8084 ins_encode %{
8085 Register s = $src$$Register;
8086 Register d = $dst$$Register;
8087 __ encode_heap_oop(d, s);
8088 %}
8089 ins_pipe(ialu_reg);
8090 %}
8091
8092 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8093 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8094 match(Set dst (EncodeP src));
8095 ins_cost(INSN_COST * 3);
8096 format %{ "encode_heap_oop_not_null $dst, $src" %}
8097 ins_encode %{
8098 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8099 %}
8100 ins_pipe(ialu_reg);
8101 %}
8102
8103 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8104 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8105 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8106 match(Set dst (DecodeN src));
8107 ins_cost(INSN_COST * 3);
8108 format %{ "decode_heap_oop $dst, $src" %}
8109 ins_encode %{
8110 Register s = $src$$Register;
8111 Register d = $dst$$Register;
8112 __ decode_heap_oop(d, s);
8113 %}
8114 ins_pipe(ialu_reg);
8115 %}
8116
8117 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8118 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8119 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8120 match(Set dst (DecodeN src));
8121 ins_cost(INSN_COST * 3);
8122 format %{ "decode_heap_oop_not_null $dst, $src" %}
8123 ins_encode %{
8124 Register s = $src$$Register;
8125 Register d = $dst$$Register;
8126 __ decode_heap_oop_not_null(d, s);
8127 %}
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131 // n.b. AArch64 implementations of encode_klass_not_null and
8132 // decode_klass_not_null do not modify the flags register so, unlike
8133 // Intel, we don't kill CR as a side effect here
8134
8135 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8136 match(Set dst (EncodePKlass src));
8137
8138 ins_cost(INSN_COST * 3);
8139 format %{ "encode_klass_not_null $dst,$src" %}
8140
8141 ins_encode %{
8142 Register src_reg = as_Register($src$$reg);
8143 Register dst_reg = as_Register($dst$$reg);
8144 __ encode_klass_not_null(dst_reg, src_reg);
8145 %}
8146
8147 ins_pipe(ialu_reg);
8148 %}
8149
8150 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8151 match(Set dst (DecodeNKlass src));
8152
8153 ins_cost(INSN_COST * 3);
8154 format %{ "decode_klass_not_null $dst,$src" %}
8155
8156 ins_encode %{
8157 Register src_reg = as_Register($src$$reg);
8158 Register dst_reg = as_Register($dst$$reg);
8159 if (dst_reg != src_reg) {
8160 __ decode_klass_not_null(dst_reg, src_reg);
8161 } else {
8162 __ decode_klass_not_null(dst_reg);
8163 }
8164 %}
8165
8166 ins_pipe(ialu_reg);
8167 %}
8168
8169 instruct checkCastPP(iRegPNoSp dst)
8170 %{
8171 match(Set dst (CheckCastPP dst));
8172
8173 size(0);
8174 format %{ "# checkcastPP of $dst" %}
8175 ins_encode(/* empty encoding */);
8176 ins_pipe(pipe_class_empty);
8177 %}
8178
8179 instruct castPP(iRegPNoSp dst)
8180 %{
8181 match(Set dst (CastPP dst));
8182
8183 size(0);
8184 format %{ "# castPP of $dst" %}
8185 ins_encode(/* empty encoding */);
8186 ins_pipe(pipe_class_empty);
8187 %}
8188
8189 instruct castII(iRegI dst)
8190 %{
8191 predicate(VerifyConstraintCasts == 0);
8192 match(Set dst (CastII dst));
8193
8194 size(0);
8195 format %{ "# castII of $dst" %}
8196 ins_encode(/* empty encoding */);
8197 ins_cost(0);
8198 ins_pipe(pipe_class_empty);
8199 %}
8200
8201 instruct castII_checked(iRegI dst, rFlagsReg cr)
8202 %{
8203 predicate(VerifyConstraintCasts > 0);
8204 match(Set dst (CastII dst));
8205 effect(KILL cr);
8206
8207 format %{ "# castII_checked of $dst" %}
8208 ins_encode %{
8209 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8210 %}
8211 ins_pipe(pipe_slow);
8212 %}
8213
8214 instruct castLL(iRegL dst)
8215 %{
8216 predicate(VerifyConstraintCasts == 0);
8217 match(Set dst (CastLL dst));
8218
8219 size(0);
8220 format %{ "# castLL of $dst" %}
8221 ins_encode(/* empty encoding */);
8222 ins_cost(0);
8223 ins_pipe(pipe_class_empty);
8224 %}
8225
8226 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8227 %{
8228 predicate(VerifyConstraintCasts > 0);
8229 match(Set dst (CastLL dst));
8230 effect(KILL cr);
8231
8232 format %{ "# castLL_checked of $dst" %}
8233 ins_encode %{
8234 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8235 %}
8236 ins_pipe(pipe_slow);
8237 %}
8238
8239 instruct castFF(vRegF dst)
8240 %{
8241 match(Set dst (CastFF dst));
8242
8243 size(0);
8244 format %{ "# castFF of $dst" %}
8245 ins_encode(/* empty encoding */);
8246 ins_cost(0);
8247 ins_pipe(pipe_class_empty);
8248 %}
8249
8250 instruct castDD(vRegD dst)
8251 %{
8252 match(Set dst (CastDD dst));
8253
8254 size(0);
8255 format %{ "# castDD of $dst" %}
8256 ins_encode(/* empty encoding */);
8257 ins_cost(0);
8258 ins_pipe(pipe_class_empty);
8259 %}
8260
8261 instruct castVV(vReg dst)
8262 %{
8263 match(Set dst (CastVV dst));
8264
8265 size(0);
8266 format %{ "# castVV of $dst" %}
8267 ins_encode(/* empty encoding */);
8268 ins_cost(0);
8269 ins_pipe(pipe_class_empty);
8270 %}
8271
8272 instruct castVVMask(pRegGov dst)
8273 %{
8274 match(Set dst (CastVV dst));
8275
8276 size(0);
8277 format %{ "# castVV of $dst" %}
8278 ins_encode(/* empty encoding */);
8279 ins_cost(0);
8280 ins_pipe(pipe_class_empty);
8281 %}
8282
8283 // ============================================================================
8284 // Atomic operation instructions
8285 //
8286
8287 // standard CompareAndSwapX when we are using barriers
8288 // these have higher priority than the rules selected by a predicate
8289
8290 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8291 // can't match them
8292
8293 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8294
8295 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8296 ins_cost(2 * VOLATILE_REF_COST);
8297
8298 effect(KILL cr);
8299
8300 format %{
8301 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8302 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8303 %}
8304
8305 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8306 aarch64_enc_cset_eq(res));
8307
8308 ins_pipe(pipe_slow);
8309 %}
8310
8311 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8312
8313 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8314 ins_cost(2 * VOLATILE_REF_COST);
8315
8316 effect(KILL cr);
8317
8318 format %{
8319 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8320 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8321 %}
8322
8323 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8324 aarch64_enc_cset_eq(res));
8325
8326 ins_pipe(pipe_slow);
8327 %}
8328
8329 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8330
8331 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8332 ins_cost(2 * VOLATILE_REF_COST);
8333
8334 effect(KILL cr);
8335
8336 format %{
8337 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8338 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8339 %}
8340
8341 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8342 aarch64_enc_cset_eq(res));
8343
8344 ins_pipe(pipe_slow);
8345 %}
8346
8347 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8348
8349 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8350 ins_cost(2 * VOLATILE_REF_COST);
8351
8352 effect(KILL cr);
8353
8354 format %{
8355 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8356 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8357 %}
8358
8359 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8360 aarch64_enc_cset_eq(res));
8361
8362 ins_pipe(pipe_slow);
8363 %}
8364
8365 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8366
8367 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8368 predicate(n->as_LoadStore()->barrier_data() == 0);
8369 ins_cost(2 * VOLATILE_REF_COST);
8370
8371 effect(KILL cr);
8372
8373 format %{
8374 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8375 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8376 %}
8377
8378 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8379 aarch64_enc_cset_eq(res));
8380
8381 ins_pipe(pipe_slow);
8382 %}
8383
8384 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8385
8386 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8387 predicate(n->as_LoadStore()->barrier_data() == 0);
8388 ins_cost(2 * VOLATILE_REF_COST);
8389
8390 effect(KILL cr);
8391
8392 format %{
8393 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8394 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8395 %}
8396
8397 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8398 aarch64_enc_cset_eq(res));
8399
8400 ins_pipe(pipe_slow);
8401 %}
8402
8403 // alternative CompareAndSwapX when we are eliding barriers
8404
8405 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8406
8407 predicate(needs_acquiring_load_exclusive(n));
8408 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8409 ins_cost(VOLATILE_REF_COST);
8410
8411 effect(KILL cr);
8412
8413 format %{
8414 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8415 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8416 %}
8417
8418 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8419 aarch64_enc_cset_eq(res));
8420
8421 ins_pipe(pipe_slow);
8422 %}
8423
8424 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8425
8426 predicate(needs_acquiring_load_exclusive(n));
8427 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8428 ins_cost(VOLATILE_REF_COST);
8429
8430 effect(KILL cr);
8431
8432 format %{
8433 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8434 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8435 %}
8436
8437 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8438 aarch64_enc_cset_eq(res));
8439
8440 ins_pipe(pipe_slow);
8441 %}
8442
8443 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8444
8445 predicate(needs_acquiring_load_exclusive(n));
8446 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8447 ins_cost(VOLATILE_REF_COST);
8448
8449 effect(KILL cr);
8450
8451 format %{
8452 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8453 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8454 %}
8455
8456 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8457 aarch64_enc_cset_eq(res));
8458
8459 ins_pipe(pipe_slow);
8460 %}
8461
8462 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8463
8464 predicate(needs_acquiring_load_exclusive(n));
8465 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8466 ins_cost(VOLATILE_REF_COST);
8467
8468 effect(KILL cr);
8469
8470 format %{
8471 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8472 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8473 %}
8474
8475 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8476 aarch64_enc_cset_eq(res));
8477
8478 ins_pipe(pipe_slow);
8479 %}
8480
8481 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8482
8483 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8484 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8485 ins_cost(VOLATILE_REF_COST);
8486
8487 effect(KILL cr);
8488
8489 format %{
8490 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8491 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8492 %}
8493
8494 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8495 aarch64_enc_cset_eq(res));
8496
8497 ins_pipe(pipe_slow);
8498 %}
8499
8500 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8501
8502 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8503 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8504 ins_cost(VOLATILE_REF_COST);
8505
8506 effect(KILL cr);
8507
8508 format %{
8509 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8510 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8511 %}
8512
8513 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8514 aarch64_enc_cset_eq(res));
8515
8516 ins_pipe(pipe_slow);
8517 %}
8518
8519
8520 // ---------------------------------------------------------------------
8521
8522 // BEGIN This section of the file is automatically generated. Do not edit --------------
8523
8524 // Sundry CAS operations. Note that release is always true,
8525 // regardless of the memory ordering of the CAS. This is because we
8526 // need the volatile case to be sequentially consistent but there is
8527 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8528 // can't check the type of memory ordering here, so we always emit a
8529 // STLXR.
8530
8531 // This section is generated from cas.m4
8532
8533
8534 // This pattern is generated automatically from cas.m4.
8535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8536 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8537 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8538 ins_cost(2 * VOLATILE_REF_COST);
8539 effect(TEMP_DEF res, KILL cr);
8540 format %{
8541 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8542 %}
8543 ins_encode %{
8544 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8545 Assembler::byte, /*acquire*/ false, /*release*/ true,
8546 /*weak*/ false, $res$$Register);
8547 __ sxtbw($res$$Register, $res$$Register);
8548 %}
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552 // This pattern is generated automatically from cas.m4.
8553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8554 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8555 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8556 ins_cost(2 * VOLATILE_REF_COST);
8557 effect(TEMP_DEF res, KILL cr);
8558 format %{
8559 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8560 %}
8561 ins_encode %{
8562 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8563 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8564 /*weak*/ false, $res$$Register);
8565 __ sxthw($res$$Register, $res$$Register);
8566 %}
8567 ins_pipe(pipe_slow);
8568 %}
8569
8570 // This pattern is generated automatically from cas.m4.
8571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8572 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8573 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8574 ins_cost(2 * VOLATILE_REF_COST);
8575 effect(TEMP_DEF res, KILL cr);
8576 format %{
8577 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8578 %}
8579 ins_encode %{
8580 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8581 Assembler::word, /*acquire*/ false, /*release*/ true,
8582 /*weak*/ false, $res$$Register);
8583 %}
8584 ins_pipe(pipe_slow);
8585 %}
8586
8587 // This pattern is generated automatically from cas.m4.
8588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8589 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8590 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8591 ins_cost(2 * VOLATILE_REF_COST);
8592 effect(TEMP_DEF res, KILL cr);
8593 format %{
8594 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8595 %}
8596 ins_encode %{
8597 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8598 Assembler::xword, /*acquire*/ false, /*release*/ true,
8599 /*weak*/ false, $res$$Register);
8600 %}
8601 ins_pipe(pipe_slow);
8602 %}
8603
8604 // This pattern is generated automatically from cas.m4.
8605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8606 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8607 predicate(n->as_LoadStore()->barrier_data() == 0);
8608 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8609 ins_cost(2 * VOLATILE_REF_COST);
8610 effect(TEMP_DEF res, KILL cr);
8611 format %{
8612 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8613 %}
8614 ins_encode %{
8615 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8616 Assembler::word, /*acquire*/ false, /*release*/ true,
8617 /*weak*/ false, $res$$Register);
8618 %}
8619 ins_pipe(pipe_slow);
8620 %}
8621
8622 // This pattern is generated automatically from cas.m4.
8623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8624 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8625 predicate(n->as_LoadStore()->barrier_data() == 0);
8626 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8627 ins_cost(2 * VOLATILE_REF_COST);
8628 effect(TEMP_DEF res, KILL cr);
8629 format %{
8630 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8631 %}
8632 ins_encode %{
8633 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8634 Assembler::xword, /*acquire*/ false, /*release*/ true,
8635 /*weak*/ false, $res$$Register);
8636 %}
8637 ins_pipe(pipe_slow);
8638 %}
8639
8640 // This pattern is generated automatically from cas.m4.
8641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8642 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8643 predicate(needs_acquiring_load_exclusive(n));
8644 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8645 ins_cost(VOLATILE_REF_COST);
8646 effect(TEMP_DEF res, KILL cr);
8647 format %{
8648 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8649 %}
8650 ins_encode %{
8651 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8652 Assembler::byte, /*acquire*/ true, /*release*/ true,
8653 /*weak*/ false, $res$$Register);
8654 __ sxtbw($res$$Register, $res$$Register);
8655 %}
8656 ins_pipe(pipe_slow);
8657 %}
8658
8659 // This pattern is generated automatically from cas.m4.
8660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8661 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8662 predicate(needs_acquiring_load_exclusive(n));
8663 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8664 ins_cost(VOLATILE_REF_COST);
8665 effect(TEMP_DEF res, KILL cr);
8666 format %{
8667 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8668 %}
8669 ins_encode %{
8670 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8671 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8672 /*weak*/ false, $res$$Register);
8673 __ sxthw($res$$Register, $res$$Register);
8674 %}
8675 ins_pipe(pipe_slow);
8676 %}
8677
8678 // This pattern is generated automatically from cas.m4.
8679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8680 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8681 predicate(needs_acquiring_load_exclusive(n));
8682 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8683 ins_cost(VOLATILE_REF_COST);
8684 effect(TEMP_DEF res, KILL cr);
8685 format %{
8686 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8687 %}
8688 ins_encode %{
8689 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8690 Assembler::word, /*acquire*/ true, /*release*/ true,
8691 /*weak*/ false, $res$$Register);
8692 %}
8693 ins_pipe(pipe_slow);
8694 %}
8695
8696 // This pattern is generated automatically from cas.m4.
8697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8698 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8699 predicate(needs_acquiring_load_exclusive(n));
8700 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8701 ins_cost(VOLATILE_REF_COST);
8702 effect(TEMP_DEF res, KILL cr);
8703 format %{
8704 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8705 %}
8706 ins_encode %{
8707 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8708 Assembler::xword, /*acquire*/ true, /*release*/ true,
8709 /*weak*/ false, $res$$Register);
8710 %}
8711 ins_pipe(pipe_slow);
8712 %}
8713
8714 // This pattern is generated automatically from cas.m4.
8715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8716 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8717 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8718 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8719 ins_cost(VOLATILE_REF_COST);
8720 effect(TEMP_DEF res, KILL cr);
8721 format %{
8722 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8723 %}
8724 ins_encode %{
8725 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8726 Assembler::word, /*acquire*/ true, /*release*/ true,
8727 /*weak*/ false, $res$$Register);
8728 %}
8729 ins_pipe(pipe_slow);
8730 %}
8731
8732 // This pattern is generated automatically from cas.m4.
8733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8734 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8735 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8736 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8737 ins_cost(VOLATILE_REF_COST);
8738 effect(TEMP_DEF res, KILL cr);
8739 format %{
8740 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8741 %}
8742 ins_encode %{
8743 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8744 Assembler::xword, /*acquire*/ true, /*release*/ true,
8745 /*weak*/ false, $res$$Register);
8746 %}
8747 ins_pipe(pipe_slow);
8748 %}
8749
8750 // This pattern is generated automatically from cas.m4.
8751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8752 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8753 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8754 ins_cost(2 * VOLATILE_REF_COST);
8755 effect(KILL cr);
8756 format %{
8757 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8758 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8759 %}
8760 ins_encode %{
8761 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8762 Assembler::byte, /*acquire*/ false, /*release*/ true,
8763 /*weak*/ true, noreg);
8764 __ csetw($res$$Register, Assembler::EQ);
8765 %}
8766 ins_pipe(pipe_slow);
8767 %}
8768
8769 // This pattern is generated automatically from cas.m4.
8770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8771 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8772 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8773 ins_cost(2 * VOLATILE_REF_COST);
8774 effect(KILL cr);
8775 format %{
8776 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8777 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8778 %}
8779 ins_encode %{
8780 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8781 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8782 /*weak*/ true, noreg);
8783 __ csetw($res$$Register, Assembler::EQ);
8784 %}
8785 ins_pipe(pipe_slow);
8786 %}
8787
8788 // This pattern is generated automatically from cas.m4.
8789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8790 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8791 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8792 ins_cost(2 * VOLATILE_REF_COST);
8793 effect(KILL cr);
8794 format %{
8795 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8796 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8797 %}
8798 ins_encode %{
8799 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8800 Assembler::word, /*acquire*/ false, /*release*/ true,
8801 /*weak*/ true, noreg);
8802 __ csetw($res$$Register, Assembler::EQ);
8803 %}
8804 ins_pipe(pipe_slow);
8805 %}
8806
8807 // This pattern is generated automatically from cas.m4.
8808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8809 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8810 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8811 ins_cost(2 * VOLATILE_REF_COST);
8812 effect(KILL cr);
8813 format %{
8814 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8815 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8816 %}
8817 ins_encode %{
8818 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8819 Assembler::xword, /*acquire*/ false, /*release*/ true,
8820 /*weak*/ true, noreg);
8821 __ csetw($res$$Register, Assembler::EQ);
8822 %}
8823 ins_pipe(pipe_slow);
8824 %}
8825
8826 // This pattern is generated automatically from cas.m4.
8827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8828 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8829 predicate(n->as_LoadStore()->barrier_data() == 0);
8830 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8831 ins_cost(2 * VOLATILE_REF_COST);
8832 effect(KILL cr);
8833 format %{
8834 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8835 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8836 %}
8837 ins_encode %{
8838 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8839 Assembler::word, /*acquire*/ false, /*release*/ true,
8840 /*weak*/ true, noreg);
8841 __ csetw($res$$Register, Assembler::EQ);
8842 %}
8843 ins_pipe(pipe_slow);
8844 %}
8845
8846 // This pattern is generated automatically from cas.m4.
8847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8848 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8849 predicate(n->as_LoadStore()->barrier_data() == 0);
8850 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8851 ins_cost(2 * VOLATILE_REF_COST);
8852 effect(KILL cr);
8853 format %{
8854 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8855 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8856 %}
8857 ins_encode %{
8858 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8859 Assembler::xword, /*acquire*/ false, /*release*/ true,
8860 /*weak*/ true, noreg);
8861 __ csetw($res$$Register, Assembler::EQ);
8862 %}
8863 ins_pipe(pipe_slow);
8864 %}
8865
8866 // This pattern is generated automatically from cas.m4.
8867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8868 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8869 predicate(needs_acquiring_load_exclusive(n));
8870 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8871 ins_cost(VOLATILE_REF_COST);
8872 effect(KILL cr);
8873 format %{
8874 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8875 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8876 %}
8877 ins_encode %{
8878 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8879 Assembler::byte, /*acquire*/ true, /*release*/ true,
8880 /*weak*/ true, noreg);
8881 __ csetw($res$$Register, Assembler::EQ);
8882 %}
8883 ins_pipe(pipe_slow);
8884 %}
8885
8886 // This pattern is generated automatically from cas.m4.
8887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8888 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8889 predicate(needs_acquiring_load_exclusive(n));
8890 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8891 ins_cost(VOLATILE_REF_COST);
8892 effect(KILL cr);
8893 format %{
8894 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8895 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8896 %}
8897 ins_encode %{
8898 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8899 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8900 /*weak*/ true, noreg);
8901 __ csetw($res$$Register, Assembler::EQ);
8902 %}
8903 ins_pipe(pipe_slow);
8904 %}
8905
8906 // This pattern is generated automatically from cas.m4.
8907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8908 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8909 predicate(needs_acquiring_load_exclusive(n));
8910 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8911 ins_cost(VOLATILE_REF_COST);
8912 effect(KILL cr);
8913 format %{
8914 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8915 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8916 %}
8917 ins_encode %{
8918 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8919 Assembler::word, /*acquire*/ true, /*release*/ true,
8920 /*weak*/ true, noreg);
8921 __ csetw($res$$Register, Assembler::EQ);
8922 %}
8923 ins_pipe(pipe_slow);
8924 %}
8925
8926 // This pattern is generated automatically from cas.m4.
8927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8928 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8929 predicate(needs_acquiring_load_exclusive(n));
8930 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8931 ins_cost(VOLATILE_REF_COST);
8932 effect(KILL cr);
8933 format %{
8934 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8935 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8936 %}
8937 ins_encode %{
8938 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8939 Assembler::xword, /*acquire*/ true, /*release*/ true,
8940 /*weak*/ true, noreg);
8941 __ csetw($res$$Register, Assembler::EQ);
8942 %}
8943 ins_pipe(pipe_slow);
8944 %}
8945
8946 // This pattern is generated automatically from cas.m4.
8947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8948 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8949 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8950 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8951 ins_cost(VOLATILE_REF_COST);
8952 effect(KILL cr);
8953 format %{
8954 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8955 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8956 %}
8957 ins_encode %{
8958 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8959 Assembler::word, /*acquire*/ true, /*release*/ true,
8960 /*weak*/ true, noreg);
8961 __ csetw($res$$Register, Assembler::EQ);
8962 %}
8963 ins_pipe(pipe_slow);
8964 %}
8965
8966 // This pattern is generated automatically from cas.m4.
8967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8968 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8969 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8970 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8971 ins_cost(VOLATILE_REF_COST);
8972 effect(KILL cr);
8973 format %{
8974 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8975 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8976 %}
8977 ins_encode %{
8978 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8979 Assembler::xword, /*acquire*/ true, /*release*/ true,
8980 /*weak*/ true, noreg);
8981 __ csetw($res$$Register, Assembler::EQ);
8982 %}
8983 ins_pipe(pipe_slow);
8984 %}
8985
8986 // END This section of the file is automatically generated. Do not edit --------------
8987 // ---------------------------------------------------------------------
8988
8989 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8990 match(Set prev (GetAndSetI mem newv));
8991 ins_cost(2 * VOLATILE_REF_COST);
8992 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8993 ins_encode %{
8994 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8995 %}
8996 ins_pipe(pipe_serial);
8997 %}
8998
8999 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9000 match(Set prev (GetAndSetL mem newv));
9001 ins_cost(2 * VOLATILE_REF_COST);
9002 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9003 ins_encode %{
9004 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9005 %}
9006 ins_pipe(pipe_serial);
9007 %}
9008
9009 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9010 predicate(n->as_LoadStore()->barrier_data() == 0);
9011 match(Set prev (GetAndSetN mem newv));
9012 ins_cost(2 * VOLATILE_REF_COST);
9013 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9014 ins_encode %{
9015 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9016 %}
9017 ins_pipe(pipe_serial);
9018 %}
9019
9020 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9021 predicate(n->as_LoadStore()->barrier_data() == 0);
9022 match(Set prev (GetAndSetP mem newv));
9023 ins_cost(2 * VOLATILE_REF_COST);
9024 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9025 ins_encode %{
9026 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9027 %}
9028 ins_pipe(pipe_serial);
9029 %}
9030
9031 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9032 predicate(needs_acquiring_load_exclusive(n));
9033 match(Set prev (GetAndSetI mem newv));
9034 ins_cost(VOLATILE_REF_COST);
9035 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9036 ins_encode %{
9037 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9038 %}
9039 ins_pipe(pipe_serial);
9040 %}
9041
9042 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9043 predicate(needs_acquiring_load_exclusive(n));
9044 match(Set prev (GetAndSetL mem newv));
9045 ins_cost(VOLATILE_REF_COST);
9046 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9047 ins_encode %{
9048 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9049 %}
9050 ins_pipe(pipe_serial);
9051 %}
9052
9053 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9054 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9055 match(Set prev (GetAndSetN mem newv));
9056 ins_cost(VOLATILE_REF_COST);
9057 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9058 ins_encode %{
9059 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9060 %}
9061 ins_pipe(pipe_serial);
9062 %}
9063
9064 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9065 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9066 match(Set prev (GetAndSetP mem newv));
9067 ins_cost(VOLATILE_REF_COST);
9068 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9069 ins_encode %{
9070 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9071 %}
9072 ins_pipe(pipe_serial);
9073 %}
9074
9075
9076 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9077 match(Set newval (GetAndAddL mem incr));
9078 ins_cost(2 * VOLATILE_REF_COST + 1);
9079 format %{ "get_and_addL $newval, [$mem], $incr" %}
9080 ins_encode %{
9081 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9082 %}
9083 ins_pipe(pipe_serial);
9084 %}
9085
9086 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9087 predicate(n->as_LoadStore()->result_not_used());
9088 match(Set dummy (GetAndAddL mem incr));
9089 ins_cost(2 * VOLATILE_REF_COST);
9090 format %{ "get_and_addL [$mem], $incr" %}
9091 ins_encode %{
9092 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9093 %}
9094 ins_pipe(pipe_serial);
9095 %}
9096
9097 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9098 match(Set newval (GetAndAddL mem incr));
9099 ins_cost(2 * VOLATILE_REF_COST + 1);
9100 format %{ "get_and_addL $newval, [$mem], $incr" %}
9101 ins_encode %{
9102 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9103 %}
9104 ins_pipe(pipe_serial);
9105 %}
9106
9107 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9108 predicate(n->as_LoadStore()->result_not_used());
9109 match(Set dummy (GetAndAddL mem incr));
9110 ins_cost(2 * VOLATILE_REF_COST);
9111 format %{ "get_and_addL [$mem], $incr" %}
9112 ins_encode %{
9113 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9114 %}
9115 ins_pipe(pipe_serial);
9116 %}
9117
9118 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9119 match(Set newval (GetAndAddI mem incr));
9120 ins_cost(2 * VOLATILE_REF_COST + 1);
9121 format %{ "get_and_addI $newval, [$mem], $incr" %}
9122 ins_encode %{
9123 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9124 %}
9125 ins_pipe(pipe_serial);
9126 %}
9127
9128 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9129 predicate(n->as_LoadStore()->result_not_used());
9130 match(Set dummy (GetAndAddI mem incr));
9131 ins_cost(2 * VOLATILE_REF_COST);
9132 format %{ "get_and_addI [$mem], $incr" %}
9133 ins_encode %{
9134 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9135 %}
9136 ins_pipe(pipe_serial);
9137 %}
9138
9139 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9140 match(Set newval (GetAndAddI mem incr));
9141 ins_cost(2 * VOLATILE_REF_COST + 1);
9142 format %{ "get_and_addI $newval, [$mem], $incr" %}
9143 ins_encode %{
9144 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9145 %}
9146 ins_pipe(pipe_serial);
9147 %}
9148
9149 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9150 predicate(n->as_LoadStore()->result_not_used());
9151 match(Set dummy (GetAndAddI mem incr));
9152 ins_cost(2 * VOLATILE_REF_COST);
9153 format %{ "get_and_addI [$mem], $incr" %}
9154 ins_encode %{
9155 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9156 %}
9157 ins_pipe(pipe_serial);
9158 %}
9159
9160 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9161 predicate(needs_acquiring_load_exclusive(n));
9162 match(Set newval (GetAndAddL mem incr));
9163 ins_cost(VOLATILE_REF_COST + 1);
9164 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9165 ins_encode %{
9166 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9167 %}
9168 ins_pipe(pipe_serial);
9169 %}
9170
9171 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9172 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9173 match(Set dummy (GetAndAddL mem incr));
9174 ins_cost(VOLATILE_REF_COST);
9175 format %{ "get_and_addL_acq [$mem], $incr" %}
9176 ins_encode %{
9177 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9178 %}
9179 ins_pipe(pipe_serial);
9180 %}
9181
9182 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9183 predicate(needs_acquiring_load_exclusive(n));
9184 match(Set newval (GetAndAddL mem incr));
9185 ins_cost(VOLATILE_REF_COST + 1);
9186 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9187 ins_encode %{
9188 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9189 %}
9190 ins_pipe(pipe_serial);
9191 %}
9192
9193 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9194 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9195 match(Set dummy (GetAndAddL mem incr));
9196 ins_cost(VOLATILE_REF_COST);
9197 format %{ "get_and_addL_acq [$mem], $incr" %}
9198 ins_encode %{
9199 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9200 %}
9201 ins_pipe(pipe_serial);
9202 %}
9203
9204 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9205 predicate(needs_acquiring_load_exclusive(n));
9206 match(Set newval (GetAndAddI mem incr));
9207 ins_cost(VOLATILE_REF_COST + 1);
9208 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9209 ins_encode %{
9210 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9211 %}
9212 ins_pipe(pipe_serial);
9213 %}
9214
9215 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9216 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9217 match(Set dummy (GetAndAddI mem incr));
9218 ins_cost(VOLATILE_REF_COST);
9219 format %{ "get_and_addI_acq [$mem], $incr" %}
9220 ins_encode %{
9221 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9222 %}
9223 ins_pipe(pipe_serial);
9224 %}
9225
9226 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9227 predicate(needs_acquiring_load_exclusive(n));
9228 match(Set newval (GetAndAddI mem incr));
9229 ins_cost(VOLATILE_REF_COST + 1);
9230 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9231 ins_encode %{
9232 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9233 %}
9234 ins_pipe(pipe_serial);
9235 %}
9236
9237 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9238 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9239 match(Set dummy (GetAndAddI mem incr));
9240 ins_cost(VOLATILE_REF_COST);
9241 format %{ "get_and_addI_acq [$mem], $incr" %}
9242 ins_encode %{
9243 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9244 %}
9245 ins_pipe(pipe_serial);
9246 %}
9247
9248 // Manifest a CmpU result in an integer register.
9249 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9250 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9251 %{
9252 match(Set dst (CmpU3 src1 src2));
9253 effect(KILL flags);
9254
9255 ins_cost(INSN_COST * 3);
9256 format %{
9257 "cmpw $src1, $src2\n\t"
9258 "csetw $dst, ne\n\t"
9259 "cnegw $dst, lo\t# CmpU3(reg)"
9260 %}
9261 ins_encode %{
9262 __ cmpw($src1$$Register, $src2$$Register);
9263 __ csetw($dst$$Register, Assembler::NE);
9264 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9265 %}
9266
9267 ins_pipe(pipe_class_default);
9268 %}
9269
9270 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9271 %{
9272 match(Set dst (CmpU3 src1 src2));
9273 effect(KILL flags);
9274
9275 ins_cost(INSN_COST * 3);
9276 format %{
9277 "subsw zr, $src1, $src2\n\t"
9278 "csetw $dst, ne\n\t"
9279 "cnegw $dst, lo\t# CmpU3(imm)"
9280 %}
9281 ins_encode %{
9282 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9283 __ csetw($dst$$Register, Assembler::NE);
9284 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9285 %}
9286
9287 ins_pipe(pipe_class_default);
9288 %}
9289
9290 // Manifest a CmpUL result in an integer register.
9291 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9292 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9293 %{
9294 match(Set dst (CmpUL3 src1 src2));
9295 effect(KILL flags);
9296
9297 ins_cost(INSN_COST * 3);
9298 format %{
9299 "cmp $src1, $src2\n\t"
9300 "csetw $dst, ne\n\t"
9301 "cnegw $dst, lo\t# CmpUL3(reg)"
9302 %}
9303 ins_encode %{
9304 __ cmp($src1$$Register, $src2$$Register);
9305 __ csetw($dst$$Register, Assembler::NE);
9306 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9307 %}
9308
9309 ins_pipe(pipe_class_default);
9310 %}
9311
9312 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9313 %{
9314 match(Set dst (CmpUL3 src1 src2));
9315 effect(KILL flags);
9316
9317 ins_cost(INSN_COST * 3);
9318 format %{
9319 "subs zr, $src1, $src2\n\t"
9320 "csetw $dst, ne\n\t"
9321 "cnegw $dst, lo\t# CmpUL3(imm)"
9322 %}
9323 ins_encode %{
9324 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9325 __ csetw($dst$$Register, Assembler::NE);
9326 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9327 %}
9328
9329 ins_pipe(pipe_class_default);
9330 %}
9331
9332 // Manifest a CmpL result in an integer register.
9333 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9334 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9335 %{
9336 match(Set dst (CmpL3 src1 src2));
9337 effect(KILL flags);
9338
9339 ins_cost(INSN_COST * 3);
9340 format %{
9341 "cmp $src1, $src2\n\t"
9342 "csetw $dst, ne\n\t"
9343 "cnegw $dst, lt\t# CmpL3(reg)"
9344 %}
9345 ins_encode %{
9346 __ cmp($src1$$Register, $src2$$Register);
9347 __ csetw($dst$$Register, Assembler::NE);
9348 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9349 %}
9350
9351 ins_pipe(pipe_class_default);
9352 %}
9353
9354 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9355 %{
9356 match(Set dst (CmpL3 src1 src2));
9357 effect(KILL flags);
9358
9359 ins_cost(INSN_COST * 3);
9360 format %{
9361 "subs zr, $src1, $src2\n\t"
9362 "csetw $dst, ne\n\t"
9363 "cnegw $dst, lt\t# CmpL3(imm)"
9364 %}
9365 ins_encode %{
9366 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9367 __ csetw($dst$$Register, Assembler::NE);
9368 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9369 %}
9370
9371 ins_pipe(pipe_class_default);
9372 %}
9373
9374 // ============================================================================
9375 // Conditional Move Instructions
9376
9377 // n.b. we have identical rules for both a signed compare op (cmpOp)
9378 // and an unsigned compare op (cmpOpU). it would be nice if we could
9379 // define an op class which merged both inputs and use it to type the
9380 // argument to a single rule. unfortunatelyt his fails because the
9381 // opclass does not live up to the COND_INTER interface of its
9382 // component operands. When the generic code tries to negate the
9383 // operand it ends up running the generci Machoper::negate method
9384 // which throws a ShouldNotHappen. So, we have to provide two flavours
9385 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9386
9387 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9388 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9389
9390 ins_cost(INSN_COST * 2);
9391 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9392
9393 ins_encode %{
9394 __ cselw(as_Register($dst$$reg),
9395 as_Register($src2$$reg),
9396 as_Register($src1$$reg),
9397 (Assembler::Condition)$cmp$$cmpcode);
9398 %}
9399
9400 ins_pipe(icond_reg_reg);
9401 %}
9402
9403 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9404 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9405
9406 ins_cost(INSN_COST * 2);
9407 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9408
9409 ins_encode %{
9410 __ cselw(as_Register($dst$$reg),
9411 as_Register($src2$$reg),
9412 as_Register($src1$$reg),
9413 (Assembler::Condition)$cmp$$cmpcode);
9414 %}
9415
9416 ins_pipe(icond_reg_reg);
9417 %}
9418
9419 // special cases where one arg is zero
9420
9421 // n.b. this is selected in preference to the rule above because it
9422 // avoids loading constant 0 into a source register
9423
9424 // TODO
9425 // we ought only to be able to cull one of these variants as the ideal
9426 // transforms ought always to order the zero consistently (to left/right?)
9427
9428 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9429 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9430
9431 ins_cost(INSN_COST * 2);
9432 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9433
9434 ins_encode %{
9435 __ cselw(as_Register($dst$$reg),
9436 as_Register($src$$reg),
9437 zr,
9438 (Assembler::Condition)$cmp$$cmpcode);
9439 %}
9440
9441 ins_pipe(icond_reg);
9442 %}
9443
9444 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9445 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9446
9447 ins_cost(INSN_COST * 2);
9448 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9449
9450 ins_encode %{
9451 __ cselw(as_Register($dst$$reg),
9452 as_Register($src$$reg),
9453 zr,
9454 (Assembler::Condition)$cmp$$cmpcode);
9455 %}
9456
9457 ins_pipe(icond_reg);
9458 %}
9459
9460 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9461 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9462
9463 ins_cost(INSN_COST * 2);
9464 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9465
9466 ins_encode %{
9467 __ cselw(as_Register($dst$$reg),
9468 zr,
9469 as_Register($src$$reg),
9470 (Assembler::Condition)$cmp$$cmpcode);
9471 %}
9472
9473 ins_pipe(icond_reg);
9474 %}
9475
9476 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9477 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9478
9479 ins_cost(INSN_COST * 2);
9480 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9481
9482 ins_encode %{
9483 __ cselw(as_Register($dst$$reg),
9484 zr,
9485 as_Register($src$$reg),
9486 (Assembler::Condition)$cmp$$cmpcode);
9487 %}
9488
9489 ins_pipe(icond_reg);
9490 %}
9491
9492 // special case for creating a boolean 0 or 1
9493
9494 // n.b. this is selected in preference to the rule above because it
9495 // avoids loading constants 0 and 1 into a source register
9496
9497 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9498 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9499
9500 ins_cost(INSN_COST * 2);
9501 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9502
9503 ins_encode %{
9504 // equivalently
9505 // cset(as_Register($dst$$reg),
9506 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9507 __ csincw(as_Register($dst$$reg),
9508 zr,
9509 zr,
9510 (Assembler::Condition)$cmp$$cmpcode);
9511 %}
9512
9513 ins_pipe(icond_none);
9514 %}
9515
9516 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9517 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9518
9519 ins_cost(INSN_COST * 2);
9520 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9521
9522 ins_encode %{
9523 // equivalently
9524 // cset(as_Register($dst$$reg),
9525 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9526 __ csincw(as_Register($dst$$reg),
9527 zr,
9528 zr,
9529 (Assembler::Condition)$cmp$$cmpcode);
9530 %}
9531
9532 ins_pipe(icond_none);
9533 %}
9534
9535 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9536 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9537
9538 ins_cost(INSN_COST * 2);
9539 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9540
9541 ins_encode %{
9542 __ csel(as_Register($dst$$reg),
9543 as_Register($src2$$reg),
9544 as_Register($src1$$reg),
9545 (Assembler::Condition)$cmp$$cmpcode);
9546 %}
9547
9548 ins_pipe(icond_reg_reg);
9549 %}
9550
9551 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9552 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9553
9554 ins_cost(INSN_COST * 2);
9555 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9556
9557 ins_encode %{
9558 __ csel(as_Register($dst$$reg),
9559 as_Register($src2$$reg),
9560 as_Register($src1$$reg),
9561 (Assembler::Condition)$cmp$$cmpcode);
9562 %}
9563
9564 ins_pipe(icond_reg_reg);
9565 %}
9566
9567 // special cases where one arg is zero
9568
9569 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9570 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9571
9572 ins_cost(INSN_COST * 2);
9573 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9574
9575 ins_encode %{
9576 __ csel(as_Register($dst$$reg),
9577 zr,
9578 as_Register($src$$reg),
9579 (Assembler::Condition)$cmp$$cmpcode);
9580 %}
9581
9582 ins_pipe(icond_reg);
9583 %}
9584
9585 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9586 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9587
9588 ins_cost(INSN_COST * 2);
9589 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9590
9591 ins_encode %{
9592 __ csel(as_Register($dst$$reg),
9593 zr,
9594 as_Register($src$$reg),
9595 (Assembler::Condition)$cmp$$cmpcode);
9596 %}
9597
9598 ins_pipe(icond_reg);
9599 %}
9600
9601 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9602 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9603
9604 ins_cost(INSN_COST * 2);
9605 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9606
9607 ins_encode %{
9608 __ csel(as_Register($dst$$reg),
9609 as_Register($src$$reg),
9610 zr,
9611 (Assembler::Condition)$cmp$$cmpcode);
9612 %}
9613
9614 ins_pipe(icond_reg);
9615 %}
9616
9617 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9618 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9619
9620 ins_cost(INSN_COST * 2);
9621 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9622
9623 ins_encode %{
9624 __ csel(as_Register($dst$$reg),
9625 as_Register($src$$reg),
9626 zr,
9627 (Assembler::Condition)$cmp$$cmpcode);
9628 %}
9629
9630 ins_pipe(icond_reg);
9631 %}
9632
9633 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9634 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9635
9636 ins_cost(INSN_COST * 2);
9637 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9638
9639 ins_encode %{
9640 __ csel(as_Register($dst$$reg),
9641 as_Register($src2$$reg),
9642 as_Register($src1$$reg),
9643 (Assembler::Condition)$cmp$$cmpcode);
9644 %}
9645
9646 ins_pipe(icond_reg_reg);
9647 %}
9648
9649 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9650 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9651
9652 ins_cost(INSN_COST * 2);
9653 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9654
9655 ins_encode %{
9656 __ csel(as_Register($dst$$reg),
9657 as_Register($src2$$reg),
9658 as_Register($src1$$reg),
9659 (Assembler::Condition)$cmp$$cmpcode);
9660 %}
9661
9662 ins_pipe(icond_reg_reg);
9663 %}
9664
9665 // special cases where one arg is zero
9666
9667 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9668 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9669
9670 ins_cost(INSN_COST * 2);
9671 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9672
9673 ins_encode %{
9674 __ csel(as_Register($dst$$reg),
9675 zr,
9676 as_Register($src$$reg),
9677 (Assembler::Condition)$cmp$$cmpcode);
9678 %}
9679
9680 ins_pipe(icond_reg);
9681 %}
9682
9683 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9684 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9685
9686 ins_cost(INSN_COST * 2);
9687 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9688
9689 ins_encode %{
9690 __ csel(as_Register($dst$$reg),
9691 zr,
9692 as_Register($src$$reg),
9693 (Assembler::Condition)$cmp$$cmpcode);
9694 %}
9695
9696 ins_pipe(icond_reg);
9697 %}
9698
9699 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9700 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9701
9702 ins_cost(INSN_COST * 2);
9703 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9704
9705 ins_encode %{
9706 __ csel(as_Register($dst$$reg),
9707 as_Register($src$$reg),
9708 zr,
9709 (Assembler::Condition)$cmp$$cmpcode);
9710 %}
9711
9712 ins_pipe(icond_reg);
9713 %}
9714
9715 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9716 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9717
9718 ins_cost(INSN_COST * 2);
9719 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9720
9721 ins_encode %{
9722 __ csel(as_Register($dst$$reg),
9723 as_Register($src$$reg),
9724 zr,
9725 (Assembler::Condition)$cmp$$cmpcode);
9726 %}
9727
9728 ins_pipe(icond_reg);
9729 %}
9730
9731 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9732 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9733
9734 ins_cost(INSN_COST * 2);
9735 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9736
9737 ins_encode %{
9738 __ cselw(as_Register($dst$$reg),
9739 as_Register($src2$$reg),
9740 as_Register($src1$$reg),
9741 (Assembler::Condition)$cmp$$cmpcode);
9742 %}
9743
9744 ins_pipe(icond_reg_reg);
9745 %}
9746
9747 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9748 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9749
9750 ins_cost(INSN_COST * 2);
9751 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9752
9753 ins_encode %{
9754 __ cselw(as_Register($dst$$reg),
9755 as_Register($src2$$reg),
9756 as_Register($src1$$reg),
9757 (Assembler::Condition)$cmp$$cmpcode);
9758 %}
9759
9760 ins_pipe(icond_reg_reg);
9761 %}
9762
9763 // special cases where one arg is zero
9764
9765 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9766 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9767
9768 ins_cost(INSN_COST * 2);
9769 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9770
9771 ins_encode %{
9772 __ cselw(as_Register($dst$$reg),
9773 zr,
9774 as_Register($src$$reg),
9775 (Assembler::Condition)$cmp$$cmpcode);
9776 %}
9777
9778 ins_pipe(icond_reg);
9779 %}
9780
9781 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9782 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9783
9784 ins_cost(INSN_COST * 2);
9785 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9786
9787 ins_encode %{
9788 __ cselw(as_Register($dst$$reg),
9789 zr,
9790 as_Register($src$$reg),
9791 (Assembler::Condition)$cmp$$cmpcode);
9792 %}
9793
9794 ins_pipe(icond_reg);
9795 %}
9796
9797 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9798 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9799
9800 ins_cost(INSN_COST * 2);
9801 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9802
9803 ins_encode %{
9804 __ cselw(as_Register($dst$$reg),
9805 as_Register($src$$reg),
9806 zr,
9807 (Assembler::Condition)$cmp$$cmpcode);
9808 %}
9809
9810 ins_pipe(icond_reg);
9811 %}
9812
9813 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9814 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9815
9816 ins_cost(INSN_COST * 2);
9817 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9818
9819 ins_encode %{
9820 __ cselw(as_Register($dst$$reg),
9821 as_Register($src$$reg),
9822 zr,
9823 (Assembler::Condition)$cmp$$cmpcode);
9824 %}
9825
9826 ins_pipe(icond_reg);
9827 %}
9828
9829 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9830 %{
9831 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9832
9833 ins_cost(INSN_COST * 3);
9834
9835 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9836 ins_encode %{
9837 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9838 __ fcsels(as_FloatRegister($dst$$reg),
9839 as_FloatRegister($src2$$reg),
9840 as_FloatRegister($src1$$reg),
9841 cond);
9842 %}
9843
9844 ins_pipe(fp_cond_reg_reg_s);
9845 %}
9846
9847 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9848 %{
9849 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9850
9851 ins_cost(INSN_COST * 3);
9852
9853 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9854 ins_encode %{
9855 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9856 __ fcsels(as_FloatRegister($dst$$reg),
9857 as_FloatRegister($src2$$reg),
9858 as_FloatRegister($src1$$reg),
9859 cond);
9860 %}
9861
9862 ins_pipe(fp_cond_reg_reg_s);
9863 %}
9864
9865 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9866 %{
9867 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9868
9869 ins_cost(INSN_COST * 3);
9870
9871 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9872 ins_encode %{
9873 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9874 __ fcseld(as_FloatRegister($dst$$reg),
9875 as_FloatRegister($src2$$reg),
9876 as_FloatRegister($src1$$reg),
9877 cond);
9878 %}
9879
9880 ins_pipe(fp_cond_reg_reg_d);
9881 %}
9882
9883 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9884 %{
9885 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9886
9887 ins_cost(INSN_COST * 3);
9888
9889 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9890 ins_encode %{
9891 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9892 __ fcseld(as_FloatRegister($dst$$reg),
9893 as_FloatRegister($src2$$reg),
9894 as_FloatRegister($src1$$reg),
9895 cond);
9896 %}
9897
9898 ins_pipe(fp_cond_reg_reg_d);
9899 %}
9900
9901 // ============================================================================
9902 // Arithmetic Instructions
9903 //
9904
9905 // Integer Addition
9906
9907 // TODO
9908 // these currently employ operations which do not set CR and hence are
9909 // not flagged as killing CR but we would like to isolate the cases
9910 // where we want to set flags from those where we don't. need to work
9911 // out how to do that.
9912
9913 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9914 match(Set dst (AddI src1 src2));
9915
9916 ins_cost(INSN_COST);
9917 format %{ "addw $dst, $src1, $src2" %}
9918
9919 ins_encode %{
9920 __ addw(as_Register($dst$$reg),
9921 as_Register($src1$$reg),
9922 as_Register($src2$$reg));
9923 %}
9924
9925 ins_pipe(ialu_reg_reg);
9926 %}
9927
9928 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9929 match(Set dst (AddI src1 src2));
9930
9931 ins_cost(INSN_COST);
9932 format %{ "addw $dst, $src1, $src2" %}
9933
9934 // use opcode to indicate that this is an add not a sub
9935 opcode(0x0);
9936
9937 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9938
9939 ins_pipe(ialu_reg_imm);
9940 %}
9941
9942 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9943 match(Set dst (AddI (ConvL2I src1) src2));
9944
9945 ins_cost(INSN_COST);
9946 format %{ "addw $dst, $src1, $src2" %}
9947
9948 // use opcode to indicate that this is an add not a sub
9949 opcode(0x0);
9950
9951 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9952
9953 ins_pipe(ialu_reg_imm);
9954 %}
9955
9956 // Pointer Addition
9957 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9958 match(Set dst (AddP src1 src2));
9959
9960 ins_cost(INSN_COST);
9961 format %{ "add $dst, $src1, $src2\t# ptr" %}
9962
9963 ins_encode %{
9964 __ add(as_Register($dst$$reg),
9965 as_Register($src1$$reg),
9966 as_Register($src2$$reg));
9967 %}
9968
9969 ins_pipe(ialu_reg_reg);
9970 %}
9971
9972 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9973 match(Set dst (AddP src1 (ConvI2L src2)));
9974
9975 ins_cost(1.9 * INSN_COST);
9976 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9977
9978 ins_encode %{
9979 __ add(as_Register($dst$$reg),
9980 as_Register($src1$$reg),
9981 as_Register($src2$$reg), ext::sxtw);
9982 %}
9983
9984 ins_pipe(ialu_reg_reg);
9985 %}
9986
9987 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9988 match(Set dst (AddP src1 (LShiftL src2 scale)));
9989
9990 ins_cost(1.9 * INSN_COST);
9991 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9992
9993 ins_encode %{
9994 __ lea(as_Register($dst$$reg),
9995 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9996 Address::lsl($scale$$constant)));
9997 %}
9998
9999 ins_pipe(ialu_reg_reg_shift);
10000 %}
10001
10002 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10003 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10004
10005 ins_cost(1.9 * INSN_COST);
10006 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10007
10008 ins_encode %{
10009 __ lea(as_Register($dst$$reg),
10010 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10011 Address::sxtw($scale$$constant)));
10012 %}
10013
10014 ins_pipe(ialu_reg_reg_shift);
10015 %}
10016
10017 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10018 match(Set dst (LShiftL (ConvI2L src) scale));
10019
10020 ins_cost(INSN_COST);
10021 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10022
10023 ins_encode %{
10024 __ sbfiz(as_Register($dst$$reg),
10025 as_Register($src$$reg),
10026 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10027 %}
10028
10029 ins_pipe(ialu_reg_shift);
10030 %}
10031
10032 // Pointer Immediate Addition
10033 // n.b. this needs to be more expensive than using an indirect memory
10034 // operand
10035 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10036 match(Set dst (AddP src1 src2));
10037
10038 ins_cost(INSN_COST);
10039 format %{ "add $dst, $src1, $src2\t# ptr" %}
10040
10041 // use opcode to indicate that this is an add not a sub
10042 opcode(0x0);
10043
10044 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10045
10046 ins_pipe(ialu_reg_imm);
10047 %}
10048
10049 // Long Addition
10050 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10051
10052 match(Set dst (AddL src1 src2));
10053
10054 ins_cost(INSN_COST);
10055 format %{ "add $dst, $src1, $src2" %}
10056
10057 ins_encode %{
10058 __ add(as_Register($dst$$reg),
10059 as_Register($src1$$reg),
10060 as_Register($src2$$reg));
10061 %}
10062
10063 ins_pipe(ialu_reg_reg);
10064 %}
10065
10066 // No constant pool entries requiredLong Immediate Addition.
10067 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10068 match(Set dst (AddL src1 src2));
10069
10070 ins_cost(INSN_COST);
10071 format %{ "add $dst, $src1, $src2" %}
10072
10073 // use opcode to indicate that this is an add not a sub
10074 opcode(0x0);
10075
10076 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10077
10078 ins_pipe(ialu_reg_imm);
10079 %}
10080
10081 // Integer Subtraction
10082 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10083 match(Set dst (SubI src1 src2));
10084
10085 ins_cost(INSN_COST);
10086 format %{ "subw $dst, $src1, $src2" %}
10087
10088 ins_encode %{
10089 __ subw(as_Register($dst$$reg),
10090 as_Register($src1$$reg),
10091 as_Register($src2$$reg));
10092 %}
10093
10094 ins_pipe(ialu_reg_reg);
10095 %}
10096
10097 // Immediate Subtraction
10098 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10099 match(Set dst (SubI src1 src2));
10100
10101 ins_cost(INSN_COST);
10102 format %{ "subw $dst, $src1, $src2" %}
10103
10104 // use opcode to indicate that this is a sub not an add
10105 opcode(0x1);
10106
10107 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10108
10109 ins_pipe(ialu_reg_imm);
10110 %}
10111
10112 // Long Subtraction
10113 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10114
10115 match(Set dst (SubL src1 src2));
10116
10117 ins_cost(INSN_COST);
10118 format %{ "sub $dst, $src1, $src2" %}
10119
10120 ins_encode %{
10121 __ sub(as_Register($dst$$reg),
10122 as_Register($src1$$reg),
10123 as_Register($src2$$reg));
10124 %}
10125
10126 ins_pipe(ialu_reg_reg);
10127 %}
10128
10129 // No constant pool entries requiredLong Immediate Subtraction.
10130 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10131 match(Set dst (SubL src1 src2));
10132
10133 ins_cost(INSN_COST);
10134 format %{ "sub$dst, $src1, $src2" %}
10135
10136 // use opcode to indicate that this is a sub not an add
10137 opcode(0x1);
10138
10139 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10140
10141 ins_pipe(ialu_reg_imm);
10142 %}
10143
10144 // Integer Negation (special case for sub)
10145
10146 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10147 match(Set dst (SubI zero src));
10148
10149 ins_cost(INSN_COST);
10150 format %{ "negw $dst, $src\t# int" %}
10151
10152 ins_encode %{
10153 __ negw(as_Register($dst$$reg),
10154 as_Register($src$$reg));
10155 %}
10156
10157 ins_pipe(ialu_reg);
10158 %}
10159
10160 // Long Negation
10161
10162 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10163 match(Set dst (SubL zero src));
10164
10165 ins_cost(INSN_COST);
10166 format %{ "neg $dst, $src\t# long" %}
10167
10168 ins_encode %{
10169 __ neg(as_Register($dst$$reg),
10170 as_Register($src$$reg));
10171 %}
10172
10173 ins_pipe(ialu_reg);
10174 %}
10175
10176 // Integer Multiply
10177
10178 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10179 match(Set dst (MulI src1 src2));
10180
10181 ins_cost(INSN_COST * 3);
10182 format %{ "mulw $dst, $src1, $src2" %}
10183
10184 ins_encode %{
10185 __ mulw(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg));
10188 %}
10189
10190 ins_pipe(imul_reg_reg);
10191 %}
10192
10193 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10194 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10195
10196 ins_cost(INSN_COST * 3);
10197 format %{ "smull $dst, $src1, $src2" %}
10198
10199 ins_encode %{
10200 __ smull(as_Register($dst$$reg),
10201 as_Register($src1$$reg),
10202 as_Register($src2$$reg));
10203 %}
10204
10205 ins_pipe(imul_reg_reg);
10206 %}
10207
10208 // Long Multiply
10209
10210 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10211 match(Set dst (MulL src1 src2));
10212
10213 ins_cost(INSN_COST * 5);
10214 format %{ "mul $dst, $src1, $src2" %}
10215
10216 ins_encode %{
10217 __ mul(as_Register($dst$$reg),
10218 as_Register($src1$$reg),
10219 as_Register($src2$$reg));
10220 %}
10221
10222 ins_pipe(lmul_reg_reg);
10223 %}
10224
10225 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10226 %{
10227 match(Set dst (MulHiL src1 src2));
10228
10229 ins_cost(INSN_COST * 7);
10230 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10231
10232 ins_encode %{
10233 __ smulh(as_Register($dst$$reg),
10234 as_Register($src1$$reg),
10235 as_Register($src2$$reg));
10236 %}
10237
10238 ins_pipe(lmul_reg_reg);
10239 %}
10240
10241 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10242 %{
10243 match(Set dst (UMulHiL src1 src2));
10244
10245 ins_cost(INSN_COST * 7);
10246 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10247
10248 ins_encode %{
10249 __ umulh(as_Register($dst$$reg),
10250 as_Register($src1$$reg),
10251 as_Register($src2$$reg));
10252 %}
10253
10254 ins_pipe(lmul_reg_reg);
10255 %}
10256
10257 // Combined Integer Multiply & Add/Sub
10258
10259 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10260 match(Set dst (AddI src3 (MulI src1 src2)));
10261
10262 ins_cost(INSN_COST * 3);
10263 format %{ "madd $dst, $src1, $src2, $src3" %}
10264
10265 ins_encode %{
10266 __ maddw(as_Register($dst$$reg),
10267 as_Register($src1$$reg),
10268 as_Register($src2$$reg),
10269 as_Register($src3$$reg));
10270 %}
10271
10272 ins_pipe(imac_reg_reg);
10273 %}
10274
10275 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10276 match(Set dst (SubI src3 (MulI src1 src2)));
10277
10278 ins_cost(INSN_COST * 3);
10279 format %{ "msub $dst, $src1, $src2, $src3" %}
10280
10281 ins_encode %{
10282 __ msubw(as_Register($dst$$reg),
10283 as_Register($src1$$reg),
10284 as_Register($src2$$reg),
10285 as_Register($src3$$reg));
10286 %}
10287
10288 ins_pipe(imac_reg_reg);
10289 %}
10290
10291 // Combined Integer Multiply & Neg
10292
10293 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10294 match(Set dst (MulI (SubI zero src1) src2));
10295
10296 ins_cost(INSN_COST * 3);
10297 format %{ "mneg $dst, $src1, $src2" %}
10298
10299 ins_encode %{
10300 __ mnegw(as_Register($dst$$reg),
10301 as_Register($src1$$reg),
10302 as_Register($src2$$reg));
10303 %}
10304
10305 ins_pipe(imac_reg_reg);
10306 %}
10307
10308 // Combined Long Multiply & Add/Sub
10309
10310 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10311 match(Set dst (AddL src3 (MulL src1 src2)));
10312
10313 ins_cost(INSN_COST * 5);
10314 format %{ "madd $dst, $src1, $src2, $src3" %}
10315
10316 ins_encode %{
10317 __ madd(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg),
10320 as_Register($src3$$reg));
10321 %}
10322
10323 ins_pipe(lmac_reg_reg);
10324 %}
10325
10326 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10327 match(Set dst (SubL src3 (MulL src1 src2)));
10328
10329 ins_cost(INSN_COST * 5);
10330 format %{ "msub $dst, $src1, $src2, $src3" %}
10331
10332 ins_encode %{
10333 __ msub(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 as_Register($src2$$reg),
10336 as_Register($src3$$reg));
10337 %}
10338
10339 ins_pipe(lmac_reg_reg);
10340 %}
10341
10342 // Combined Long Multiply & Neg
10343
10344 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10345 match(Set dst (MulL (SubL zero src1) src2));
10346
10347 ins_cost(INSN_COST * 5);
10348 format %{ "mneg $dst, $src1, $src2" %}
10349
10350 ins_encode %{
10351 __ mneg(as_Register($dst$$reg),
10352 as_Register($src1$$reg),
10353 as_Register($src2$$reg));
10354 %}
10355
10356 ins_pipe(lmac_reg_reg);
10357 %}
10358
10359 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10360
10361 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10362 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10363
10364 ins_cost(INSN_COST * 3);
10365 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10366
10367 ins_encode %{
10368 __ smaddl(as_Register($dst$$reg),
10369 as_Register($src1$$reg),
10370 as_Register($src2$$reg),
10371 as_Register($src3$$reg));
10372 %}
10373
10374 ins_pipe(imac_reg_reg);
10375 %}
10376
10377 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10378 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10379
10380 ins_cost(INSN_COST * 3);
10381 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10382
10383 ins_encode %{
10384 __ smsubl(as_Register($dst$$reg),
10385 as_Register($src1$$reg),
10386 as_Register($src2$$reg),
10387 as_Register($src3$$reg));
10388 %}
10389
10390 ins_pipe(imac_reg_reg);
10391 %}
10392
10393 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10394 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10395
10396 ins_cost(INSN_COST * 3);
10397 format %{ "smnegl $dst, $src1, $src2" %}
10398
10399 ins_encode %{
10400 __ smnegl(as_Register($dst$$reg),
10401 as_Register($src1$$reg),
10402 as_Register($src2$$reg));
10403 %}
10404
10405 ins_pipe(imac_reg_reg);
10406 %}
10407
10408 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10409
10410 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10411 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10412
10413 ins_cost(INSN_COST * 5);
10414 format %{ "mulw rscratch1, $src1, $src2\n\t"
10415 "maddw $dst, $src3, $src4, rscratch1" %}
10416
10417 ins_encode %{
10418 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10419 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10420
10421 ins_pipe(imac_reg_reg);
10422 %}
10423
10424 // Integer Divide
10425
10426 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10427 match(Set dst (DivI src1 src2));
10428
10429 ins_cost(INSN_COST * 19);
10430 format %{ "sdivw $dst, $src1, $src2" %}
10431
10432 ins_encode(aarch64_enc_divw(dst, src1, src2));
10433 ins_pipe(idiv_reg_reg);
10434 %}
10435
10436 // Long Divide
10437
10438 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10439 match(Set dst (DivL src1 src2));
10440
10441 ins_cost(INSN_COST * 35);
10442 format %{ "sdiv $dst, $src1, $src2" %}
10443
10444 ins_encode(aarch64_enc_div(dst, src1, src2));
10445 ins_pipe(ldiv_reg_reg);
10446 %}
10447
10448 // Integer Remainder
10449
10450 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10451 match(Set dst (ModI src1 src2));
10452
10453 ins_cost(INSN_COST * 22);
10454 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10455 "msubw $dst, rscratch1, $src2, $src1" %}
10456
10457 ins_encode(aarch64_enc_modw(dst, src1, src2));
10458 ins_pipe(idiv_reg_reg);
10459 %}
10460
10461 // Long Remainder
10462
10463 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10464 match(Set dst (ModL src1 src2));
10465
10466 ins_cost(INSN_COST * 38);
10467 format %{ "sdiv rscratch1, $src1, $src2\n"
10468 "msub $dst, rscratch1, $src2, $src1" %}
10469
10470 ins_encode(aarch64_enc_mod(dst, src1, src2));
10471 ins_pipe(ldiv_reg_reg);
10472 %}
10473
10474 // Unsigned Integer Divide
10475
10476 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10477 match(Set dst (UDivI src1 src2));
10478
10479 ins_cost(INSN_COST * 19);
10480 format %{ "udivw $dst, $src1, $src2" %}
10481
10482 ins_encode %{
10483 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10484 %}
10485
10486 ins_pipe(idiv_reg_reg);
10487 %}
10488
10489 // Unsigned Long Divide
10490
10491 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10492 match(Set dst (UDivL src1 src2));
10493
10494 ins_cost(INSN_COST * 35);
10495 format %{ "udiv $dst, $src1, $src2" %}
10496
10497 ins_encode %{
10498 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10499 %}
10500
10501 ins_pipe(ldiv_reg_reg);
10502 %}
10503
10504 // Unsigned Integer Remainder
10505
10506 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10507 match(Set dst (UModI src1 src2));
10508
10509 ins_cost(INSN_COST * 22);
10510 format %{ "udivw rscratch1, $src1, $src2\n\t"
10511 "msubw $dst, rscratch1, $src2, $src1" %}
10512
10513 ins_encode %{
10514 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10515 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10516 %}
10517
10518 ins_pipe(idiv_reg_reg);
10519 %}
10520
10521 // Unsigned Long Remainder
10522
10523 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10524 match(Set dst (UModL src1 src2));
10525
10526 ins_cost(INSN_COST * 38);
10527 format %{ "udiv rscratch1, $src1, $src2\n"
10528 "msub $dst, rscratch1, $src2, $src1" %}
10529
10530 ins_encode %{
10531 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10532 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10533 %}
10534
10535 ins_pipe(ldiv_reg_reg);
10536 %}
10537
10538 // Integer Shifts
10539
10540 // Shift Left Register
10541 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10542 match(Set dst (LShiftI src1 src2));
10543
10544 ins_cost(INSN_COST * 2);
10545 format %{ "lslvw $dst, $src1, $src2" %}
10546
10547 ins_encode %{
10548 __ lslvw(as_Register($dst$$reg),
10549 as_Register($src1$$reg),
10550 as_Register($src2$$reg));
10551 %}
10552
10553 ins_pipe(ialu_reg_reg_vshift);
10554 %}
10555
10556 // Shift Left Immediate
10557 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10558 match(Set dst (LShiftI src1 src2));
10559
10560 ins_cost(INSN_COST);
10561 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10562
10563 ins_encode %{
10564 __ lslw(as_Register($dst$$reg),
10565 as_Register($src1$$reg),
10566 $src2$$constant & 0x1f);
10567 %}
10568
10569 ins_pipe(ialu_reg_shift);
10570 %}
10571
10572 // Shift Right Logical Register
10573 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10574 match(Set dst (URShiftI src1 src2));
10575
10576 ins_cost(INSN_COST * 2);
10577 format %{ "lsrvw $dst, $src1, $src2" %}
10578
10579 ins_encode %{
10580 __ lsrvw(as_Register($dst$$reg),
10581 as_Register($src1$$reg),
10582 as_Register($src2$$reg));
10583 %}
10584
10585 ins_pipe(ialu_reg_reg_vshift);
10586 %}
10587
10588 // Shift Right Logical Immediate
10589 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10590 match(Set dst (URShiftI src1 src2));
10591
10592 ins_cost(INSN_COST);
10593 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10594
10595 ins_encode %{
10596 __ lsrw(as_Register($dst$$reg),
10597 as_Register($src1$$reg),
10598 $src2$$constant & 0x1f);
10599 %}
10600
10601 ins_pipe(ialu_reg_shift);
10602 %}
10603
10604 // Shift Right Arithmetic Register
10605 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10606 match(Set dst (RShiftI src1 src2));
10607
10608 ins_cost(INSN_COST * 2);
10609 format %{ "asrvw $dst, $src1, $src2" %}
10610
10611 ins_encode %{
10612 __ asrvw(as_Register($dst$$reg),
10613 as_Register($src1$$reg),
10614 as_Register($src2$$reg));
10615 %}
10616
10617 ins_pipe(ialu_reg_reg_vshift);
10618 %}
10619
10620 // Shift Right Arithmetic Immediate
10621 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10622 match(Set dst (RShiftI src1 src2));
10623
10624 ins_cost(INSN_COST);
10625 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10626
10627 ins_encode %{
10628 __ asrw(as_Register($dst$$reg),
10629 as_Register($src1$$reg),
10630 $src2$$constant & 0x1f);
10631 %}
10632
10633 ins_pipe(ialu_reg_shift);
10634 %}
10635
10636 // Combined Int Mask and Right Shift (using UBFM)
10637 // TODO
10638
10639 // Long Shifts
10640
10641 // Shift Left Register
10642 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10643 match(Set dst (LShiftL src1 src2));
10644
10645 ins_cost(INSN_COST * 2);
10646 format %{ "lslv $dst, $src1, $src2" %}
10647
10648 ins_encode %{
10649 __ lslv(as_Register($dst$$reg),
10650 as_Register($src1$$reg),
10651 as_Register($src2$$reg));
10652 %}
10653
10654 ins_pipe(ialu_reg_reg_vshift);
10655 %}
10656
10657 // Shift Left Immediate
10658 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10659 match(Set dst (LShiftL src1 src2));
10660
10661 ins_cost(INSN_COST);
10662 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10663
10664 ins_encode %{
10665 __ lsl(as_Register($dst$$reg),
10666 as_Register($src1$$reg),
10667 $src2$$constant & 0x3f);
10668 %}
10669
10670 ins_pipe(ialu_reg_shift);
10671 %}
10672
10673 // Shift Right Logical Register
10674 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10675 match(Set dst (URShiftL src1 src2));
10676
10677 ins_cost(INSN_COST * 2);
10678 format %{ "lsrv $dst, $src1, $src2" %}
10679
10680 ins_encode %{
10681 __ lsrv(as_Register($dst$$reg),
10682 as_Register($src1$$reg),
10683 as_Register($src2$$reg));
10684 %}
10685
10686 ins_pipe(ialu_reg_reg_vshift);
10687 %}
10688
10689 // Shift Right Logical Immediate
10690 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10691 match(Set dst (URShiftL src1 src2));
10692
10693 ins_cost(INSN_COST);
10694 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10695
10696 ins_encode %{
10697 __ lsr(as_Register($dst$$reg),
10698 as_Register($src1$$reg),
10699 $src2$$constant & 0x3f);
10700 %}
10701
10702 ins_pipe(ialu_reg_shift);
10703 %}
10704
10705 // A special-case pattern for card table stores.
10706 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10707 match(Set dst (URShiftL (CastP2X src1) src2));
10708
10709 ins_cost(INSN_COST);
10710 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10711
10712 ins_encode %{
10713 __ lsr(as_Register($dst$$reg),
10714 as_Register($src1$$reg),
10715 $src2$$constant & 0x3f);
10716 %}
10717
10718 ins_pipe(ialu_reg_shift);
10719 %}
10720
10721 // Shift Right Arithmetic Register
10722 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10723 match(Set dst (RShiftL src1 src2));
10724
10725 ins_cost(INSN_COST * 2);
10726 format %{ "asrv $dst, $src1, $src2" %}
10727
10728 ins_encode %{
10729 __ asrv(as_Register($dst$$reg),
10730 as_Register($src1$$reg),
10731 as_Register($src2$$reg));
10732 %}
10733
10734 ins_pipe(ialu_reg_reg_vshift);
10735 %}
10736
10737 // Shift Right Arithmetic Immediate
10738 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10739 match(Set dst (RShiftL src1 src2));
10740
10741 ins_cost(INSN_COST);
10742 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10743
10744 ins_encode %{
10745 __ asr(as_Register($dst$$reg),
10746 as_Register($src1$$reg),
10747 $src2$$constant & 0x3f);
10748 %}
10749
10750 ins_pipe(ialu_reg_shift);
10751 %}
10752
10753 // BEGIN This section of the file is automatically generated. Do not edit --------------
10754 // This section is generated from aarch64_ad.m4
10755
10756 // This pattern is automatically generated from aarch64_ad.m4
10757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10758 instruct regL_not_reg(iRegLNoSp dst,
10759 iRegL src1, immL_M1 m1,
10760 rFlagsReg cr) %{
10761 match(Set dst (XorL src1 m1));
10762 ins_cost(INSN_COST);
10763 format %{ "eon $dst, $src1, zr" %}
10764
10765 ins_encode %{
10766 __ eon(as_Register($dst$$reg),
10767 as_Register($src1$$reg),
10768 zr,
10769 Assembler::LSL, 0);
10770 %}
10771
10772 ins_pipe(ialu_reg);
10773 %}
10774
10775 // This pattern is automatically generated from aarch64_ad.m4
10776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10777 instruct regI_not_reg(iRegINoSp dst,
10778 iRegIorL2I src1, immI_M1 m1,
10779 rFlagsReg cr) %{
10780 match(Set dst (XorI src1 m1));
10781 ins_cost(INSN_COST);
10782 format %{ "eonw $dst, $src1, zr" %}
10783
10784 ins_encode %{
10785 __ eonw(as_Register($dst$$reg),
10786 as_Register($src1$$reg),
10787 zr,
10788 Assembler::LSL, 0);
10789 %}
10790
10791 ins_pipe(ialu_reg);
10792 %}
10793
10794 // This pattern is automatically generated from aarch64_ad.m4
10795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10796 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10797 immI0 zero, iRegIorL2I src1, immI src2) %{
10798 match(Set dst (SubI zero (URShiftI src1 src2)));
10799
10800 ins_cost(1.9 * INSN_COST);
10801 format %{ "negw $dst, $src1, LSR $src2" %}
10802
10803 ins_encode %{
10804 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10805 Assembler::LSR, $src2$$constant & 0x1f);
10806 %}
10807
10808 ins_pipe(ialu_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10814 immI0 zero, iRegIorL2I src1, immI src2) %{
10815 match(Set dst (SubI zero (RShiftI src1 src2)));
10816
10817 ins_cost(1.9 * INSN_COST);
10818 format %{ "negw $dst, $src1, ASR $src2" %}
10819
10820 ins_encode %{
10821 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10822 Assembler::ASR, $src2$$constant & 0x1f);
10823 %}
10824
10825 ins_pipe(ialu_reg_shift);
10826 %}
10827
10828 // This pattern is automatically generated from aarch64_ad.m4
10829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10830 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10831 immI0 zero, iRegIorL2I src1, immI src2) %{
10832 match(Set dst (SubI zero (LShiftI src1 src2)));
10833
10834 ins_cost(1.9 * INSN_COST);
10835 format %{ "negw $dst, $src1, LSL $src2" %}
10836
10837 ins_encode %{
10838 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10839 Assembler::LSL, $src2$$constant & 0x1f);
10840 %}
10841
10842 ins_pipe(ialu_reg_shift);
10843 %}
10844
10845 // This pattern is automatically generated from aarch64_ad.m4
10846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10847 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10848 immL0 zero, iRegL src1, immI src2) %{
10849 match(Set dst (SubL zero (URShiftL src1 src2)));
10850
10851 ins_cost(1.9 * INSN_COST);
10852 format %{ "neg $dst, $src1, LSR $src2" %}
10853
10854 ins_encode %{
10855 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10856 Assembler::LSR, $src2$$constant & 0x3f);
10857 %}
10858
10859 ins_pipe(ialu_reg_shift);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10865 immL0 zero, iRegL src1, immI src2) %{
10866 match(Set dst (SubL zero (RShiftL src1 src2)));
10867
10868 ins_cost(1.9 * INSN_COST);
10869 format %{ "neg $dst, $src1, ASR $src2" %}
10870
10871 ins_encode %{
10872 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10873 Assembler::ASR, $src2$$constant & 0x3f);
10874 %}
10875
10876 ins_pipe(ialu_reg_shift);
10877 %}
10878
10879 // This pattern is automatically generated from aarch64_ad.m4
10880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10881 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10882 immL0 zero, iRegL src1, immI src2) %{
10883 match(Set dst (SubL zero (LShiftL src1 src2)));
10884
10885 ins_cost(1.9 * INSN_COST);
10886 format %{ "neg $dst, $src1, LSL $src2" %}
10887
10888 ins_encode %{
10889 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10890 Assembler::LSL, $src2$$constant & 0x3f);
10891 %}
10892
10893 ins_pipe(ialu_reg_shift);
10894 %}
10895
10896 // This pattern is automatically generated from aarch64_ad.m4
10897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10898 instruct AndI_reg_not_reg(iRegINoSp dst,
10899 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10900 match(Set dst (AndI src1 (XorI src2 m1)));
10901 ins_cost(INSN_COST);
10902 format %{ "bicw $dst, $src1, $src2" %}
10903
10904 ins_encode %{
10905 __ bicw(as_Register($dst$$reg),
10906 as_Register($src1$$reg),
10907 as_Register($src2$$reg),
10908 Assembler::LSL, 0);
10909 %}
10910
10911 ins_pipe(ialu_reg_reg);
10912 %}
10913
10914 // This pattern is automatically generated from aarch64_ad.m4
10915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10916 instruct AndL_reg_not_reg(iRegLNoSp dst,
10917 iRegL src1, iRegL src2, immL_M1 m1) %{
10918 match(Set dst (AndL src1 (XorL src2 m1)));
10919 ins_cost(INSN_COST);
10920 format %{ "bic $dst, $src1, $src2" %}
10921
10922 ins_encode %{
10923 __ bic(as_Register($dst$$reg),
10924 as_Register($src1$$reg),
10925 as_Register($src2$$reg),
10926 Assembler::LSL, 0);
10927 %}
10928
10929 ins_pipe(ialu_reg_reg);
10930 %}
10931
10932 // This pattern is automatically generated from aarch64_ad.m4
10933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10934 instruct OrI_reg_not_reg(iRegINoSp dst,
10935 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10936 match(Set dst (OrI src1 (XorI src2 m1)));
10937 ins_cost(INSN_COST);
10938 format %{ "ornw $dst, $src1, $src2" %}
10939
10940 ins_encode %{
10941 __ ornw(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::LSL, 0);
10945 %}
10946
10947 ins_pipe(ialu_reg_reg);
10948 %}
10949
10950 // This pattern is automatically generated from aarch64_ad.m4
10951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10952 instruct OrL_reg_not_reg(iRegLNoSp dst,
10953 iRegL src1, iRegL src2, immL_M1 m1) %{
10954 match(Set dst (OrL src1 (XorL src2 m1)));
10955 ins_cost(INSN_COST);
10956 format %{ "orn $dst, $src1, $src2" %}
10957
10958 ins_encode %{
10959 __ orn(as_Register($dst$$reg),
10960 as_Register($src1$$reg),
10961 as_Register($src2$$reg),
10962 Assembler::LSL, 0);
10963 %}
10964
10965 ins_pipe(ialu_reg_reg);
10966 %}
10967
10968 // This pattern is automatically generated from aarch64_ad.m4
10969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10970 instruct XorI_reg_not_reg(iRegINoSp dst,
10971 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10972 match(Set dst (XorI m1 (XorI src2 src1)));
10973 ins_cost(INSN_COST);
10974 format %{ "eonw $dst, $src1, $src2" %}
10975
10976 ins_encode %{
10977 __ eonw(as_Register($dst$$reg),
10978 as_Register($src1$$reg),
10979 as_Register($src2$$reg),
10980 Assembler::LSL, 0);
10981 %}
10982
10983 ins_pipe(ialu_reg_reg);
10984 %}
10985
10986 // This pattern is automatically generated from aarch64_ad.m4
10987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10988 instruct XorL_reg_not_reg(iRegLNoSp dst,
10989 iRegL src1, iRegL src2, immL_M1 m1) %{
10990 match(Set dst (XorL m1 (XorL src2 src1)));
10991 ins_cost(INSN_COST);
10992 format %{ "eon $dst, $src1, $src2" %}
10993
10994 ins_encode %{
10995 __ eon(as_Register($dst$$reg),
10996 as_Register($src1$$reg),
10997 as_Register($src2$$reg),
10998 Assembler::LSL, 0);
10999 %}
11000
11001 ins_pipe(ialu_reg_reg);
11002 %}
11003
11004 // This pattern is automatically generated from aarch64_ad.m4
11005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11006 // val & (-1 ^ (val >>> shift)) ==> bicw
11007 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11008 iRegIorL2I src1, iRegIorL2I src2,
11009 immI src3, immI_M1 src4) %{
11010 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11011 ins_cost(1.9 * INSN_COST);
11012 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11013
11014 ins_encode %{
11015 __ bicw(as_Register($dst$$reg),
11016 as_Register($src1$$reg),
11017 as_Register($src2$$reg),
11018 Assembler::LSR,
11019 $src3$$constant & 0x1f);
11020 %}
11021
11022 ins_pipe(ialu_reg_reg_shift);
11023 %}
11024
11025 // This pattern is automatically generated from aarch64_ad.m4
11026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11027 // val & (-1 ^ (val >>> shift)) ==> bic
11028 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11029 iRegL src1, iRegL src2,
11030 immI src3, immL_M1 src4) %{
11031 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11032 ins_cost(1.9 * INSN_COST);
11033 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11034
11035 ins_encode %{
11036 __ bic(as_Register($dst$$reg),
11037 as_Register($src1$$reg),
11038 as_Register($src2$$reg),
11039 Assembler::LSR,
11040 $src3$$constant & 0x3f);
11041 %}
11042
11043 ins_pipe(ialu_reg_reg_shift);
11044 %}
11045
11046 // This pattern is automatically generated from aarch64_ad.m4
11047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11048 // val & (-1 ^ (val >> shift)) ==> bicw
11049 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11050 iRegIorL2I src1, iRegIorL2I src2,
11051 immI src3, immI_M1 src4) %{
11052 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11053 ins_cost(1.9 * INSN_COST);
11054 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11055
11056 ins_encode %{
11057 __ bicw(as_Register($dst$$reg),
11058 as_Register($src1$$reg),
11059 as_Register($src2$$reg),
11060 Assembler::ASR,
11061 $src3$$constant & 0x1f);
11062 %}
11063
11064 ins_pipe(ialu_reg_reg_shift);
11065 %}
11066
11067 // This pattern is automatically generated from aarch64_ad.m4
11068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11069 // val & (-1 ^ (val >> shift)) ==> bic
11070 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11071 iRegL src1, iRegL src2,
11072 immI src3, immL_M1 src4) %{
11073 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11074 ins_cost(1.9 * INSN_COST);
11075 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11076
11077 ins_encode %{
11078 __ bic(as_Register($dst$$reg),
11079 as_Register($src1$$reg),
11080 as_Register($src2$$reg),
11081 Assembler::ASR,
11082 $src3$$constant & 0x3f);
11083 %}
11084
11085 ins_pipe(ialu_reg_reg_shift);
11086 %}
11087
11088 // This pattern is automatically generated from aarch64_ad.m4
11089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11090 // val & (-1 ^ (val ror shift)) ==> bicw
11091 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11092 iRegIorL2I src1, iRegIorL2I src2,
11093 immI src3, immI_M1 src4) %{
11094 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11095 ins_cost(1.9 * INSN_COST);
11096 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11097
11098 ins_encode %{
11099 __ bicw(as_Register($dst$$reg),
11100 as_Register($src1$$reg),
11101 as_Register($src2$$reg),
11102 Assembler::ROR,
11103 $src3$$constant & 0x1f);
11104 %}
11105
11106 ins_pipe(ialu_reg_reg_shift);
11107 %}
11108
11109 // This pattern is automatically generated from aarch64_ad.m4
11110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11111 // val & (-1 ^ (val ror shift)) ==> bic
11112 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11113 iRegL src1, iRegL src2,
11114 immI src3, immL_M1 src4) %{
11115 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11116 ins_cost(1.9 * INSN_COST);
11117 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11118
11119 ins_encode %{
11120 __ bic(as_Register($dst$$reg),
11121 as_Register($src1$$reg),
11122 as_Register($src2$$reg),
11123 Assembler::ROR,
11124 $src3$$constant & 0x3f);
11125 %}
11126
11127 ins_pipe(ialu_reg_reg_shift);
11128 %}
11129
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 // val & (-1 ^ (val << shift)) ==> bicw
11133 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11134 iRegIorL2I src1, iRegIorL2I src2,
11135 immI src3, immI_M1 src4) %{
11136 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11137 ins_cost(1.9 * INSN_COST);
11138 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11139
11140 ins_encode %{
11141 __ bicw(as_Register($dst$$reg),
11142 as_Register($src1$$reg),
11143 as_Register($src2$$reg),
11144 Assembler::LSL,
11145 $src3$$constant & 0x1f);
11146 %}
11147
11148 ins_pipe(ialu_reg_reg_shift);
11149 %}
11150
11151 // This pattern is automatically generated from aarch64_ad.m4
11152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11153 // val & (-1 ^ (val << shift)) ==> bic
11154 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11155 iRegL src1, iRegL src2,
11156 immI src3, immL_M1 src4) %{
11157 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11158 ins_cost(1.9 * INSN_COST);
11159 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11160
11161 ins_encode %{
11162 __ bic(as_Register($dst$$reg),
11163 as_Register($src1$$reg),
11164 as_Register($src2$$reg),
11165 Assembler::LSL,
11166 $src3$$constant & 0x3f);
11167 %}
11168
11169 ins_pipe(ialu_reg_reg_shift);
11170 %}
11171
11172 // This pattern is automatically generated from aarch64_ad.m4
11173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11174 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11175 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11176 iRegIorL2I src1, iRegIorL2I src2,
11177 immI src3, immI_M1 src4) %{
11178 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11179 ins_cost(1.9 * INSN_COST);
11180 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11181
11182 ins_encode %{
11183 __ eonw(as_Register($dst$$reg),
11184 as_Register($src1$$reg),
11185 as_Register($src2$$reg),
11186 Assembler::LSR,
11187 $src3$$constant & 0x1f);
11188 %}
11189
11190 ins_pipe(ialu_reg_reg_shift);
11191 %}
11192
11193 // This pattern is automatically generated from aarch64_ad.m4
11194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11195 // val ^ (-1 ^ (val >>> shift)) ==> eon
11196 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11197 iRegL src1, iRegL src2,
11198 immI src3, immL_M1 src4) %{
11199 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11200 ins_cost(1.9 * INSN_COST);
11201 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11202
11203 ins_encode %{
11204 __ eon(as_Register($dst$$reg),
11205 as_Register($src1$$reg),
11206 as_Register($src2$$reg),
11207 Assembler::LSR,
11208 $src3$$constant & 0x3f);
11209 %}
11210
11211 ins_pipe(ialu_reg_reg_shift);
11212 %}
11213
11214 // This pattern is automatically generated from aarch64_ad.m4
11215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11216 // val ^ (-1 ^ (val >> shift)) ==> eonw
11217 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11218 iRegIorL2I src1, iRegIorL2I src2,
11219 immI src3, immI_M1 src4) %{
11220 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11221 ins_cost(1.9 * INSN_COST);
11222 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11223
11224 ins_encode %{
11225 __ eonw(as_Register($dst$$reg),
11226 as_Register($src1$$reg),
11227 as_Register($src2$$reg),
11228 Assembler::ASR,
11229 $src3$$constant & 0x1f);
11230 %}
11231
11232 ins_pipe(ialu_reg_reg_shift);
11233 %}
11234
11235 // This pattern is automatically generated from aarch64_ad.m4
11236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11237 // val ^ (-1 ^ (val >> shift)) ==> eon
11238 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11239 iRegL src1, iRegL src2,
11240 immI src3, immL_M1 src4) %{
11241 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11242 ins_cost(1.9 * INSN_COST);
11243 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11244
11245 ins_encode %{
11246 __ eon(as_Register($dst$$reg),
11247 as_Register($src1$$reg),
11248 as_Register($src2$$reg),
11249 Assembler::ASR,
11250 $src3$$constant & 0x3f);
11251 %}
11252
11253 ins_pipe(ialu_reg_reg_shift);
11254 %}
11255
11256 // This pattern is automatically generated from aarch64_ad.m4
11257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11258 // val ^ (-1 ^ (val ror shift)) ==> eonw
11259 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11260 iRegIorL2I src1, iRegIorL2I src2,
11261 immI src3, immI_M1 src4) %{
11262 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11263 ins_cost(1.9 * INSN_COST);
11264 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11265
11266 ins_encode %{
11267 __ eonw(as_Register($dst$$reg),
11268 as_Register($src1$$reg),
11269 as_Register($src2$$reg),
11270 Assembler::ROR,
11271 $src3$$constant & 0x1f);
11272 %}
11273
11274 ins_pipe(ialu_reg_reg_shift);
11275 %}
11276
11277 // This pattern is automatically generated from aarch64_ad.m4
11278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11279 // val ^ (-1 ^ (val ror shift)) ==> eon
11280 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11281 iRegL src1, iRegL src2,
11282 immI src3, immL_M1 src4) %{
11283 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11284 ins_cost(1.9 * INSN_COST);
11285 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11286
11287 ins_encode %{
11288 __ eon(as_Register($dst$$reg),
11289 as_Register($src1$$reg),
11290 as_Register($src2$$reg),
11291 Assembler::ROR,
11292 $src3$$constant & 0x3f);
11293 %}
11294
11295 ins_pipe(ialu_reg_reg_shift);
11296 %}
11297
11298 // This pattern is automatically generated from aarch64_ad.m4
11299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11300 // val ^ (-1 ^ (val << shift)) ==> eonw
11301 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11302 iRegIorL2I src1, iRegIorL2I src2,
11303 immI src3, immI_M1 src4) %{
11304 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11305 ins_cost(1.9 * INSN_COST);
11306 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11307
11308 ins_encode %{
11309 __ eonw(as_Register($dst$$reg),
11310 as_Register($src1$$reg),
11311 as_Register($src2$$reg),
11312 Assembler::LSL,
11313 $src3$$constant & 0x1f);
11314 %}
11315
11316 ins_pipe(ialu_reg_reg_shift);
11317 %}
11318
11319 // This pattern is automatically generated from aarch64_ad.m4
11320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11321 // val ^ (-1 ^ (val << shift)) ==> eon
11322 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11323 iRegL src1, iRegL src2,
11324 immI src3, immL_M1 src4) %{
11325 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11326 ins_cost(1.9 * INSN_COST);
11327 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11328
11329 ins_encode %{
11330 __ eon(as_Register($dst$$reg),
11331 as_Register($src1$$reg),
11332 as_Register($src2$$reg),
11333 Assembler::LSL,
11334 $src3$$constant & 0x3f);
11335 %}
11336
11337 ins_pipe(ialu_reg_reg_shift);
11338 %}
11339
11340 // This pattern is automatically generated from aarch64_ad.m4
11341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11342 // val | (-1 ^ (val >>> shift)) ==> ornw
11343 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11344 iRegIorL2I src1, iRegIorL2I src2,
11345 immI src3, immI_M1 src4) %{
11346 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11347 ins_cost(1.9 * INSN_COST);
11348 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11349
11350 ins_encode %{
11351 __ ornw(as_Register($dst$$reg),
11352 as_Register($src1$$reg),
11353 as_Register($src2$$reg),
11354 Assembler::LSR,
11355 $src3$$constant & 0x1f);
11356 %}
11357
11358 ins_pipe(ialu_reg_reg_shift);
11359 %}
11360
11361 // This pattern is automatically generated from aarch64_ad.m4
11362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11363 // val | (-1 ^ (val >>> shift)) ==> orn
11364 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11365 iRegL src1, iRegL src2,
11366 immI src3, immL_M1 src4) %{
11367 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11368 ins_cost(1.9 * INSN_COST);
11369 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11370
11371 ins_encode %{
11372 __ orn(as_Register($dst$$reg),
11373 as_Register($src1$$reg),
11374 as_Register($src2$$reg),
11375 Assembler::LSR,
11376 $src3$$constant & 0x3f);
11377 %}
11378
11379 ins_pipe(ialu_reg_reg_shift);
11380 %}
11381
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 // val | (-1 ^ (val >> shift)) ==> ornw
11385 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11386 iRegIorL2I src1, iRegIorL2I src2,
11387 immI src3, immI_M1 src4) %{
11388 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11389 ins_cost(1.9 * INSN_COST);
11390 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11391
11392 ins_encode %{
11393 __ ornw(as_Register($dst$$reg),
11394 as_Register($src1$$reg),
11395 as_Register($src2$$reg),
11396 Assembler::ASR,
11397 $src3$$constant & 0x1f);
11398 %}
11399
11400 ins_pipe(ialu_reg_reg_shift);
11401 %}
11402
11403 // This pattern is automatically generated from aarch64_ad.m4
11404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11405 // val | (-1 ^ (val >> shift)) ==> orn
11406 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11407 iRegL src1, iRegL src2,
11408 immI src3, immL_M1 src4) %{
11409 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11410 ins_cost(1.9 * INSN_COST);
11411 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11412
11413 ins_encode %{
11414 __ orn(as_Register($dst$$reg),
11415 as_Register($src1$$reg),
11416 as_Register($src2$$reg),
11417 Assembler::ASR,
11418 $src3$$constant & 0x3f);
11419 %}
11420
11421 ins_pipe(ialu_reg_reg_shift);
11422 %}
11423
11424 // This pattern is automatically generated from aarch64_ad.m4
11425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11426 // val | (-1 ^ (val ror shift)) ==> ornw
11427 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11428 iRegIorL2I src1, iRegIorL2I src2,
11429 immI src3, immI_M1 src4) %{
11430 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11431 ins_cost(1.9 * INSN_COST);
11432 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11433
11434 ins_encode %{
11435 __ ornw(as_Register($dst$$reg),
11436 as_Register($src1$$reg),
11437 as_Register($src2$$reg),
11438 Assembler::ROR,
11439 $src3$$constant & 0x1f);
11440 %}
11441
11442 ins_pipe(ialu_reg_reg_shift);
11443 %}
11444
11445 // This pattern is automatically generated from aarch64_ad.m4
11446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11447 // val | (-1 ^ (val ror shift)) ==> orn
11448 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11449 iRegL src1, iRegL src2,
11450 immI src3, immL_M1 src4) %{
11451 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11452 ins_cost(1.9 * INSN_COST);
11453 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11454
11455 ins_encode %{
11456 __ orn(as_Register($dst$$reg),
11457 as_Register($src1$$reg),
11458 as_Register($src2$$reg),
11459 Assembler::ROR,
11460 $src3$$constant & 0x3f);
11461 %}
11462
11463 ins_pipe(ialu_reg_reg_shift);
11464 %}
11465
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468 // val | (-1 ^ (val << shift)) ==> ornw
11469 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11470 iRegIorL2I src1, iRegIorL2I src2,
11471 immI src3, immI_M1 src4) %{
11472 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11473 ins_cost(1.9 * INSN_COST);
11474 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11475
11476 ins_encode %{
11477 __ ornw(as_Register($dst$$reg),
11478 as_Register($src1$$reg),
11479 as_Register($src2$$reg),
11480 Assembler::LSL,
11481 $src3$$constant & 0x1f);
11482 %}
11483
11484 ins_pipe(ialu_reg_reg_shift);
11485 %}
11486
11487 // This pattern is automatically generated from aarch64_ad.m4
11488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11489 // val | (-1 ^ (val << shift)) ==> orn
11490 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11491 iRegL src1, iRegL src2,
11492 immI src3, immL_M1 src4) %{
11493 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11494 ins_cost(1.9 * INSN_COST);
11495 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11496
11497 ins_encode %{
11498 __ orn(as_Register($dst$$reg),
11499 as_Register($src1$$reg),
11500 as_Register($src2$$reg),
11501 Assembler::LSL,
11502 $src3$$constant & 0x3f);
11503 %}
11504
11505 ins_pipe(ialu_reg_reg_shift);
11506 %}
11507
11508 // This pattern is automatically generated from aarch64_ad.m4
11509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11510 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11511 iRegIorL2I src1, iRegIorL2I src2,
11512 immI src3) %{
11513 match(Set dst (AndI src1 (URShiftI src2 src3)));
11514
11515 ins_cost(1.9 * INSN_COST);
11516 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11517
11518 ins_encode %{
11519 __ andw(as_Register($dst$$reg),
11520 as_Register($src1$$reg),
11521 as_Register($src2$$reg),
11522 Assembler::LSR,
11523 $src3$$constant & 0x1f);
11524 %}
11525
11526 ins_pipe(ialu_reg_reg_shift);
11527 %}
11528
11529 // This pattern is automatically generated from aarch64_ad.m4
11530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11531 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11532 iRegL src1, iRegL src2,
11533 immI src3) %{
11534 match(Set dst (AndL src1 (URShiftL src2 src3)));
11535
11536 ins_cost(1.9 * INSN_COST);
11537 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11538
11539 ins_encode %{
11540 __ andr(as_Register($dst$$reg),
11541 as_Register($src1$$reg),
11542 as_Register($src2$$reg),
11543 Assembler::LSR,
11544 $src3$$constant & 0x3f);
11545 %}
11546
11547 ins_pipe(ialu_reg_reg_shift);
11548 %}
11549
11550 // This pattern is automatically generated from aarch64_ad.m4
11551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11552 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11553 iRegIorL2I src1, iRegIorL2I src2,
11554 immI src3) %{
11555 match(Set dst (AndI src1 (RShiftI src2 src3)));
11556
11557 ins_cost(1.9 * INSN_COST);
11558 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11559
11560 ins_encode %{
11561 __ andw(as_Register($dst$$reg),
11562 as_Register($src1$$reg),
11563 as_Register($src2$$reg),
11564 Assembler::ASR,
11565 $src3$$constant & 0x1f);
11566 %}
11567
11568 ins_pipe(ialu_reg_reg_shift);
11569 %}
11570
11571 // This pattern is automatically generated from aarch64_ad.m4
11572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11573 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11574 iRegL src1, iRegL src2,
11575 immI src3) %{
11576 match(Set dst (AndL src1 (RShiftL src2 src3)));
11577
11578 ins_cost(1.9 * INSN_COST);
11579 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11580
11581 ins_encode %{
11582 __ andr(as_Register($dst$$reg),
11583 as_Register($src1$$reg),
11584 as_Register($src2$$reg),
11585 Assembler::ASR,
11586 $src3$$constant & 0x3f);
11587 %}
11588
11589 ins_pipe(ialu_reg_reg_shift);
11590 %}
11591
11592 // This pattern is automatically generated from aarch64_ad.m4
11593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11594 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11595 iRegIorL2I src1, iRegIorL2I src2,
11596 immI src3) %{
11597 match(Set dst (AndI src1 (LShiftI src2 src3)));
11598
11599 ins_cost(1.9 * INSN_COST);
11600 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11601
11602 ins_encode %{
11603 __ andw(as_Register($dst$$reg),
11604 as_Register($src1$$reg),
11605 as_Register($src2$$reg),
11606 Assembler::LSL,
11607 $src3$$constant & 0x1f);
11608 %}
11609
11610 ins_pipe(ialu_reg_reg_shift);
11611 %}
11612
11613 // This pattern is automatically generated from aarch64_ad.m4
11614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11615 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11616 iRegL src1, iRegL src2,
11617 immI src3) %{
11618 match(Set dst (AndL src1 (LShiftL src2 src3)));
11619
11620 ins_cost(1.9 * INSN_COST);
11621 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11622
11623 ins_encode %{
11624 __ andr(as_Register($dst$$reg),
11625 as_Register($src1$$reg),
11626 as_Register($src2$$reg),
11627 Assembler::LSL,
11628 $src3$$constant & 0x3f);
11629 %}
11630
11631 ins_pipe(ialu_reg_reg_shift);
11632 %}
11633
11634 // This pattern is automatically generated from aarch64_ad.m4
11635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11636 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11637 iRegIorL2I src1, iRegIorL2I src2,
11638 immI src3) %{
11639 match(Set dst (AndI src1 (RotateRight src2 src3)));
11640
11641 ins_cost(1.9 * INSN_COST);
11642 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11643
11644 ins_encode %{
11645 __ andw(as_Register($dst$$reg),
11646 as_Register($src1$$reg),
11647 as_Register($src2$$reg),
11648 Assembler::ROR,
11649 $src3$$constant & 0x1f);
11650 %}
11651
11652 ins_pipe(ialu_reg_reg_shift);
11653 %}
11654
11655 // This pattern is automatically generated from aarch64_ad.m4
11656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11657 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11658 iRegL src1, iRegL src2,
11659 immI src3) %{
11660 match(Set dst (AndL src1 (RotateRight src2 src3)));
11661
11662 ins_cost(1.9 * INSN_COST);
11663 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11664
11665 ins_encode %{
11666 __ andr(as_Register($dst$$reg),
11667 as_Register($src1$$reg),
11668 as_Register($src2$$reg),
11669 Assembler::ROR,
11670 $src3$$constant & 0x3f);
11671 %}
11672
11673 ins_pipe(ialu_reg_reg_shift);
11674 %}
11675
11676 // This pattern is automatically generated from aarch64_ad.m4
11677 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11678 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11679 iRegIorL2I src1, iRegIorL2I src2,
11680 immI src3) %{
11681 match(Set dst (XorI src1 (URShiftI src2 src3)));
11682
11683 ins_cost(1.9 * INSN_COST);
11684 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11685
11686 ins_encode %{
11687 __ eorw(as_Register($dst$$reg),
11688 as_Register($src1$$reg),
11689 as_Register($src2$$reg),
11690 Assembler::LSR,
11691 $src3$$constant & 0x1f);
11692 %}
11693
11694 ins_pipe(ialu_reg_reg_shift);
11695 %}
11696
11697 // This pattern is automatically generated from aarch64_ad.m4
11698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11699 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11700 iRegL src1, iRegL src2,
11701 immI src3) %{
11702 match(Set dst (XorL src1 (URShiftL src2 src3)));
11703
11704 ins_cost(1.9 * INSN_COST);
11705 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11706
11707 ins_encode %{
11708 __ eor(as_Register($dst$$reg),
11709 as_Register($src1$$reg),
11710 as_Register($src2$$reg),
11711 Assembler::LSR,
11712 $src3$$constant & 0x3f);
11713 %}
11714
11715 ins_pipe(ialu_reg_reg_shift);
11716 %}
11717
11718 // This pattern is automatically generated from aarch64_ad.m4
11719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11720 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11721 iRegIorL2I src1, iRegIorL2I src2,
11722 immI src3) %{
11723 match(Set dst (XorI src1 (RShiftI src2 src3)));
11724
11725 ins_cost(1.9 * INSN_COST);
11726 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11727
11728 ins_encode %{
11729 __ eorw(as_Register($dst$$reg),
11730 as_Register($src1$$reg),
11731 as_Register($src2$$reg),
11732 Assembler::ASR,
11733 $src3$$constant & 0x1f);
11734 %}
11735
11736 ins_pipe(ialu_reg_reg_shift);
11737 %}
11738
11739 // This pattern is automatically generated from aarch64_ad.m4
11740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11741 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11742 iRegL src1, iRegL src2,
11743 immI src3) %{
11744 match(Set dst (XorL src1 (RShiftL src2 src3)));
11745
11746 ins_cost(1.9 * INSN_COST);
11747 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11748
11749 ins_encode %{
11750 __ eor(as_Register($dst$$reg),
11751 as_Register($src1$$reg),
11752 as_Register($src2$$reg),
11753 Assembler::ASR,
11754 $src3$$constant & 0x3f);
11755 %}
11756
11757 ins_pipe(ialu_reg_reg_shift);
11758 %}
11759
11760 // This pattern is automatically generated from aarch64_ad.m4
11761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11762 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11763 iRegIorL2I src1, iRegIorL2I src2,
11764 immI src3) %{
11765 match(Set dst (XorI src1 (LShiftI src2 src3)));
11766
11767 ins_cost(1.9 * INSN_COST);
11768 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11769
11770 ins_encode %{
11771 __ eorw(as_Register($dst$$reg),
11772 as_Register($src1$$reg),
11773 as_Register($src2$$reg),
11774 Assembler::LSL,
11775 $src3$$constant & 0x1f);
11776 %}
11777
11778 ins_pipe(ialu_reg_reg_shift);
11779 %}
11780
11781 // This pattern is automatically generated from aarch64_ad.m4
11782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11783 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11784 iRegL src1, iRegL src2,
11785 immI src3) %{
11786 match(Set dst (XorL src1 (LShiftL src2 src3)));
11787
11788 ins_cost(1.9 * INSN_COST);
11789 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11790
11791 ins_encode %{
11792 __ eor(as_Register($dst$$reg),
11793 as_Register($src1$$reg),
11794 as_Register($src2$$reg),
11795 Assembler::LSL,
11796 $src3$$constant & 0x3f);
11797 %}
11798
11799 ins_pipe(ialu_reg_reg_shift);
11800 %}
11801
11802 // This pattern is automatically generated from aarch64_ad.m4
11803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11804 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11805 iRegIorL2I src1, iRegIorL2I src2,
11806 immI src3) %{
11807 match(Set dst (XorI src1 (RotateRight src2 src3)));
11808
11809 ins_cost(1.9 * INSN_COST);
11810 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11811
11812 ins_encode %{
11813 __ eorw(as_Register($dst$$reg),
11814 as_Register($src1$$reg),
11815 as_Register($src2$$reg),
11816 Assembler::ROR,
11817 $src3$$constant & 0x1f);
11818 %}
11819
11820 ins_pipe(ialu_reg_reg_shift);
11821 %}
11822
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11826 iRegL src1, iRegL src2,
11827 immI src3) %{
11828 match(Set dst (XorL src1 (RotateRight src2 src3)));
11829
11830 ins_cost(1.9 * INSN_COST);
11831 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11832
11833 ins_encode %{
11834 __ eor(as_Register($dst$$reg),
11835 as_Register($src1$$reg),
11836 as_Register($src2$$reg),
11837 Assembler::ROR,
11838 $src3$$constant & 0x3f);
11839 %}
11840
11841 ins_pipe(ialu_reg_reg_shift);
11842 %}
11843
11844 // This pattern is automatically generated from aarch64_ad.m4
11845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11846 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11847 iRegIorL2I src1, iRegIorL2I src2,
11848 immI src3) %{
11849 match(Set dst (OrI src1 (URShiftI src2 src3)));
11850
11851 ins_cost(1.9 * INSN_COST);
11852 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11853
11854 ins_encode %{
11855 __ orrw(as_Register($dst$$reg),
11856 as_Register($src1$$reg),
11857 as_Register($src2$$reg),
11858 Assembler::LSR,
11859 $src3$$constant & 0x1f);
11860 %}
11861
11862 ins_pipe(ialu_reg_reg_shift);
11863 %}
11864
11865 // This pattern is automatically generated from aarch64_ad.m4
11866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11867 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11868 iRegL src1, iRegL src2,
11869 immI src3) %{
11870 match(Set dst (OrL src1 (URShiftL src2 src3)));
11871
11872 ins_cost(1.9 * INSN_COST);
11873 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11874
11875 ins_encode %{
11876 __ orr(as_Register($dst$$reg),
11877 as_Register($src1$$reg),
11878 as_Register($src2$$reg),
11879 Assembler::LSR,
11880 $src3$$constant & 0x3f);
11881 %}
11882
11883 ins_pipe(ialu_reg_reg_shift);
11884 %}
11885
11886 // This pattern is automatically generated from aarch64_ad.m4
11887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11888 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11889 iRegIorL2I src1, iRegIorL2I src2,
11890 immI src3) %{
11891 match(Set dst (OrI src1 (RShiftI src2 src3)));
11892
11893 ins_cost(1.9 * INSN_COST);
11894 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11895
11896 ins_encode %{
11897 __ orrw(as_Register($dst$$reg),
11898 as_Register($src1$$reg),
11899 as_Register($src2$$reg),
11900 Assembler::ASR,
11901 $src3$$constant & 0x1f);
11902 %}
11903
11904 ins_pipe(ialu_reg_reg_shift);
11905 %}
11906
11907 // This pattern is automatically generated from aarch64_ad.m4
11908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11909 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11910 iRegL src1, iRegL src2,
11911 immI src3) %{
11912 match(Set dst (OrL src1 (RShiftL src2 src3)));
11913
11914 ins_cost(1.9 * INSN_COST);
11915 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11916
11917 ins_encode %{
11918 __ orr(as_Register($dst$$reg),
11919 as_Register($src1$$reg),
11920 as_Register($src2$$reg),
11921 Assembler::ASR,
11922 $src3$$constant & 0x3f);
11923 %}
11924
11925 ins_pipe(ialu_reg_reg_shift);
11926 %}
11927
11928 // This pattern is automatically generated from aarch64_ad.m4
11929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11930 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11931 iRegIorL2I src1, iRegIorL2I src2,
11932 immI src3) %{
11933 match(Set dst (OrI src1 (LShiftI src2 src3)));
11934
11935 ins_cost(1.9 * INSN_COST);
11936 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11937
11938 ins_encode %{
11939 __ orrw(as_Register($dst$$reg),
11940 as_Register($src1$$reg),
11941 as_Register($src2$$reg),
11942 Assembler::LSL,
11943 $src3$$constant & 0x1f);
11944 %}
11945
11946 ins_pipe(ialu_reg_reg_shift);
11947 %}
11948
11949 // This pattern is automatically generated from aarch64_ad.m4
11950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11951 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11952 iRegL src1, iRegL src2,
11953 immI src3) %{
11954 match(Set dst (OrL src1 (LShiftL src2 src3)));
11955
11956 ins_cost(1.9 * INSN_COST);
11957 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11958
11959 ins_encode %{
11960 __ orr(as_Register($dst$$reg),
11961 as_Register($src1$$reg),
11962 as_Register($src2$$reg),
11963 Assembler::LSL,
11964 $src3$$constant & 0x3f);
11965 %}
11966
11967 ins_pipe(ialu_reg_reg_shift);
11968 %}
11969
11970 // This pattern is automatically generated from aarch64_ad.m4
11971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11972 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11973 iRegIorL2I src1, iRegIorL2I src2,
11974 immI src3) %{
11975 match(Set dst (OrI src1 (RotateRight src2 src3)));
11976
11977 ins_cost(1.9 * INSN_COST);
11978 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11979
11980 ins_encode %{
11981 __ orrw(as_Register($dst$$reg),
11982 as_Register($src1$$reg),
11983 as_Register($src2$$reg),
11984 Assembler::ROR,
11985 $src3$$constant & 0x1f);
11986 %}
11987
11988 ins_pipe(ialu_reg_reg_shift);
11989 %}
11990
11991 // This pattern is automatically generated from aarch64_ad.m4
11992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11993 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11994 iRegL src1, iRegL src2,
11995 immI src3) %{
11996 match(Set dst (OrL src1 (RotateRight src2 src3)));
11997
11998 ins_cost(1.9 * INSN_COST);
11999 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12000
12001 ins_encode %{
12002 __ orr(as_Register($dst$$reg),
12003 as_Register($src1$$reg),
12004 as_Register($src2$$reg),
12005 Assembler::ROR,
12006 $src3$$constant & 0x3f);
12007 %}
12008
12009 ins_pipe(ialu_reg_reg_shift);
12010 %}
12011
12012 // This pattern is automatically generated from aarch64_ad.m4
12013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12014 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12015 iRegIorL2I src1, iRegIorL2I src2,
12016 immI src3) %{
12017 match(Set dst (AddI src1 (URShiftI src2 src3)));
12018
12019 ins_cost(1.9 * INSN_COST);
12020 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12021
12022 ins_encode %{
12023 __ addw(as_Register($dst$$reg),
12024 as_Register($src1$$reg),
12025 as_Register($src2$$reg),
12026 Assembler::LSR,
12027 $src3$$constant & 0x1f);
12028 %}
12029
12030 ins_pipe(ialu_reg_reg_shift);
12031 %}
12032
12033 // This pattern is automatically generated from aarch64_ad.m4
12034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12035 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12036 iRegL src1, iRegL src2,
12037 immI src3) %{
12038 match(Set dst (AddL src1 (URShiftL src2 src3)));
12039
12040 ins_cost(1.9 * INSN_COST);
12041 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12042
12043 ins_encode %{
12044 __ add(as_Register($dst$$reg),
12045 as_Register($src1$$reg),
12046 as_Register($src2$$reg),
12047 Assembler::LSR,
12048 $src3$$constant & 0x3f);
12049 %}
12050
12051 ins_pipe(ialu_reg_reg_shift);
12052 %}
12053
12054 // This pattern is automatically generated from aarch64_ad.m4
12055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12056 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12057 iRegIorL2I src1, iRegIorL2I src2,
12058 immI src3) %{
12059 match(Set dst (AddI src1 (RShiftI src2 src3)));
12060
12061 ins_cost(1.9 * INSN_COST);
12062 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12063
12064 ins_encode %{
12065 __ addw(as_Register($dst$$reg),
12066 as_Register($src1$$reg),
12067 as_Register($src2$$reg),
12068 Assembler::ASR,
12069 $src3$$constant & 0x1f);
12070 %}
12071
12072 ins_pipe(ialu_reg_reg_shift);
12073 %}
12074
12075 // This pattern is automatically generated from aarch64_ad.m4
12076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12077 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12078 iRegL src1, iRegL src2,
12079 immI src3) %{
12080 match(Set dst (AddL src1 (RShiftL src2 src3)));
12081
12082 ins_cost(1.9 * INSN_COST);
12083 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12084
12085 ins_encode %{
12086 __ add(as_Register($dst$$reg),
12087 as_Register($src1$$reg),
12088 as_Register($src2$$reg),
12089 Assembler::ASR,
12090 $src3$$constant & 0x3f);
12091 %}
12092
12093 ins_pipe(ialu_reg_reg_shift);
12094 %}
12095
12096 // This pattern is automatically generated from aarch64_ad.m4
12097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12098 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12099 iRegIorL2I src1, iRegIorL2I src2,
12100 immI src3) %{
12101 match(Set dst (AddI src1 (LShiftI src2 src3)));
12102
12103 ins_cost(1.9 * INSN_COST);
12104 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12105
12106 ins_encode %{
12107 __ addw(as_Register($dst$$reg),
12108 as_Register($src1$$reg),
12109 as_Register($src2$$reg),
12110 Assembler::LSL,
12111 $src3$$constant & 0x1f);
12112 %}
12113
12114 ins_pipe(ialu_reg_reg_shift);
12115 %}
12116
12117 // This pattern is automatically generated from aarch64_ad.m4
12118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12119 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12120 iRegL src1, iRegL src2,
12121 immI src3) %{
12122 match(Set dst (AddL src1 (LShiftL src2 src3)));
12123
12124 ins_cost(1.9 * INSN_COST);
12125 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12126
12127 ins_encode %{
12128 __ add(as_Register($dst$$reg),
12129 as_Register($src1$$reg),
12130 as_Register($src2$$reg),
12131 Assembler::LSL,
12132 $src3$$constant & 0x3f);
12133 %}
12134
12135 ins_pipe(ialu_reg_reg_shift);
12136 %}
12137
12138 // This pattern is automatically generated from aarch64_ad.m4
12139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12140 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12141 iRegIorL2I src1, iRegIorL2I src2,
12142 immI src3) %{
12143 match(Set dst (SubI src1 (URShiftI src2 src3)));
12144
12145 ins_cost(1.9 * INSN_COST);
12146 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12147
12148 ins_encode %{
12149 __ subw(as_Register($dst$$reg),
12150 as_Register($src1$$reg),
12151 as_Register($src2$$reg),
12152 Assembler::LSR,
12153 $src3$$constant & 0x1f);
12154 %}
12155
12156 ins_pipe(ialu_reg_reg_shift);
12157 %}
12158
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12162 iRegL src1, iRegL src2,
12163 immI src3) %{
12164 match(Set dst (SubL src1 (URShiftL src2 src3)));
12165
12166 ins_cost(1.9 * INSN_COST);
12167 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12168
12169 ins_encode %{
12170 __ sub(as_Register($dst$$reg),
12171 as_Register($src1$$reg),
12172 as_Register($src2$$reg),
12173 Assembler::LSR,
12174 $src3$$constant & 0x3f);
12175 %}
12176
12177 ins_pipe(ialu_reg_reg_shift);
12178 %}
12179
12180 // This pattern is automatically generated from aarch64_ad.m4
12181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12182 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12183 iRegIorL2I src1, iRegIorL2I src2,
12184 immI src3) %{
12185 match(Set dst (SubI src1 (RShiftI src2 src3)));
12186
12187 ins_cost(1.9 * INSN_COST);
12188 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12189
12190 ins_encode %{
12191 __ subw(as_Register($dst$$reg),
12192 as_Register($src1$$reg),
12193 as_Register($src2$$reg),
12194 Assembler::ASR,
12195 $src3$$constant & 0x1f);
12196 %}
12197
12198 ins_pipe(ialu_reg_reg_shift);
12199 %}
12200
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12204 iRegL src1, iRegL src2,
12205 immI src3) %{
12206 match(Set dst (SubL src1 (RShiftL src2 src3)));
12207
12208 ins_cost(1.9 * INSN_COST);
12209 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12210
12211 ins_encode %{
12212 __ sub(as_Register($dst$$reg),
12213 as_Register($src1$$reg),
12214 as_Register($src2$$reg),
12215 Assembler::ASR,
12216 $src3$$constant & 0x3f);
12217 %}
12218
12219 ins_pipe(ialu_reg_reg_shift);
12220 %}
12221
12222 // This pattern is automatically generated from aarch64_ad.m4
12223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12224 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12225 iRegIorL2I src1, iRegIorL2I src2,
12226 immI src3) %{
12227 match(Set dst (SubI src1 (LShiftI src2 src3)));
12228
12229 ins_cost(1.9 * INSN_COST);
12230 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12231
12232 ins_encode %{
12233 __ subw(as_Register($dst$$reg),
12234 as_Register($src1$$reg),
12235 as_Register($src2$$reg),
12236 Assembler::LSL,
12237 $src3$$constant & 0x1f);
12238 %}
12239
12240 ins_pipe(ialu_reg_reg_shift);
12241 %}
12242
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12246 iRegL src1, iRegL src2,
12247 immI src3) %{
12248 match(Set dst (SubL src1 (LShiftL src2 src3)));
12249
12250 ins_cost(1.9 * INSN_COST);
12251 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12252
12253 ins_encode %{
12254 __ sub(as_Register($dst$$reg),
12255 as_Register($src1$$reg),
12256 as_Register($src2$$reg),
12257 Assembler::LSL,
12258 $src3$$constant & 0x3f);
12259 %}
12260
12261 ins_pipe(ialu_reg_reg_shift);
12262 %}
12263
12264 // This pattern is automatically generated from aarch64_ad.m4
12265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12266
12267 // Shift Left followed by Shift Right.
12268 // This idiom is used by the compiler for the i2b bytecode etc.
12269 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12270 %{
12271 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12272 ins_cost(INSN_COST * 2);
12273 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12274 ins_encode %{
12275 int lshift = $lshift_count$$constant & 63;
12276 int rshift = $rshift_count$$constant & 63;
12277 int s = 63 - lshift;
12278 int r = (rshift - lshift) & 63;
12279 __ sbfm(as_Register($dst$$reg),
12280 as_Register($src$$reg),
12281 r, s);
12282 %}
12283
12284 ins_pipe(ialu_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289
12290 // Shift Left followed by Shift Right.
12291 // This idiom is used by the compiler for the i2b bytecode etc.
12292 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12293 %{
12294 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12295 ins_cost(INSN_COST * 2);
12296 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12297 ins_encode %{
12298 int lshift = $lshift_count$$constant & 31;
12299 int rshift = $rshift_count$$constant & 31;
12300 int s = 31 - lshift;
12301 int r = (rshift - lshift) & 31;
12302 __ sbfmw(as_Register($dst$$reg),
12303 as_Register($src$$reg),
12304 r, s);
12305 %}
12306
12307 ins_pipe(ialu_reg_shift);
12308 %}
12309
12310 // This pattern is automatically generated from aarch64_ad.m4
12311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12312
12313 // Shift Left followed by Shift Right.
12314 // This idiom is used by the compiler for the i2b bytecode etc.
12315 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12316 %{
12317 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12318 ins_cost(INSN_COST * 2);
12319 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12320 ins_encode %{
12321 int lshift = $lshift_count$$constant & 63;
12322 int rshift = $rshift_count$$constant & 63;
12323 int s = 63 - lshift;
12324 int r = (rshift - lshift) & 63;
12325 __ ubfm(as_Register($dst$$reg),
12326 as_Register($src$$reg),
12327 r, s);
12328 %}
12329
12330 ins_pipe(ialu_reg_shift);
12331 %}
12332
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335
12336 // Shift Left followed by Shift Right.
12337 // This idiom is used by the compiler for the i2b bytecode etc.
12338 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12339 %{
12340 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12341 ins_cost(INSN_COST * 2);
12342 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12343 ins_encode %{
12344 int lshift = $lshift_count$$constant & 31;
12345 int rshift = $rshift_count$$constant & 31;
12346 int s = 31 - lshift;
12347 int r = (rshift - lshift) & 31;
12348 __ ubfmw(as_Register($dst$$reg),
12349 as_Register($src$$reg),
12350 r, s);
12351 %}
12352
12353 ins_pipe(ialu_reg_shift);
12354 %}
12355
12356 // Bitfield extract with shift & mask
12357
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12361 %{
12362 match(Set dst (AndI (URShiftI src rshift) mask));
12363 // Make sure we are not going to exceed what ubfxw can do.
12364 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12365
12366 ins_cost(INSN_COST);
12367 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12368 ins_encode %{
12369 int rshift = $rshift$$constant & 31;
12370 intptr_t mask = $mask$$constant;
12371 int width = exact_log2(mask+1);
12372 __ ubfxw(as_Register($dst$$reg),
12373 as_Register($src$$reg), rshift, width);
12374 %}
12375 ins_pipe(ialu_reg_shift);
12376 %}
12377
12378 // This pattern is automatically generated from aarch64_ad.m4
12379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12380 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12381 %{
12382 match(Set dst (AndL (URShiftL src rshift) mask));
12383 // Make sure we are not going to exceed what ubfx can do.
12384 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12385
12386 ins_cost(INSN_COST);
12387 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12388 ins_encode %{
12389 int rshift = $rshift$$constant & 63;
12390 intptr_t mask = $mask$$constant;
12391 int width = exact_log2_long(mask+1);
12392 __ ubfx(as_Register($dst$$reg),
12393 as_Register($src$$reg), rshift, width);
12394 %}
12395 ins_pipe(ialu_reg_shift);
12396 %}
12397
12398
12399 // This pattern is automatically generated from aarch64_ad.m4
12400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12401
12402 // We can use ubfx when extending an And with a mask when we know mask
12403 // is positive. We know that because immI_bitmask guarantees it.
12404 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12405 %{
12406 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12407 // Make sure we are not going to exceed what ubfxw can do.
12408 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12409
12410 ins_cost(INSN_COST * 2);
12411 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12412 ins_encode %{
12413 int rshift = $rshift$$constant & 31;
12414 intptr_t mask = $mask$$constant;
12415 int width = exact_log2(mask+1);
12416 __ ubfx(as_Register($dst$$reg),
12417 as_Register($src$$reg), rshift, width);
12418 %}
12419 ins_pipe(ialu_reg_shift);
12420 %}
12421
12422
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425
12426 // We can use ubfiz when masking by a positive number and then left shifting the result.
12427 // We know that the mask is positive because immI_bitmask guarantees it.
12428 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12429 %{
12430 match(Set dst (LShiftI (AndI src mask) lshift));
12431 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12432
12433 ins_cost(INSN_COST);
12434 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12435 ins_encode %{
12436 int lshift = $lshift$$constant & 31;
12437 intptr_t mask = $mask$$constant;
12438 int width = exact_log2(mask+1);
12439 __ ubfizw(as_Register($dst$$reg),
12440 as_Register($src$$reg), lshift, width);
12441 %}
12442 ins_pipe(ialu_reg_shift);
12443 %}
12444
12445 // This pattern is automatically generated from aarch64_ad.m4
12446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12447
12448 // We can use ubfiz when masking by a positive number and then left shifting the result.
12449 // We know that the mask is positive because immL_bitmask guarantees it.
12450 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12451 %{
12452 match(Set dst (LShiftL (AndL src mask) lshift));
12453 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12454
12455 ins_cost(INSN_COST);
12456 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12457 ins_encode %{
12458 int lshift = $lshift$$constant & 63;
12459 intptr_t mask = $mask$$constant;
12460 int width = exact_log2_long(mask+1);
12461 __ ubfiz(as_Register($dst$$reg),
12462 as_Register($src$$reg), lshift, width);
12463 %}
12464 ins_pipe(ialu_reg_shift);
12465 %}
12466
12467 // This pattern is automatically generated from aarch64_ad.m4
12468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12469
12470 // We can use ubfiz when masking by a positive number and then left shifting the result.
12471 // We know that the mask is positive because immI_bitmask guarantees it.
12472 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12473 %{
12474 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12475 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12476
12477 ins_cost(INSN_COST);
12478 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12479 ins_encode %{
12480 int lshift = $lshift$$constant & 31;
12481 intptr_t mask = $mask$$constant;
12482 int width = exact_log2(mask+1);
12483 __ ubfizw(as_Register($dst$$reg),
12484 as_Register($src$$reg), lshift, width);
12485 %}
12486 ins_pipe(ialu_reg_shift);
12487 %}
12488
12489 // This pattern is automatically generated from aarch64_ad.m4
12490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12491
12492 // We can use ubfiz when masking by a positive number and then left shifting the result.
12493 // We know that the mask is positive because immL_bitmask guarantees it.
12494 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12495 %{
12496 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12497 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12498
12499 ins_cost(INSN_COST);
12500 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12501 ins_encode %{
12502 int lshift = $lshift$$constant & 63;
12503 intptr_t mask = $mask$$constant;
12504 int width = exact_log2_long(mask+1);
12505 __ ubfiz(as_Register($dst$$reg),
12506 as_Register($src$$reg), lshift, width);
12507 %}
12508 ins_pipe(ialu_reg_shift);
12509 %}
12510
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514
12515 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12516 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12517 %{
12518 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12519 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12520
12521 ins_cost(INSN_COST);
12522 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12523 ins_encode %{
12524 int lshift = $lshift$$constant & 63;
12525 intptr_t mask = $mask$$constant;
12526 int width = exact_log2(mask+1);
12527 __ ubfiz(as_Register($dst$$reg),
12528 as_Register($src$$reg), lshift, width);
12529 %}
12530 ins_pipe(ialu_reg_shift);
12531 %}
12532
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535
12536 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12537 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12538 %{
12539 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12540 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12541
12542 ins_cost(INSN_COST);
12543 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12544 ins_encode %{
12545 int lshift = $lshift$$constant & 31;
12546 intptr_t mask = $mask$$constant;
12547 int width = exact_log2(mask+1);
12548 __ ubfiz(as_Register($dst$$reg),
12549 as_Register($src$$reg), lshift, width);
12550 %}
12551 ins_pipe(ialu_reg_shift);
12552 %}
12553
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556
12557 // Can skip int2long conversions after AND with small bitmask
12558 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12559 %{
12560 match(Set dst (ConvI2L (AndI src msk)));
12561 ins_cost(INSN_COST);
12562 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12563 ins_encode %{
12564 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12565 %}
12566 ins_pipe(ialu_reg_shift);
12567 %}
12568
12569
12570 // Rotations
12571
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12575 %{
12576 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12577 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12578
12579 ins_cost(INSN_COST);
12580 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12581
12582 ins_encode %{
12583 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12584 $rshift$$constant & 63);
12585 %}
12586 ins_pipe(ialu_reg_reg_extr);
12587 %}
12588
12589
12590 // This pattern is automatically generated from aarch64_ad.m4
12591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12592 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12593 %{
12594 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12595 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12596
12597 ins_cost(INSN_COST);
12598 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12599
12600 ins_encode %{
12601 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12602 $rshift$$constant & 31);
12603 %}
12604 ins_pipe(ialu_reg_reg_extr);
12605 %}
12606
12607
12608 // This pattern is automatically generated from aarch64_ad.m4
12609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12610 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12611 %{
12612 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12613 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12614
12615 ins_cost(INSN_COST);
12616 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12617
12618 ins_encode %{
12619 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12620 $rshift$$constant & 63);
12621 %}
12622 ins_pipe(ialu_reg_reg_extr);
12623 %}
12624
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12629 %{
12630 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12631 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12632
12633 ins_cost(INSN_COST);
12634 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12635
12636 ins_encode %{
12637 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12638 $rshift$$constant & 31);
12639 %}
12640 ins_pipe(ialu_reg_reg_extr);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12646 %{
12647 match(Set dst (RotateRight src shift));
12648
12649 ins_cost(INSN_COST);
12650 format %{ "ror $dst, $src, $shift" %}
12651
12652 ins_encode %{
12653 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12654 $shift$$constant & 0x1f);
12655 %}
12656 ins_pipe(ialu_reg_reg_vshift);
12657 %}
12658
12659 // This pattern is automatically generated from aarch64_ad.m4
12660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12661 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12662 %{
12663 match(Set dst (RotateRight src shift));
12664
12665 ins_cost(INSN_COST);
12666 format %{ "ror $dst, $src, $shift" %}
12667
12668 ins_encode %{
12669 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12670 $shift$$constant & 0x3f);
12671 %}
12672 ins_pipe(ialu_reg_reg_vshift);
12673 %}
12674
12675 // This pattern is automatically generated from aarch64_ad.m4
12676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12677 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12678 %{
12679 match(Set dst (RotateRight src shift));
12680
12681 ins_cost(INSN_COST);
12682 format %{ "ror $dst, $src, $shift" %}
12683
12684 ins_encode %{
12685 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12686 %}
12687 ins_pipe(ialu_reg_reg_vshift);
12688 %}
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12693 %{
12694 match(Set dst (RotateRight src shift));
12695
12696 ins_cost(INSN_COST);
12697 format %{ "ror $dst, $src, $shift" %}
12698
12699 ins_encode %{
12700 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12701 %}
12702 ins_pipe(ialu_reg_reg_vshift);
12703 %}
12704
12705 // This pattern is automatically generated from aarch64_ad.m4
12706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12707 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12708 %{
12709 match(Set dst (RotateLeft src shift));
12710
12711 ins_cost(INSN_COST);
12712 format %{ "rol $dst, $src, $shift" %}
12713
12714 ins_encode %{
12715 __ subw(rscratch1, zr, as_Register($shift$$reg));
12716 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12717 %}
12718 ins_pipe(ialu_reg_reg_vshift);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12724 %{
12725 match(Set dst (RotateLeft src shift));
12726
12727 ins_cost(INSN_COST);
12728 format %{ "rol $dst, $src, $shift" %}
12729
12730 ins_encode %{
12731 __ subw(rscratch1, zr, as_Register($shift$$reg));
12732 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12733 %}
12734 ins_pipe(ialu_reg_reg_vshift);
12735 %}
12736
12737
12738 // Add/subtract (extended)
12739
12740 // This pattern is automatically generated from aarch64_ad.m4
12741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12742 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12743 %{
12744 match(Set dst (AddL src1 (ConvI2L src2)));
12745 ins_cost(INSN_COST);
12746 format %{ "add $dst, $src1, $src2, sxtw" %}
12747
12748 ins_encode %{
12749 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12750 as_Register($src2$$reg), ext::sxtw);
12751 %}
12752 ins_pipe(ialu_reg_reg);
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12758 %{
12759 match(Set dst (SubL src1 (ConvI2L src2)));
12760 ins_cost(INSN_COST);
12761 format %{ "sub $dst, $src1, $src2, sxtw" %}
12762
12763 ins_encode %{
12764 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12765 as_Register($src2$$reg), ext::sxtw);
12766 %}
12767 ins_pipe(ialu_reg_reg);
12768 %}
12769
12770 // This pattern is automatically generated from aarch64_ad.m4
12771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12772 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12773 %{
12774 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12775 ins_cost(INSN_COST);
12776 format %{ "add $dst, $src1, $src2, sxth" %}
12777
12778 ins_encode %{
12779 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12780 as_Register($src2$$reg), ext::sxth);
12781 %}
12782 ins_pipe(ialu_reg_reg);
12783 %}
12784
12785 // This pattern is automatically generated from aarch64_ad.m4
12786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12787 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12788 %{
12789 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12790 ins_cost(INSN_COST);
12791 format %{ "add $dst, $src1, $src2, sxtb" %}
12792
12793 ins_encode %{
12794 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12795 as_Register($src2$$reg), ext::sxtb);
12796 %}
12797 ins_pipe(ialu_reg_reg);
12798 %}
12799
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12803 %{
12804 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12805 ins_cost(INSN_COST);
12806 format %{ "add $dst, $src1, $src2, uxtb" %}
12807
12808 ins_encode %{
12809 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12810 as_Register($src2$$reg), ext::uxtb);
12811 %}
12812 ins_pipe(ialu_reg_reg);
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12818 %{
12819 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12820 ins_cost(INSN_COST);
12821 format %{ "add $dst, $src1, $src2, sxth" %}
12822
12823 ins_encode %{
12824 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12825 as_Register($src2$$reg), ext::sxth);
12826 %}
12827 ins_pipe(ialu_reg_reg);
12828 %}
12829
12830 // This pattern is automatically generated from aarch64_ad.m4
12831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12832 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12833 %{
12834 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12835 ins_cost(INSN_COST);
12836 format %{ "add $dst, $src1, $src2, sxtw" %}
12837
12838 ins_encode %{
12839 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12840 as_Register($src2$$reg), ext::sxtw);
12841 %}
12842 ins_pipe(ialu_reg_reg);
12843 %}
12844
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12848 %{
12849 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12850 ins_cost(INSN_COST);
12851 format %{ "add $dst, $src1, $src2, sxtb" %}
12852
12853 ins_encode %{
12854 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12855 as_Register($src2$$reg), ext::sxtb);
12856 %}
12857 ins_pipe(ialu_reg_reg);
12858 %}
12859
12860 // This pattern is automatically generated from aarch64_ad.m4
12861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12862 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12863 %{
12864 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12865 ins_cost(INSN_COST);
12866 format %{ "add $dst, $src1, $src2, uxtb" %}
12867
12868 ins_encode %{
12869 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12870 as_Register($src2$$reg), ext::uxtb);
12871 %}
12872 ins_pipe(ialu_reg_reg);
12873 %}
12874
12875 // This pattern is automatically generated from aarch64_ad.m4
12876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12877 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12878 %{
12879 match(Set dst (AddI src1 (AndI src2 mask)));
12880 ins_cost(INSN_COST);
12881 format %{ "addw $dst, $src1, $src2, uxtb" %}
12882
12883 ins_encode %{
12884 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12885 as_Register($src2$$reg), ext::uxtb);
12886 %}
12887 ins_pipe(ialu_reg_reg);
12888 %}
12889
12890 // This pattern is automatically generated from aarch64_ad.m4
12891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12892 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12893 %{
12894 match(Set dst (AddI src1 (AndI src2 mask)));
12895 ins_cost(INSN_COST);
12896 format %{ "addw $dst, $src1, $src2, uxth" %}
12897
12898 ins_encode %{
12899 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12900 as_Register($src2$$reg), ext::uxth);
12901 %}
12902 ins_pipe(ialu_reg_reg);
12903 %}
12904
12905 // This pattern is automatically generated from aarch64_ad.m4
12906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12908 %{
12909 match(Set dst (AddL src1 (AndL src2 mask)));
12910 ins_cost(INSN_COST);
12911 format %{ "add $dst, $src1, $src2, uxtb" %}
12912
12913 ins_encode %{
12914 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12915 as_Register($src2$$reg), ext::uxtb);
12916 %}
12917 ins_pipe(ialu_reg_reg);
12918 %}
12919
12920 // This pattern is automatically generated from aarch64_ad.m4
12921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12922 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12923 %{
12924 match(Set dst (AddL src1 (AndL src2 mask)));
12925 ins_cost(INSN_COST);
12926 format %{ "add $dst, $src1, $src2, uxth" %}
12927
12928 ins_encode %{
12929 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12930 as_Register($src2$$reg), ext::uxth);
12931 %}
12932 ins_pipe(ialu_reg_reg);
12933 %}
12934
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12938 %{
12939 match(Set dst (AddL src1 (AndL src2 mask)));
12940 ins_cost(INSN_COST);
12941 format %{ "add $dst, $src1, $src2, uxtw" %}
12942
12943 ins_encode %{
12944 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12945 as_Register($src2$$reg), ext::uxtw);
12946 %}
12947 ins_pipe(ialu_reg_reg);
12948 %}
12949
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12953 %{
12954 match(Set dst (SubI src1 (AndI src2 mask)));
12955 ins_cost(INSN_COST);
12956 format %{ "subw $dst, $src1, $src2, uxtb" %}
12957
12958 ins_encode %{
12959 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12960 as_Register($src2$$reg), ext::uxtb);
12961 %}
12962 ins_pipe(ialu_reg_reg);
12963 %}
12964
12965 // This pattern is automatically generated from aarch64_ad.m4
12966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12967 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12968 %{
12969 match(Set dst (SubI src1 (AndI src2 mask)));
12970 ins_cost(INSN_COST);
12971 format %{ "subw $dst, $src1, $src2, uxth" %}
12972
12973 ins_encode %{
12974 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12975 as_Register($src2$$reg), ext::uxth);
12976 %}
12977 ins_pipe(ialu_reg_reg);
12978 %}
12979
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12983 %{
12984 match(Set dst (SubL src1 (AndL src2 mask)));
12985 ins_cost(INSN_COST);
12986 format %{ "sub $dst, $src1, $src2, uxtb" %}
12987
12988 ins_encode %{
12989 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12990 as_Register($src2$$reg), ext::uxtb);
12991 %}
12992 ins_pipe(ialu_reg_reg);
12993 %}
12994
12995 // This pattern is automatically generated from aarch64_ad.m4
12996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12997 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12998 %{
12999 match(Set dst (SubL src1 (AndL src2 mask)));
13000 ins_cost(INSN_COST);
13001 format %{ "sub $dst, $src1, $src2, uxth" %}
13002
13003 ins_encode %{
13004 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13005 as_Register($src2$$reg), ext::uxth);
13006 %}
13007 ins_pipe(ialu_reg_reg);
13008 %}
13009
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13013 %{
13014 match(Set dst (SubL src1 (AndL src2 mask)));
13015 ins_cost(INSN_COST);
13016 format %{ "sub $dst, $src1, $src2, uxtw" %}
13017
13018 ins_encode %{
13019 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13020 as_Register($src2$$reg), ext::uxtw);
13021 %}
13022 ins_pipe(ialu_reg_reg);
13023 %}
13024
13025
13026 // This pattern is automatically generated from aarch64_ad.m4
13027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13028 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13029 %{
13030 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13031 ins_cost(1.9 * INSN_COST);
13032 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13033
13034 ins_encode %{
13035 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13036 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13037 %}
13038 ins_pipe(ialu_reg_reg_shift);
13039 %}
13040
13041 // This pattern is automatically generated from aarch64_ad.m4
13042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13043 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13044 %{
13045 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13046 ins_cost(1.9 * INSN_COST);
13047 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13048
13049 ins_encode %{
13050 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13051 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13052 %}
13053 ins_pipe(ialu_reg_reg_shift);
13054 %}
13055
13056 // This pattern is automatically generated from aarch64_ad.m4
13057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13058 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13059 %{
13060 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13061 ins_cost(1.9 * INSN_COST);
13062 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13063
13064 ins_encode %{
13065 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13066 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13067 %}
13068 ins_pipe(ialu_reg_reg_shift);
13069 %}
13070
13071 // This pattern is automatically generated from aarch64_ad.m4
13072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13073 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13074 %{
13075 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13076 ins_cost(1.9 * INSN_COST);
13077 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13078
13079 ins_encode %{
13080 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13081 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13082 %}
13083 ins_pipe(ialu_reg_reg_shift);
13084 %}
13085
13086 // This pattern is automatically generated from aarch64_ad.m4
13087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13088 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13089 %{
13090 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13091 ins_cost(1.9 * INSN_COST);
13092 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13093
13094 ins_encode %{
13095 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13096 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13097 %}
13098 ins_pipe(ialu_reg_reg_shift);
13099 %}
13100
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13104 %{
13105 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13106 ins_cost(1.9 * INSN_COST);
13107 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13108
13109 ins_encode %{
13110 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13111 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13112 %}
13113 ins_pipe(ialu_reg_reg_shift);
13114 %}
13115
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13119 %{
13120 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13121 ins_cost(1.9 * INSN_COST);
13122 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13123
13124 ins_encode %{
13125 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13126 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13127 %}
13128 ins_pipe(ialu_reg_reg_shift);
13129 %}
13130
13131 // This pattern is automatically generated from aarch64_ad.m4
13132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13133 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13134 %{
13135 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13136 ins_cost(1.9 * INSN_COST);
13137 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13138
13139 ins_encode %{
13140 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13141 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13142 %}
13143 ins_pipe(ialu_reg_reg_shift);
13144 %}
13145
13146 // This pattern is automatically generated from aarch64_ad.m4
13147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13148 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13149 %{
13150 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13151 ins_cost(1.9 * INSN_COST);
13152 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13153
13154 ins_encode %{
13155 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13156 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13157 %}
13158 ins_pipe(ialu_reg_reg_shift);
13159 %}
13160
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13164 %{
13165 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13166 ins_cost(1.9 * INSN_COST);
13167 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13168
13169 ins_encode %{
13170 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13171 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13172 %}
13173 ins_pipe(ialu_reg_reg_shift);
13174 %}
13175
13176 // This pattern is automatically generated from aarch64_ad.m4
13177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13178 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13179 %{
13180 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13181 ins_cost(1.9 * INSN_COST);
13182 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13183
13184 ins_encode %{
13185 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13186 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13187 %}
13188 ins_pipe(ialu_reg_reg_shift);
13189 %}
13190
13191 // This pattern is automatically generated from aarch64_ad.m4
13192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13193 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13194 %{
13195 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13196 ins_cost(1.9 * INSN_COST);
13197 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13198
13199 ins_encode %{
13200 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13201 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13202 %}
13203 ins_pipe(ialu_reg_reg_shift);
13204 %}
13205
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13209 %{
13210 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13211 ins_cost(1.9 * INSN_COST);
13212 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13213
13214 ins_encode %{
13215 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13216 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13217 %}
13218 ins_pipe(ialu_reg_reg_shift);
13219 %}
13220
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13224 %{
13225 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13226 ins_cost(1.9 * INSN_COST);
13227 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13228
13229 ins_encode %{
13230 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13231 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13232 %}
13233 ins_pipe(ialu_reg_reg_shift);
13234 %}
13235
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13239 %{
13240 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13241 ins_cost(1.9 * INSN_COST);
13242 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13243
13244 ins_encode %{
13245 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13246 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13247 %}
13248 ins_pipe(ialu_reg_reg_shift);
13249 %}
13250
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13254 %{
13255 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13256 ins_cost(1.9 * INSN_COST);
13257 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13258
13259 ins_encode %{
13260 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13261 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13262 %}
13263 ins_pipe(ialu_reg_reg_shift);
13264 %}
13265
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13269 %{
13270 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13271 ins_cost(1.9 * INSN_COST);
13272 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13273
13274 ins_encode %{
13275 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13276 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13277 %}
13278 ins_pipe(ialu_reg_reg_shift);
13279 %}
13280
13281 // This pattern is automatically generated from aarch64_ad.m4
13282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13283 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13284 %{
13285 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13286 ins_cost(1.9 * INSN_COST);
13287 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13288
13289 ins_encode %{
13290 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13291 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13292 %}
13293 ins_pipe(ialu_reg_reg_shift);
13294 %}
13295
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13299 %{
13300 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13301 ins_cost(1.9 * INSN_COST);
13302 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13303
13304 ins_encode %{
13305 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13306 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13307 %}
13308 ins_pipe(ialu_reg_reg_shift);
13309 %}
13310
13311 // This pattern is automatically generated from aarch64_ad.m4
13312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13313 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13314 %{
13315 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13316 ins_cost(1.9 * INSN_COST);
13317 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13318
13319 ins_encode %{
13320 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13321 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13322 %}
13323 ins_pipe(ialu_reg_reg_shift);
13324 %}
13325
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13329 %{
13330 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13331 ins_cost(1.9 * INSN_COST);
13332 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13333
13334 ins_encode %{
13335 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13336 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13337 %}
13338 ins_pipe(ialu_reg_reg_shift);
13339 %}
13340
13341 // This pattern is automatically generated from aarch64_ad.m4
13342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13343 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13344 %{
13345 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13346 ins_cost(1.9 * INSN_COST);
13347 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13348
13349 ins_encode %{
13350 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13351 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13352 %}
13353 ins_pipe(ialu_reg_reg_shift);
13354 %}
13355
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13359 %{
13360 effect(DEF dst, USE src1, USE src2, USE cr);
13361 ins_cost(INSN_COST * 2);
13362 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13363
13364 ins_encode %{
13365 __ cselw($dst$$Register,
13366 $src1$$Register,
13367 $src2$$Register,
13368 Assembler::LT);
13369 %}
13370 ins_pipe(icond_reg_reg);
13371 %}
13372
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13376 %{
13377 effect(DEF dst, USE src1, USE src2, USE cr);
13378 ins_cost(INSN_COST * 2);
13379 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13380
13381 ins_encode %{
13382 __ cselw($dst$$Register,
13383 $src1$$Register,
13384 $src2$$Register,
13385 Assembler::GT);
13386 %}
13387 ins_pipe(icond_reg_reg);
13388 %}
13389
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13393 %{
13394 effect(DEF dst, USE src1, USE cr);
13395 ins_cost(INSN_COST * 2);
13396 format %{ "cselw $dst, $src1, zr lt\t" %}
13397
13398 ins_encode %{
13399 __ cselw($dst$$Register,
13400 $src1$$Register,
13401 zr,
13402 Assembler::LT);
13403 %}
13404 ins_pipe(icond_reg);
13405 %}
13406
13407 // This pattern is automatically generated from aarch64_ad.m4
13408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13409 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13410 %{
13411 effect(DEF dst, USE src1, USE cr);
13412 ins_cost(INSN_COST * 2);
13413 format %{ "cselw $dst, $src1, zr gt\t" %}
13414
13415 ins_encode %{
13416 __ cselw($dst$$Register,
13417 $src1$$Register,
13418 zr,
13419 Assembler::GT);
13420 %}
13421 ins_pipe(icond_reg);
13422 %}
13423
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13427 %{
13428 effect(DEF dst, USE src1, USE cr);
13429 ins_cost(INSN_COST * 2);
13430 format %{ "csincw $dst, $src1, zr le\t" %}
13431
13432 ins_encode %{
13433 __ csincw($dst$$Register,
13434 $src1$$Register,
13435 zr,
13436 Assembler::LE);
13437 %}
13438 ins_pipe(icond_reg);
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13444 %{
13445 effect(DEF dst, USE src1, USE cr);
13446 ins_cost(INSN_COST * 2);
13447 format %{ "csincw $dst, $src1, zr gt\t" %}
13448
13449 ins_encode %{
13450 __ csincw($dst$$Register,
13451 $src1$$Register,
13452 zr,
13453 Assembler::GT);
13454 %}
13455 ins_pipe(icond_reg);
13456 %}
13457
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13461 %{
13462 effect(DEF dst, USE src1, USE cr);
13463 ins_cost(INSN_COST * 2);
13464 format %{ "csinvw $dst, $src1, zr lt\t" %}
13465
13466 ins_encode %{
13467 __ csinvw($dst$$Register,
13468 $src1$$Register,
13469 zr,
13470 Assembler::LT);
13471 %}
13472 ins_pipe(icond_reg);
13473 %}
13474
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13478 %{
13479 effect(DEF dst, USE src1, USE cr);
13480 ins_cost(INSN_COST * 2);
13481 format %{ "csinvw $dst, $src1, zr ge\t" %}
13482
13483 ins_encode %{
13484 __ csinvw($dst$$Register,
13485 $src1$$Register,
13486 zr,
13487 Assembler::GE);
13488 %}
13489 ins_pipe(icond_reg);
13490 %}
13491
13492 // This pattern is automatically generated from aarch64_ad.m4
13493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13494 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13495 %{
13496 match(Set dst (MinI src imm));
13497 ins_cost(INSN_COST * 3);
13498 expand %{
13499 rFlagsReg cr;
13500 compI_reg_imm0(cr, src);
13501 cmovI_reg_imm0_lt(dst, src, cr);
13502 %}
13503 %}
13504
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13508 %{
13509 match(Set dst (MinI imm src));
13510 ins_cost(INSN_COST * 3);
13511 expand %{
13512 rFlagsReg cr;
13513 compI_reg_imm0(cr, src);
13514 cmovI_reg_imm0_lt(dst, src, cr);
13515 %}
13516 %}
13517
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13521 %{
13522 match(Set dst (MinI src imm));
13523 ins_cost(INSN_COST * 3);
13524 expand %{
13525 rFlagsReg cr;
13526 compI_reg_imm0(cr, src);
13527 cmovI_reg_imm1_le(dst, src, cr);
13528 %}
13529 %}
13530
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13534 %{
13535 match(Set dst (MinI imm src));
13536 ins_cost(INSN_COST * 3);
13537 expand %{
13538 rFlagsReg cr;
13539 compI_reg_imm0(cr, src);
13540 cmovI_reg_imm1_le(dst, src, cr);
13541 %}
13542 %}
13543
13544 // This pattern is automatically generated from aarch64_ad.m4
13545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13546 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13547 %{
13548 match(Set dst (MinI src imm));
13549 ins_cost(INSN_COST * 3);
13550 expand %{
13551 rFlagsReg cr;
13552 compI_reg_imm0(cr, src);
13553 cmovI_reg_immM1_lt(dst, src, cr);
13554 %}
13555 %}
13556
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13560 %{
13561 match(Set dst (MinI imm src));
13562 ins_cost(INSN_COST * 3);
13563 expand %{
13564 rFlagsReg cr;
13565 compI_reg_imm0(cr, src);
13566 cmovI_reg_immM1_lt(dst, src, cr);
13567 %}
13568 %}
13569
13570 // This pattern is automatically generated from aarch64_ad.m4
13571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13572 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13573 %{
13574 match(Set dst (MaxI src imm));
13575 ins_cost(INSN_COST * 3);
13576 expand %{
13577 rFlagsReg cr;
13578 compI_reg_imm0(cr, src);
13579 cmovI_reg_imm0_gt(dst, src, cr);
13580 %}
13581 %}
13582
13583 // This pattern is automatically generated from aarch64_ad.m4
13584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13585 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13586 %{
13587 match(Set dst (MaxI imm src));
13588 ins_cost(INSN_COST * 3);
13589 expand %{
13590 rFlagsReg cr;
13591 compI_reg_imm0(cr, src);
13592 cmovI_reg_imm0_gt(dst, src, cr);
13593 %}
13594 %}
13595
13596 // This pattern is automatically generated from aarch64_ad.m4
13597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13598 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13599 %{
13600 match(Set dst (MaxI src imm));
13601 ins_cost(INSN_COST * 3);
13602 expand %{
13603 rFlagsReg cr;
13604 compI_reg_imm0(cr, src);
13605 cmovI_reg_imm1_gt(dst, src, cr);
13606 %}
13607 %}
13608
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13612 %{
13613 match(Set dst (MaxI imm src));
13614 ins_cost(INSN_COST * 3);
13615 expand %{
13616 rFlagsReg cr;
13617 compI_reg_imm0(cr, src);
13618 cmovI_reg_imm1_gt(dst, src, cr);
13619 %}
13620 %}
13621
13622 // This pattern is automatically generated from aarch64_ad.m4
13623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13624 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13625 %{
13626 match(Set dst (MaxI src imm));
13627 ins_cost(INSN_COST * 3);
13628 expand %{
13629 rFlagsReg cr;
13630 compI_reg_imm0(cr, src);
13631 cmovI_reg_immM1_ge(dst, src, cr);
13632 %}
13633 %}
13634
13635 // This pattern is automatically generated from aarch64_ad.m4
13636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13637 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13638 %{
13639 match(Set dst (MaxI imm src));
13640 ins_cost(INSN_COST * 3);
13641 expand %{
13642 rFlagsReg cr;
13643 compI_reg_imm0(cr, src);
13644 cmovI_reg_immM1_ge(dst, src, cr);
13645 %}
13646 %}
13647
13648 // This pattern is automatically generated from aarch64_ad.m4
13649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13650 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13651 %{
13652 match(Set dst (ReverseI src));
13653 ins_cost(INSN_COST);
13654 format %{ "rbitw $dst, $src" %}
13655 ins_encode %{
13656 __ rbitw($dst$$Register, $src$$Register);
13657 %}
13658 ins_pipe(ialu_reg);
13659 %}
13660
13661 // This pattern is automatically generated from aarch64_ad.m4
13662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13663 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13664 %{
13665 match(Set dst (ReverseL src));
13666 ins_cost(INSN_COST);
13667 format %{ "rbit $dst, $src" %}
13668 ins_encode %{
13669 __ rbit($dst$$Register, $src$$Register);
13670 %}
13671 ins_pipe(ialu_reg);
13672 %}
13673
13674
13675 // END This section of the file is automatically generated. Do not edit --------------
13676
13677
13678 // ============================================================================
13679 // Floating Point Arithmetic Instructions
13680
13681 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13682 match(Set dst (AddHF src1 src2));
13683 format %{ "faddh $dst, $src1, $src2" %}
13684 ins_encode %{
13685 __ faddh($dst$$FloatRegister,
13686 $src1$$FloatRegister,
13687 $src2$$FloatRegister);
13688 %}
13689 ins_pipe(fp_dop_reg_reg_s);
13690 %}
13691
13692 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13693 match(Set dst (AddF src1 src2));
13694
13695 ins_cost(INSN_COST * 5);
13696 format %{ "fadds $dst, $src1, $src2" %}
13697
13698 ins_encode %{
13699 __ fadds(as_FloatRegister($dst$$reg),
13700 as_FloatRegister($src1$$reg),
13701 as_FloatRegister($src2$$reg));
13702 %}
13703
13704 ins_pipe(fp_dop_reg_reg_s);
13705 %}
13706
13707 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13708 match(Set dst (AddD src1 src2));
13709
13710 ins_cost(INSN_COST * 5);
13711 format %{ "faddd $dst, $src1, $src2" %}
13712
13713 ins_encode %{
13714 __ faddd(as_FloatRegister($dst$$reg),
13715 as_FloatRegister($src1$$reg),
13716 as_FloatRegister($src2$$reg));
13717 %}
13718
13719 ins_pipe(fp_dop_reg_reg_d);
13720 %}
13721
13722 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13723 match(Set dst (SubHF src1 src2));
13724 format %{ "fsubh $dst, $src1, $src2" %}
13725 ins_encode %{
13726 __ fsubh($dst$$FloatRegister,
13727 $src1$$FloatRegister,
13728 $src2$$FloatRegister);
13729 %}
13730 ins_pipe(fp_dop_reg_reg_s);
13731 %}
13732
13733 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13734 match(Set dst (SubF src1 src2));
13735
13736 ins_cost(INSN_COST * 5);
13737 format %{ "fsubs $dst, $src1, $src2" %}
13738
13739 ins_encode %{
13740 __ fsubs(as_FloatRegister($dst$$reg),
13741 as_FloatRegister($src1$$reg),
13742 as_FloatRegister($src2$$reg));
13743 %}
13744
13745 ins_pipe(fp_dop_reg_reg_s);
13746 %}
13747
13748 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13749 match(Set dst (SubD src1 src2));
13750
13751 ins_cost(INSN_COST * 5);
13752 format %{ "fsubd $dst, $src1, $src2" %}
13753
13754 ins_encode %{
13755 __ fsubd(as_FloatRegister($dst$$reg),
13756 as_FloatRegister($src1$$reg),
13757 as_FloatRegister($src2$$reg));
13758 %}
13759
13760 ins_pipe(fp_dop_reg_reg_d);
13761 %}
13762
13763 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13764 match(Set dst (MulHF src1 src2));
13765 format %{ "fmulh $dst, $src1, $src2" %}
13766 ins_encode %{
13767 __ fmulh($dst$$FloatRegister,
13768 $src1$$FloatRegister,
13769 $src2$$FloatRegister);
13770 %}
13771 ins_pipe(fp_dop_reg_reg_s);
13772 %}
13773
13774 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13775 match(Set dst (MulF src1 src2));
13776
13777 ins_cost(INSN_COST * 6);
13778 format %{ "fmuls $dst, $src1, $src2" %}
13779
13780 ins_encode %{
13781 __ fmuls(as_FloatRegister($dst$$reg),
13782 as_FloatRegister($src1$$reg),
13783 as_FloatRegister($src2$$reg));
13784 %}
13785
13786 ins_pipe(fp_dop_reg_reg_s);
13787 %}
13788
13789 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13790 match(Set dst (MulD src1 src2));
13791
13792 ins_cost(INSN_COST * 6);
13793 format %{ "fmuld $dst, $src1, $src2" %}
13794
13795 ins_encode %{
13796 __ fmuld(as_FloatRegister($dst$$reg),
13797 as_FloatRegister($src1$$reg),
13798 as_FloatRegister($src2$$reg));
13799 %}
13800
13801 ins_pipe(fp_dop_reg_reg_d);
13802 %}
13803
13804 // src1 * src2 + src3 (half-precision float)
13805 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13806 match(Set dst (FmaHF src3 (Binary src1 src2)));
13807 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13808 ins_encode %{
13809 assert(UseFMA, "Needs FMA instructions support.");
13810 __ fmaddh($dst$$FloatRegister,
13811 $src1$$FloatRegister,
13812 $src2$$FloatRegister,
13813 $src3$$FloatRegister);
13814 %}
13815 ins_pipe(pipe_class_default);
13816 %}
13817
13818 // src1 * src2 + src3
13819 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13820 match(Set dst (FmaF src3 (Binary src1 src2)));
13821
13822 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13823
13824 ins_encode %{
13825 assert(UseFMA, "Needs FMA instructions support.");
13826 __ fmadds(as_FloatRegister($dst$$reg),
13827 as_FloatRegister($src1$$reg),
13828 as_FloatRegister($src2$$reg),
13829 as_FloatRegister($src3$$reg));
13830 %}
13831
13832 ins_pipe(pipe_class_default);
13833 %}
13834
13835 // src1 * src2 + src3
13836 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13837 match(Set dst (FmaD src3 (Binary src1 src2)));
13838
13839 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13840
13841 ins_encode %{
13842 assert(UseFMA, "Needs FMA instructions support.");
13843 __ fmaddd(as_FloatRegister($dst$$reg),
13844 as_FloatRegister($src1$$reg),
13845 as_FloatRegister($src2$$reg),
13846 as_FloatRegister($src3$$reg));
13847 %}
13848
13849 ins_pipe(pipe_class_default);
13850 %}
13851
13852 // src1 * (-src2) + src3
13853 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13854 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13855 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13856
13857 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13858
13859 ins_encode %{
13860 assert(UseFMA, "Needs FMA instructions support.");
13861 __ fmsubs(as_FloatRegister($dst$$reg),
13862 as_FloatRegister($src1$$reg),
13863 as_FloatRegister($src2$$reg),
13864 as_FloatRegister($src3$$reg));
13865 %}
13866
13867 ins_pipe(pipe_class_default);
13868 %}
13869
13870 // src1 * (-src2) + src3
13871 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13872 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13873 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13874
13875 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13876
13877 ins_encode %{
13878 assert(UseFMA, "Needs FMA instructions support.");
13879 __ fmsubd(as_FloatRegister($dst$$reg),
13880 as_FloatRegister($src1$$reg),
13881 as_FloatRegister($src2$$reg),
13882 as_FloatRegister($src3$$reg));
13883 %}
13884
13885 ins_pipe(pipe_class_default);
13886 %}
13887
13888 // src1 * (-src2) - src3
13889 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13890 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13891 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13892
13893 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13894
13895 ins_encode %{
13896 assert(UseFMA, "Needs FMA instructions support.");
13897 __ fnmadds(as_FloatRegister($dst$$reg),
13898 as_FloatRegister($src1$$reg),
13899 as_FloatRegister($src2$$reg),
13900 as_FloatRegister($src3$$reg));
13901 %}
13902
13903 ins_pipe(pipe_class_default);
13904 %}
13905
13906 // src1 * (-src2) - src3
13907 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13908 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13909 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13910
13911 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13912
13913 ins_encode %{
13914 assert(UseFMA, "Needs FMA instructions support.");
13915 __ fnmaddd(as_FloatRegister($dst$$reg),
13916 as_FloatRegister($src1$$reg),
13917 as_FloatRegister($src2$$reg),
13918 as_FloatRegister($src3$$reg));
13919 %}
13920
13921 ins_pipe(pipe_class_default);
13922 %}
13923
13924 // src1 * src2 - src3
13925 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13926 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13927
13928 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13929
13930 ins_encode %{
13931 assert(UseFMA, "Needs FMA instructions support.");
13932 __ fnmsubs(as_FloatRegister($dst$$reg),
13933 as_FloatRegister($src1$$reg),
13934 as_FloatRegister($src2$$reg),
13935 as_FloatRegister($src3$$reg));
13936 %}
13937
13938 ins_pipe(pipe_class_default);
13939 %}
13940
13941 // src1 * src2 - src3
13942 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13943 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13944
13945 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13946
13947 ins_encode %{
13948 assert(UseFMA, "Needs FMA instructions support.");
13949 // n.b. insn name should be fnmsubd
13950 __ fnmsub(as_FloatRegister($dst$$reg),
13951 as_FloatRegister($src1$$reg),
13952 as_FloatRegister($src2$$reg),
13953 as_FloatRegister($src3$$reg));
13954 %}
13955
13956 ins_pipe(pipe_class_default);
13957 %}
13958
13959 // Math.max(HH)H (half-precision float)
13960 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13961 match(Set dst (MaxHF src1 src2));
13962 format %{ "fmaxh $dst, $src1, $src2" %}
13963 ins_encode %{
13964 __ fmaxh($dst$$FloatRegister,
13965 $src1$$FloatRegister,
13966 $src2$$FloatRegister);
13967 %}
13968 ins_pipe(fp_dop_reg_reg_s);
13969 %}
13970
13971 // Math.min(HH)H (half-precision float)
13972 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13973 match(Set dst (MinHF src1 src2));
13974 format %{ "fminh $dst, $src1, $src2" %}
13975 ins_encode %{
13976 __ fminh($dst$$FloatRegister,
13977 $src1$$FloatRegister,
13978 $src2$$FloatRegister);
13979 %}
13980 ins_pipe(fp_dop_reg_reg_s);
13981 %}
13982
13983 // Math.max(FF)F
13984 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13985 match(Set dst (MaxF src1 src2));
13986
13987 format %{ "fmaxs $dst, $src1, $src2" %}
13988 ins_encode %{
13989 __ fmaxs(as_FloatRegister($dst$$reg),
13990 as_FloatRegister($src1$$reg),
13991 as_FloatRegister($src2$$reg));
13992 %}
13993
13994 ins_pipe(fp_dop_reg_reg_s);
13995 %}
13996
13997 // Math.min(FF)F
13998 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13999 match(Set dst (MinF src1 src2));
14000
14001 format %{ "fmins $dst, $src1, $src2" %}
14002 ins_encode %{
14003 __ fmins(as_FloatRegister($dst$$reg),
14004 as_FloatRegister($src1$$reg),
14005 as_FloatRegister($src2$$reg));
14006 %}
14007
14008 ins_pipe(fp_dop_reg_reg_s);
14009 %}
14010
14011 // Math.max(DD)D
14012 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14013 match(Set dst (MaxD src1 src2));
14014
14015 format %{ "fmaxd $dst, $src1, $src2" %}
14016 ins_encode %{
14017 __ fmaxd(as_FloatRegister($dst$$reg),
14018 as_FloatRegister($src1$$reg),
14019 as_FloatRegister($src2$$reg));
14020 %}
14021
14022 ins_pipe(fp_dop_reg_reg_d);
14023 %}
14024
14025 // Math.min(DD)D
14026 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14027 match(Set dst (MinD src1 src2));
14028
14029 format %{ "fmind $dst, $src1, $src2" %}
14030 ins_encode %{
14031 __ fmind(as_FloatRegister($dst$$reg),
14032 as_FloatRegister($src1$$reg),
14033 as_FloatRegister($src2$$reg));
14034 %}
14035
14036 ins_pipe(fp_dop_reg_reg_d);
14037 %}
14038
14039 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14040 match(Set dst (DivHF src1 src2));
14041 format %{ "fdivh $dst, $src1, $src2" %}
14042 ins_encode %{
14043 __ fdivh($dst$$FloatRegister,
14044 $src1$$FloatRegister,
14045 $src2$$FloatRegister);
14046 %}
14047 ins_pipe(fp_div_s);
14048 %}
14049
14050 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14051 match(Set dst (DivF src1 src2));
14052
14053 ins_cost(INSN_COST * 18);
14054 format %{ "fdivs $dst, $src1, $src2" %}
14055
14056 ins_encode %{
14057 __ fdivs(as_FloatRegister($dst$$reg),
14058 as_FloatRegister($src1$$reg),
14059 as_FloatRegister($src2$$reg));
14060 %}
14061
14062 ins_pipe(fp_div_s);
14063 %}
14064
14065 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14066 match(Set dst (DivD src1 src2));
14067
14068 ins_cost(INSN_COST * 32);
14069 format %{ "fdivd $dst, $src1, $src2" %}
14070
14071 ins_encode %{
14072 __ fdivd(as_FloatRegister($dst$$reg),
14073 as_FloatRegister($src1$$reg),
14074 as_FloatRegister($src2$$reg));
14075 %}
14076
14077 ins_pipe(fp_div_d);
14078 %}
14079
14080 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14081 match(Set dst (NegF src));
14082
14083 ins_cost(INSN_COST * 3);
14084 format %{ "fneg $dst, $src" %}
14085
14086 ins_encode %{
14087 __ fnegs(as_FloatRegister($dst$$reg),
14088 as_FloatRegister($src$$reg));
14089 %}
14090
14091 ins_pipe(fp_uop_s);
14092 %}
14093
14094 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14095 match(Set dst (NegD src));
14096
14097 ins_cost(INSN_COST * 3);
14098 format %{ "fnegd $dst, $src" %}
14099
14100 ins_encode %{
14101 __ fnegd(as_FloatRegister($dst$$reg),
14102 as_FloatRegister($src$$reg));
14103 %}
14104
14105 ins_pipe(fp_uop_d);
14106 %}
14107
14108 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14109 %{
14110 match(Set dst (AbsI src));
14111
14112 effect(KILL cr);
14113 ins_cost(INSN_COST * 2);
14114 format %{ "cmpw $src, zr\n\t"
14115 "cnegw $dst, $src, Assembler::LT\t# int abs"
14116 %}
14117
14118 ins_encode %{
14119 __ cmpw(as_Register($src$$reg), zr);
14120 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14121 %}
14122 ins_pipe(pipe_class_default);
14123 %}
14124
14125 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14126 %{
14127 match(Set dst (AbsL src));
14128
14129 effect(KILL cr);
14130 ins_cost(INSN_COST * 2);
14131 format %{ "cmp $src, zr\n\t"
14132 "cneg $dst, $src, Assembler::LT\t# long abs"
14133 %}
14134
14135 ins_encode %{
14136 __ cmp(as_Register($src$$reg), zr);
14137 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14138 %}
14139 ins_pipe(pipe_class_default);
14140 %}
14141
14142 instruct absF_reg(vRegF dst, vRegF src) %{
14143 match(Set dst (AbsF src));
14144
14145 ins_cost(INSN_COST * 3);
14146 format %{ "fabss $dst, $src" %}
14147 ins_encode %{
14148 __ fabss(as_FloatRegister($dst$$reg),
14149 as_FloatRegister($src$$reg));
14150 %}
14151
14152 ins_pipe(fp_uop_s);
14153 %}
14154
14155 instruct absD_reg(vRegD dst, vRegD src) %{
14156 match(Set dst (AbsD src));
14157
14158 ins_cost(INSN_COST * 3);
14159 format %{ "fabsd $dst, $src" %}
14160 ins_encode %{
14161 __ fabsd(as_FloatRegister($dst$$reg),
14162 as_FloatRegister($src$$reg));
14163 %}
14164
14165 ins_pipe(fp_uop_d);
14166 %}
14167
14168 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14169 match(Set dst (AbsF (SubF src1 src2)));
14170
14171 ins_cost(INSN_COST * 3);
14172 format %{ "fabds $dst, $src1, $src2" %}
14173 ins_encode %{
14174 __ fabds(as_FloatRegister($dst$$reg),
14175 as_FloatRegister($src1$$reg),
14176 as_FloatRegister($src2$$reg));
14177 %}
14178
14179 ins_pipe(fp_uop_s);
14180 %}
14181
14182 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14183 match(Set dst (AbsD (SubD src1 src2)));
14184
14185 ins_cost(INSN_COST * 3);
14186 format %{ "fabdd $dst, $src1, $src2" %}
14187 ins_encode %{
14188 __ fabdd(as_FloatRegister($dst$$reg),
14189 as_FloatRegister($src1$$reg),
14190 as_FloatRegister($src2$$reg));
14191 %}
14192
14193 ins_pipe(fp_uop_d);
14194 %}
14195
14196 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14197 match(Set dst (SqrtD src));
14198
14199 ins_cost(INSN_COST * 50);
14200 format %{ "fsqrtd $dst, $src" %}
14201 ins_encode %{
14202 __ fsqrtd(as_FloatRegister($dst$$reg),
14203 as_FloatRegister($src$$reg));
14204 %}
14205
14206 ins_pipe(fp_div_s);
14207 %}
14208
14209 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14210 match(Set dst (SqrtF src));
14211
14212 ins_cost(INSN_COST * 50);
14213 format %{ "fsqrts $dst, $src" %}
14214 ins_encode %{
14215 __ fsqrts(as_FloatRegister($dst$$reg),
14216 as_FloatRegister($src$$reg));
14217 %}
14218
14219 ins_pipe(fp_div_d);
14220 %}
14221
14222 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14223 match(Set dst (SqrtHF src));
14224 format %{ "fsqrth $dst, $src" %}
14225 ins_encode %{
14226 __ fsqrth($dst$$FloatRegister,
14227 $src$$FloatRegister);
14228 %}
14229 ins_pipe(fp_div_s);
14230 %}
14231
14232 // Math.rint, floor, ceil
14233 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14234 match(Set dst (RoundDoubleMode src rmode));
14235 format %{ "frint $dst, $src, $rmode" %}
14236 ins_encode %{
14237 switch ($rmode$$constant) {
14238 case RoundDoubleModeNode::rmode_rint:
14239 __ frintnd(as_FloatRegister($dst$$reg),
14240 as_FloatRegister($src$$reg));
14241 break;
14242 case RoundDoubleModeNode::rmode_floor:
14243 __ frintmd(as_FloatRegister($dst$$reg),
14244 as_FloatRegister($src$$reg));
14245 break;
14246 case RoundDoubleModeNode::rmode_ceil:
14247 __ frintpd(as_FloatRegister($dst$$reg),
14248 as_FloatRegister($src$$reg));
14249 break;
14250 }
14251 %}
14252 ins_pipe(fp_uop_d);
14253 %}
14254
14255 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14256 match(Set dst (CopySignD src1 (Binary src2 zero)));
14257 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14258 format %{ "CopySignD $dst $src1 $src2" %}
14259 ins_encode %{
14260 FloatRegister dst = as_FloatRegister($dst$$reg),
14261 src1 = as_FloatRegister($src1$$reg),
14262 src2 = as_FloatRegister($src2$$reg),
14263 zero = as_FloatRegister($zero$$reg);
14264 __ fnegd(dst, zero);
14265 __ bsl(dst, __ T8B, src2, src1);
14266 %}
14267 ins_pipe(fp_uop_d);
14268 %}
14269
14270 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14271 match(Set dst (CopySignF src1 src2));
14272 effect(TEMP_DEF dst, USE src1, USE src2);
14273 format %{ "CopySignF $dst $src1 $src2" %}
14274 ins_encode %{
14275 FloatRegister dst = as_FloatRegister($dst$$reg),
14276 src1 = as_FloatRegister($src1$$reg),
14277 src2 = as_FloatRegister($src2$$reg);
14278 __ movi(dst, __ T2S, 0x80, 24);
14279 __ bsl(dst, __ T8B, src2, src1);
14280 %}
14281 ins_pipe(fp_uop_d);
14282 %}
14283
14284 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14285 match(Set dst (SignumD src (Binary zero one)));
14286 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14287 format %{ "signumD $dst, $src" %}
14288 ins_encode %{
14289 FloatRegister src = as_FloatRegister($src$$reg),
14290 dst = as_FloatRegister($dst$$reg),
14291 zero = as_FloatRegister($zero$$reg),
14292 one = as_FloatRegister($one$$reg);
14293 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14294 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14295 // Bit selection instruction gets bit from "one" for each enabled bit in
14296 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14297 // NaN the whole "src" will be copied because "dst" is zero. For all other
14298 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14299 // from "src", and all other bits are copied from 1.0.
14300 __ bsl(dst, __ T8B, one, src);
14301 %}
14302 ins_pipe(fp_uop_d);
14303 %}
14304
14305 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14306 match(Set dst (SignumF src (Binary zero one)));
14307 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14308 format %{ "signumF $dst, $src" %}
14309 ins_encode %{
14310 FloatRegister src = as_FloatRegister($src$$reg),
14311 dst = as_FloatRegister($dst$$reg),
14312 zero = as_FloatRegister($zero$$reg),
14313 one = as_FloatRegister($one$$reg);
14314 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14315 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14316 // Bit selection instruction gets bit from "one" for each enabled bit in
14317 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14318 // NaN the whole "src" will be copied because "dst" is zero. For all other
14319 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14320 // from "src", and all other bits are copied from 1.0.
14321 __ bsl(dst, __ T8B, one, src);
14322 %}
14323 ins_pipe(fp_uop_d);
14324 %}
14325
14326 instruct onspinwait() %{
14327 match(OnSpinWait);
14328 ins_cost(INSN_COST);
14329
14330 format %{ "onspinwait" %}
14331
14332 ins_encode %{
14333 __ spin_wait();
14334 %}
14335 ins_pipe(pipe_class_empty);
14336 %}
14337
14338 // ============================================================================
14339 // Logical Instructions
14340
14341 // Integer Logical Instructions
14342
14343 // And Instructions
14344
14345
14346 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14347 match(Set dst (AndI src1 src2));
14348
14349 format %{ "andw $dst, $src1, $src2\t# int" %}
14350
14351 ins_cost(INSN_COST);
14352 ins_encode %{
14353 __ andw(as_Register($dst$$reg),
14354 as_Register($src1$$reg),
14355 as_Register($src2$$reg));
14356 %}
14357
14358 ins_pipe(ialu_reg_reg);
14359 %}
14360
14361 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14362 match(Set dst (AndI src1 src2));
14363
14364 format %{ "andsw $dst, $src1, $src2\t# int" %}
14365
14366 ins_cost(INSN_COST);
14367 ins_encode %{
14368 __ andw(as_Register($dst$$reg),
14369 as_Register($src1$$reg),
14370 (uint64_t)($src2$$constant));
14371 %}
14372
14373 ins_pipe(ialu_reg_imm);
14374 %}
14375
14376 // Or Instructions
14377
14378 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14379 match(Set dst (OrI src1 src2));
14380
14381 format %{ "orrw $dst, $src1, $src2\t# int" %}
14382
14383 ins_cost(INSN_COST);
14384 ins_encode %{
14385 __ orrw(as_Register($dst$$reg),
14386 as_Register($src1$$reg),
14387 as_Register($src2$$reg));
14388 %}
14389
14390 ins_pipe(ialu_reg_reg);
14391 %}
14392
14393 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14394 match(Set dst (OrI src1 src2));
14395
14396 format %{ "orrw $dst, $src1, $src2\t# int" %}
14397
14398 ins_cost(INSN_COST);
14399 ins_encode %{
14400 __ orrw(as_Register($dst$$reg),
14401 as_Register($src1$$reg),
14402 (uint64_t)($src2$$constant));
14403 %}
14404
14405 ins_pipe(ialu_reg_imm);
14406 %}
14407
14408 // Xor Instructions
14409
14410 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14411 match(Set dst (XorI src1 src2));
14412
14413 format %{ "eorw $dst, $src1, $src2\t# int" %}
14414
14415 ins_cost(INSN_COST);
14416 ins_encode %{
14417 __ eorw(as_Register($dst$$reg),
14418 as_Register($src1$$reg),
14419 as_Register($src2$$reg));
14420 %}
14421
14422 ins_pipe(ialu_reg_reg);
14423 %}
14424
14425 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14426 match(Set dst (XorI src1 src2));
14427
14428 format %{ "eorw $dst, $src1, $src2\t# int" %}
14429
14430 ins_cost(INSN_COST);
14431 ins_encode %{
14432 __ eorw(as_Register($dst$$reg),
14433 as_Register($src1$$reg),
14434 (uint64_t)($src2$$constant));
14435 %}
14436
14437 ins_pipe(ialu_reg_imm);
14438 %}
14439
14440 // Long Logical Instructions
14441 // TODO
14442
14443 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14444 match(Set dst (AndL src1 src2));
14445
14446 format %{ "and $dst, $src1, $src2\t# int" %}
14447
14448 ins_cost(INSN_COST);
14449 ins_encode %{
14450 __ andr(as_Register($dst$$reg),
14451 as_Register($src1$$reg),
14452 as_Register($src2$$reg));
14453 %}
14454
14455 ins_pipe(ialu_reg_reg);
14456 %}
14457
14458 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14459 match(Set dst (AndL src1 src2));
14460
14461 format %{ "and $dst, $src1, $src2\t# int" %}
14462
14463 ins_cost(INSN_COST);
14464 ins_encode %{
14465 __ andr(as_Register($dst$$reg),
14466 as_Register($src1$$reg),
14467 (uint64_t)($src2$$constant));
14468 %}
14469
14470 ins_pipe(ialu_reg_imm);
14471 %}
14472
14473 // Or Instructions
14474
14475 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14476 match(Set dst (OrL src1 src2));
14477
14478 format %{ "orr $dst, $src1, $src2\t# int" %}
14479
14480 ins_cost(INSN_COST);
14481 ins_encode %{
14482 __ orr(as_Register($dst$$reg),
14483 as_Register($src1$$reg),
14484 as_Register($src2$$reg));
14485 %}
14486
14487 ins_pipe(ialu_reg_reg);
14488 %}
14489
14490 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14491 match(Set dst (OrL src1 src2));
14492
14493 format %{ "orr $dst, $src1, $src2\t# int" %}
14494
14495 ins_cost(INSN_COST);
14496 ins_encode %{
14497 __ orr(as_Register($dst$$reg),
14498 as_Register($src1$$reg),
14499 (uint64_t)($src2$$constant));
14500 %}
14501
14502 ins_pipe(ialu_reg_imm);
14503 %}
14504
14505 // Xor Instructions
14506
14507 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14508 match(Set dst (XorL src1 src2));
14509
14510 format %{ "eor $dst, $src1, $src2\t# int" %}
14511
14512 ins_cost(INSN_COST);
14513 ins_encode %{
14514 __ eor(as_Register($dst$$reg),
14515 as_Register($src1$$reg),
14516 as_Register($src2$$reg));
14517 %}
14518
14519 ins_pipe(ialu_reg_reg);
14520 %}
14521
14522 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14523 match(Set dst (XorL src1 src2));
14524
14525 ins_cost(INSN_COST);
14526 format %{ "eor $dst, $src1, $src2\t# int" %}
14527
14528 ins_encode %{
14529 __ eor(as_Register($dst$$reg),
14530 as_Register($src1$$reg),
14531 (uint64_t)($src2$$constant));
14532 %}
14533
14534 ins_pipe(ialu_reg_imm);
14535 %}
14536
14537 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14538 %{
14539 match(Set dst (ConvI2L src));
14540
14541 ins_cost(INSN_COST);
14542 format %{ "sxtw $dst, $src\t# i2l" %}
14543 ins_encode %{
14544 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14545 %}
14546 ins_pipe(ialu_reg_shift);
14547 %}
14548
14549 // this pattern occurs in bigmath arithmetic
14550 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14551 %{
14552 match(Set dst (AndL (ConvI2L src) mask));
14553
14554 ins_cost(INSN_COST);
14555 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14556 ins_encode %{
14557 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14558 %}
14559
14560 ins_pipe(ialu_reg_shift);
14561 %}
14562
14563 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14564 match(Set dst (ConvL2I src));
14565
14566 ins_cost(INSN_COST);
14567 format %{ "movw $dst, $src \t// l2i" %}
14568
14569 ins_encode %{
14570 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14571 %}
14572
14573 ins_pipe(ialu_reg);
14574 %}
14575
14576 instruct convD2F_reg(vRegF dst, vRegD src) %{
14577 match(Set dst (ConvD2F src));
14578
14579 ins_cost(INSN_COST * 5);
14580 format %{ "fcvtd $dst, $src \t// d2f" %}
14581
14582 ins_encode %{
14583 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14584 %}
14585
14586 ins_pipe(fp_d2f);
14587 %}
14588
14589 instruct convF2D_reg(vRegD dst, vRegF src) %{
14590 match(Set dst (ConvF2D src));
14591
14592 ins_cost(INSN_COST * 5);
14593 format %{ "fcvts $dst, $src \t// f2d" %}
14594
14595 ins_encode %{
14596 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14597 %}
14598
14599 ins_pipe(fp_f2d);
14600 %}
14601
14602 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14603 match(Set dst (ConvF2I src));
14604
14605 ins_cost(INSN_COST * 5);
14606 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14607
14608 ins_encode %{
14609 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14610 %}
14611
14612 ins_pipe(fp_f2i);
14613 %}
14614
14615 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14616 match(Set dst (ConvF2L src));
14617
14618 ins_cost(INSN_COST * 5);
14619 format %{ "fcvtzs $dst, $src \t// f2l" %}
14620
14621 ins_encode %{
14622 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14623 %}
14624
14625 ins_pipe(fp_f2l);
14626 %}
14627
14628 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14629 match(Set dst (ConvF2HF src));
14630 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14631 "smov $dst, $tmp\t# move result from $tmp to $dst"
14632 %}
14633 effect(TEMP tmp);
14634 ins_encode %{
14635 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14636 %}
14637 ins_pipe(pipe_slow);
14638 %}
14639
14640 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14641 match(Set dst (ConvHF2F src));
14642 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14643 "fcvt $dst, $tmp\t# convert half to single precision"
14644 %}
14645 effect(TEMP tmp);
14646 ins_encode %{
14647 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14648 %}
14649 ins_pipe(pipe_slow);
14650 %}
14651
14652 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14653 match(Set dst (ConvI2F src));
14654
14655 ins_cost(INSN_COST * 5);
14656 format %{ "scvtfws $dst, $src \t// i2f" %}
14657
14658 ins_encode %{
14659 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14660 %}
14661
14662 ins_pipe(fp_i2f);
14663 %}
14664
14665 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14666 match(Set dst (ConvL2F src));
14667
14668 ins_cost(INSN_COST * 5);
14669 format %{ "scvtfs $dst, $src \t// l2f" %}
14670
14671 ins_encode %{
14672 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14673 %}
14674
14675 ins_pipe(fp_l2f);
14676 %}
14677
14678 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14679 match(Set dst (ConvD2I src));
14680
14681 ins_cost(INSN_COST * 5);
14682 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14683
14684 ins_encode %{
14685 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14686 %}
14687
14688 ins_pipe(fp_d2i);
14689 %}
14690
14691 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14692 match(Set dst (ConvD2L src));
14693
14694 ins_cost(INSN_COST * 5);
14695 format %{ "fcvtzd $dst, $src \t// d2l" %}
14696
14697 ins_encode %{
14698 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14699 %}
14700
14701 ins_pipe(fp_d2l);
14702 %}
14703
14704 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14705 match(Set dst (ConvI2D src));
14706
14707 ins_cost(INSN_COST * 5);
14708 format %{ "scvtfwd $dst, $src \t// i2d" %}
14709
14710 ins_encode %{
14711 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14712 %}
14713
14714 ins_pipe(fp_i2d);
14715 %}
14716
14717 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14718 match(Set dst (ConvL2D src));
14719
14720 ins_cost(INSN_COST * 5);
14721 format %{ "scvtfd $dst, $src \t// l2d" %}
14722
14723 ins_encode %{
14724 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14725 %}
14726
14727 ins_pipe(fp_l2d);
14728 %}
14729
14730 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14731 %{
14732 match(Set dst (RoundD src));
14733 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14734 format %{ "java_round_double $dst,$src"%}
14735 ins_encode %{
14736 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14737 as_FloatRegister($ftmp$$reg));
14738 %}
14739 ins_pipe(pipe_slow);
14740 %}
14741
14742 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14743 %{
14744 match(Set dst (RoundF src));
14745 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14746 format %{ "java_round_float $dst,$src"%}
14747 ins_encode %{
14748 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14749 as_FloatRegister($ftmp$$reg));
14750 %}
14751 ins_pipe(pipe_slow);
14752 %}
14753
14754 // stack <-> reg and reg <-> reg shuffles with no conversion
14755
14756 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14757
14758 match(Set dst (MoveF2I src));
14759
14760 effect(DEF dst, USE src);
14761
14762 ins_cost(4 * INSN_COST);
14763
14764 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14765
14766 ins_encode %{
14767 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14768 %}
14769
14770 ins_pipe(iload_reg_reg);
14771
14772 %}
14773
14774 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14775
14776 match(Set dst (MoveI2F src));
14777
14778 effect(DEF dst, USE src);
14779
14780 ins_cost(4 * INSN_COST);
14781
14782 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14783
14784 ins_encode %{
14785 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14786 %}
14787
14788 ins_pipe(pipe_class_memory);
14789
14790 %}
14791
14792 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14793
14794 match(Set dst (MoveD2L src));
14795
14796 effect(DEF dst, USE src);
14797
14798 ins_cost(4 * INSN_COST);
14799
14800 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14801
14802 ins_encode %{
14803 __ ldr($dst$$Register, Address(sp, $src$$disp));
14804 %}
14805
14806 ins_pipe(iload_reg_reg);
14807
14808 %}
14809
14810 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14811
14812 match(Set dst (MoveL2D src));
14813
14814 effect(DEF dst, USE src);
14815
14816 ins_cost(4 * INSN_COST);
14817
14818 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14819
14820 ins_encode %{
14821 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14822 %}
14823
14824 ins_pipe(pipe_class_memory);
14825
14826 %}
14827
14828 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14829
14830 match(Set dst (MoveF2I src));
14831
14832 effect(DEF dst, USE src);
14833
14834 ins_cost(INSN_COST);
14835
14836 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14837
14838 ins_encode %{
14839 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14840 %}
14841
14842 ins_pipe(pipe_class_memory);
14843
14844 %}
14845
14846 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14847
14848 match(Set dst (MoveI2F src));
14849
14850 effect(DEF dst, USE src);
14851
14852 ins_cost(INSN_COST);
14853
14854 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14855
14856 ins_encode %{
14857 __ strw($src$$Register, Address(sp, $dst$$disp));
14858 %}
14859
14860 ins_pipe(istore_reg_reg);
14861
14862 %}
14863
14864 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14865
14866 match(Set dst (MoveD2L src));
14867
14868 effect(DEF dst, USE src);
14869
14870 ins_cost(INSN_COST);
14871
14872 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14873
14874 ins_encode %{
14875 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14876 %}
14877
14878 ins_pipe(pipe_class_memory);
14879
14880 %}
14881
14882 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14883
14884 match(Set dst (MoveL2D src));
14885
14886 effect(DEF dst, USE src);
14887
14888 ins_cost(INSN_COST);
14889
14890 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14891
14892 ins_encode %{
14893 __ str($src$$Register, Address(sp, $dst$$disp));
14894 %}
14895
14896 ins_pipe(istore_reg_reg);
14897
14898 %}
14899
14900 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14901
14902 match(Set dst (MoveF2I src));
14903
14904 effect(DEF dst, USE src);
14905
14906 ins_cost(INSN_COST);
14907
14908 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14909
14910 ins_encode %{
14911 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14912 %}
14913
14914 ins_pipe(fp_f2i);
14915
14916 %}
14917
14918 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14919
14920 match(Set dst (MoveI2F src));
14921
14922 effect(DEF dst, USE src);
14923
14924 ins_cost(INSN_COST);
14925
14926 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14927
14928 ins_encode %{
14929 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14930 %}
14931
14932 ins_pipe(fp_i2f);
14933
14934 %}
14935
14936 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14937
14938 match(Set dst (MoveD2L src));
14939
14940 effect(DEF dst, USE src);
14941
14942 ins_cost(INSN_COST);
14943
14944 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14945
14946 ins_encode %{
14947 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14948 %}
14949
14950 ins_pipe(fp_d2l);
14951
14952 %}
14953
14954 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14955
14956 match(Set dst (MoveL2D src));
14957
14958 effect(DEF dst, USE src);
14959
14960 ins_cost(INSN_COST);
14961
14962 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14963
14964 ins_encode %{
14965 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14966 %}
14967
14968 ins_pipe(fp_l2d);
14969
14970 %}
14971
14972 // ============================================================================
14973 // clearing of an array
14974
14975 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14976 %{
14977 match(Set dummy (ClearArray cnt base));
14978 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14979
14980 ins_cost(4 * INSN_COST);
14981 format %{ "ClearArray $cnt, $base" %}
14982
14983 ins_encode %{
14984 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14985 if (tpc == nullptr) {
14986 ciEnv::current()->record_failure("CodeCache is full");
14987 return;
14988 }
14989 %}
14990
14991 ins_pipe(pipe_class_memory);
14992 %}
14993
14994 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14995 %{
14996 predicate((uint64_t)n->in(2)->get_long()
14997 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14998 match(Set dummy (ClearArray cnt base));
14999 effect(TEMP temp, USE_KILL base, KILL cr);
15000
15001 ins_cost(4 * INSN_COST);
15002 format %{ "ClearArray $cnt, $base" %}
15003
15004 ins_encode %{
15005 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15006 if (tpc == nullptr) {
15007 ciEnv::current()->record_failure("CodeCache is full");
15008 return;
15009 }
15010 %}
15011
15012 ins_pipe(pipe_class_memory);
15013 %}
15014
15015 // ============================================================================
15016 // Overflow Math Instructions
15017
15018 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15019 %{
15020 match(Set cr (OverflowAddI op1 op2));
15021
15022 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15023 ins_cost(INSN_COST);
15024 ins_encode %{
15025 __ cmnw($op1$$Register, $op2$$Register);
15026 %}
15027
15028 ins_pipe(icmp_reg_reg);
15029 %}
15030
15031 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15032 %{
15033 match(Set cr (OverflowAddI op1 op2));
15034
15035 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15036 ins_cost(INSN_COST);
15037 ins_encode %{
15038 __ cmnw($op1$$Register, $op2$$constant);
15039 %}
15040
15041 ins_pipe(icmp_reg_imm);
15042 %}
15043
15044 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15045 %{
15046 match(Set cr (OverflowAddL op1 op2));
15047
15048 format %{ "cmn $op1, $op2\t# overflow check long" %}
15049 ins_cost(INSN_COST);
15050 ins_encode %{
15051 __ cmn($op1$$Register, $op2$$Register);
15052 %}
15053
15054 ins_pipe(icmp_reg_reg);
15055 %}
15056
15057 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15058 %{
15059 match(Set cr (OverflowAddL op1 op2));
15060
15061 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15062 ins_cost(INSN_COST);
15063 ins_encode %{
15064 __ adds(zr, $op1$$Register, $op2$$constant);
15065 %}
15066
15067 ins_pipe(icmp_reg_imm);
15068 %}
15069
15070 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15071 %{
15072 match(Set cr (OverflowSubI op1 op2));
15073
15074 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15075 ins_cost(INSN_COST);
15076 ins_encode %{
15077 __ cmpw($op1$$Register, $op2$$Register);
15078 %}
15079
15080 ins_pipe(icmp_reg_reg);
15081 %}
15082
15083 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15084 %{
15085 match(Set cr (OverflowSubI op1 op2));
15086
15087 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15088 ins_cost(INSN_COST);
15089 ins_encode %{
15090 __ cmpw($op1$$Register, $op2$$constant);
15091 %}
15092
15093 ins_pipe(icmp_reg_imm);
15094 %}
15095
15096 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15097 %{
15098 match(Set cr (OverflowSubL op1 op2));
15099
15100 format %{ "cmp $op1, $op2\t# overflow check long" %}
15101 ins_cost(INSN_COST);
15102 ins_encode %{
15103 __ cmp($op1$$Register, $op2$$Register);
15104 %}
15105
15106 ins_pipe(icmp_reg_reg);
15107 %}
15108
15109 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15110 %{
15111 match(Set cr (OverflowSubL op1 op2));
15112
15113 format %{ "cmp $op1, $op2\t# overflow check long" %}
15114 ins_cost(INSN_COST);
15115 ins_encode %{
15116 __ subs(zr, $op1$$Register, $op2$$constant);
15117 %}
15118
15119 ins_pipe(icmp_reg_imm);
15120 %}
15121
15122 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15123 %{
15124 match(Set cr (OverflowSubI zero op1));
15125
15126 format %{ "cmpw zr, $op1\t# overflow check int" %}
15127 ins_cost(INSN_COST);
15128 ins_encode %{
15129 __ cmpw(zr, $op1$$Register);
15130 %}
15131
15132 ins_pipe(icmp_reg_imm);
15133 %}
15134
15135 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15136 %{
15137 match(Set cr (OverflowSubL zero op1));
15138
15139 format %{ "cmp zr, $op1\t# overflow check long" %}
15140 ins_cost(INSN_COST);
15141 ins_encode %{
15142 __ cmp(zr, $op1$$Register);
15143 %}
15144
15145 ins_pipe(icmp_reg_imm);
15146 %}
15147
15148 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15149 %{
15150 match(Set cr (OverflowMulI op1 op2));
15151
15152 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15153 "cmp rscratch1, rscratch1, sxtw\n\t"
15154 "movw rscratch1, #0x80000000\n\t"
15155 "cselw rscratch1, rscratch1, zr, NE\n\t"
15156 "cmpw rscratch1, #1" %}
15157 ins_cost(5 * INSN_COST);
15158 ins_encode %{
15159 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15160 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15161 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15162 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15163 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15164 %}
15165
15166 ins_pipe(pipe_slow);
15167 %}
15168
15169 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15170 %{
15171 match(If cmp (OverflowMulI op1 op2));
15172 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15173 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15174 effect(USE labl, KILL cr);
15175
15176 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15177 "cmp rscratch1, rscratch1, sxtw\n\t"
15178 "b$cmp $labl" %}
15179 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15180 ins_encode %{
15181 Label* L = $labl$$label;
15182 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15183 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15184 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15185 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15186 %}
15187
15188 ins_pipe(pipe_serial);
15189 %}
15190
15191 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15192 %{
15193 match(Set cr (OverflowMulL op1 op2));
15194
15195 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15196 "smulh rscratch2, $op1, $op2\n\t"
15197 "cmp rscratch2, rscratch1, ASR #63\n\t"
15198 "movw rscratch1, #0x80000000\n\t"
15199 "cselw rscratch1, rscratch1, zr, NE\n\t"
15200 "cmpw rscratch1, #1" %}
15201 ins_cost(6 * INSN_COST);
15202 ins_encode %{
15203 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15204 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15205 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15206 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15207 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15208 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15209 %}
15210
15211 ins_pipe(pipe_slow);
15212 %}
15213
15214 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15215 %{
15216 match(If cmp (OverflowMulL op1 op2));
15217 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15218 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15219 effect(USE labl, KILL cr);
15220
15221 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15222 "smulh rscratch2, $op1, $op2\n\t"
15223 "cmp rscratch2, rscratch1, ASR #63\n\t"
15224 "b$cmp $labl" %}
15225 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15226 ins_encode %{
15227 Label* L = $labl$$label;
15228 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15229 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15230 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15231 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15232 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15233 %}
15234
15235 ins_pipe(pipe_serial);
15236 %}
15237
15238 // ============================================================================
15239 // Compare Instructions
15240
15241 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15242 %{
15243 match(Set cr (CmpI op1 op2));
15244
15245 effect(DEF cr, USE op1, USE op2);
15246
15247 ins_cost(INSN_COST);
15248 format %{ "cmpw $op1, $op2" %}
15249
15250 ins_encode(aarch64_enc_cmpw(op1, op2));
15251
15252 ins_pipe(icmp_reg_reg);
15253 %}
15254
15255 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15256 %{
15257 match(Set cr (CmpI op1 zero));
15258
15259 effect(DEF cr, USE op1);
15260
15261 ins_cost(INSN_COST);
15262 format %{ "cmpw $op1, 0" %}
15263
15264 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15265
15266 ins_pipe(icmp_reg_imm);
15267 %}
15268
15269 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15270 %{
15271 match(Set cr (CmpI op1 op2));
15272
15273 effect(DEF cr, USE op1);
15274
15275 ins_cost(INSN_COST);
15276 format %{ "cmpw $op1, $op2" %}
15277
15278 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15279
15280 ins_pipe(icmp_reg_imm);
15281 %}
15282
15283 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15284 %{
15285 match(Set cr (CmpI op1 op2));
15286
15287 effect(DEF cr, USE op1);
15288
15289 ins_cost(INSN_COST * 2);
15290 format %{ "cmpw $op1, $op2" %}
15291
15292 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15293
15294 ins_pipe(icmp_reg_imm);
15295 %}
15296
15297 // Unsigned compare Instructions; really, same as signed compare
15298 // except it should only be used to feed an If or a CMovI which takes a
15299 // cmpOpU.
15300
15301 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15302 %{
15303 match(Set cr (CmpU op1 op2));
15304
15305 effect(DEF cr, USE op1, USE op2);
15306
15307 ins_cost(INSN_COST);
15308 format %{ "cmpw $op1, $op2\t# unsigned" %}
15309
15310 ins_encode(aarch64_enc_cmpw(op1, op2));
15311
15312 ins_pipe(icmp_reg_reg);
15313 %}
15314
15315 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15316 %{
15317 match(Set cr (CmpU op1 zero));
15318
15319 effect(DEF cr, USE op1);
15320
15321 ins_cost(INSN_COST);
15322 format %{ "cmpw $op1, #0\t# unsigned" %}
15323
15324 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15325
15326 ins_pipe(icmp_reg_imm);
15327 %}
15328
15329 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15330 %{
15331 match(Set cr (CmpU op1 op2));
15332
15333 effect(DEF cr, USE op1);
15334
15335 ins_cost(INSN_COST);
15336 format %{ "cmpw $op1, $op2\t# unsigned" %}
15337
15338 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15339
15340 ins_pipe(icmp_reg_imm);
15341 %}
15342
15343 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15344 %{
15345 match(Set cr (CmpU op1 op2));
15346
15347 effect(DEF cr, USE op1);
15348
15349 ins_cost(INSN_COST * 2);
15350 format %{ "cmpw $op1, $op2\t# unsigned" %}
15351
15352 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15353
15354 ins_pipe(icmp_reg_imm);
15355 %}
15356
15357 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15358 %{
15359 match(Set cr (CmpL op1 op2));
15360
15361 effect(DEF cr, USE op1, USE op2);
15362
15363 ins_cost(INSN_COST);
15364 format %{ "cmp $op1, $op2" %}
15365
15366 ins_encode(aarch64_enc_cmp(op1, op2));
15367
15368 ins_pipe(icmp_reg_reg);
15369 %}
15370
15371 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15372 %{
15373 match(Set cr (CmpL op1 zero));
15374
15375 effect(DEF cr, USE op1);
15376
15377 ins_cost(INSN_COST);
15378 format %{ "tst $op1" %}
15379
15380 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15381
15382 ins_pipe(icmp_reg_imm);
15383 %}
15384
15385 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15386 %{
15387 match(Set cr (CmpL op1 op2));
15388
15389 effect(DEF cr, USE op1);
15390
15391 ins_cost(INSN_COST);
15392 format %{ "cmp $op1, $op2" %}
15393
15394 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15395
15396 ins_pipe(icmp_reg_imm);
15397 %}
15398
15399 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15400 %{
15401 match(Set cr (CmpL op1 op2));
15402
15403 effect(DEF cr, USE op1);
15404
15405 ins_cost(INSN_COST * 2);
15406 format %{ "cmp $op1, $op2" %}
15407
15408 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15409
15410 ins_pipe(icmp_reg_imm);
15411 %}
15412
15413 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15414 %{
15415 match(Set cr (CmpUL op1 op2));
15416
15417 effect(DEF cr, USE op1, USE op2);
15418
15419 ins_cost(INSN_COST);
15420 format %{ "cmp $op1, $op2" %}
15421
15422 ins_encode(aarch64_enc_cmp(op1, op2));
15423
15424 ins_pipe(icmp_reg_reg);
15425 %}
15426
15427 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15428 %{
15429 match(Set cr (CmpUL op1 zero));
15430
15431 effect(DEF cr, USE op1);
15432
15433 ins_cost(INSN_COST);
15434 format %{ "tst $op1" %}
15435
15436 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15437
15438 ins_pipe(icmp_reg_imm);
15439 %}
15440
15441 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15442 %{
15443 match(Set cr (CmpUL op1 op2));
15444
15445 effect(DEF cr, USE op1);
15446
15447 ins_cost(INSN_COST);
15448 format %{ "cmp $op1, $op2" %}
15449
15450 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15451
15452 ins_pipe(icmp_reg_imm);
15453 %}
15454
15455 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15456 %{
15457 match(Set cr (CmpUL op1 op2));
15458
15459 effect(DEF cr, USE op1);
15460
15461 ins_cost(INSN_COST * 2);
15462 format %{ "cmp $op1, $op2" %}
15463
15464 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15465
15466 ins_pipe(icmp_reg_imm);
15467 %}
15468
15469 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15470 %{
15471 match(Set cr (CmpP op1 op2));
15472
15473 effect(DEF cr, USE op1, USE op2);
15474
15475 ins_cost(INSN_COST);
15476 format %{ "cmp $op1, $op2\t // ptr" %}
15477
15478 ins_encode(aarch64_enc_cmpp(op1, op2));
15479
15480 ins_pipe(icmp_reg_reg);
15481 %}
15482
15483 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15484 %{
15485 match(Set cr (CmpN op1 op2));
15486
15487 effect(DEF cr, USE op1, USE op2);
15488
15489 ins_cost(INSN_COST);
15490 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15491
15492 ins_encode(aarch64_enc_cmpn(op1, op2));
15493
15494 ins_pipe(icmp_reg_reg);
15495 %}
15496
15497 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15498 %{
15499 match(Set cr (CmpP op1 zero));
15500
15501 effect(DEF cr, USE op1, USE zero);
15502
15503 ins_cost(INSN_COST);
15504 format %{ "cmp $op1, 0\t // ptr" %}
15505
15506 ins_encode(aarch64_enc_testp(op1));
15507
15508 ins_pipe(icmp_reg_imm);
15509 %}
15510
15511 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15512 %{
15513 match(Set cr (CmpN op1 zero));
15514
15515 effect(DEF cr, USE op1, USE zero);
15516
15517 ins_cost(INSN_COST);
15518 format %{ "cmp $op1, 0\t // compressed ptr" %}
15519
15520 ins_encode(aarch64_enc_testn(op1));
15521
15522 ins_pipe(icmp_reg_imm);
15523 %}
15524
15525 // FP comparisons
15526 //
15527 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15528 // using normal cmpOp. See declaration of rFlagsReg for details.
15529
15530 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15531 %{
15532 match(Set cr (CmpF src1 src2));
15533
15534 ins_cost(3 * INSN_COST);
15535 format %{ "fcmps $src1, $src2" %}
15536
15537 ins_encode %{
15538 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15539 %}
15540
15541 ins_pipe(pipe_class_compare);
15542 %}
15543
15544 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15545 %{
15546 match(Set cr (CmpF src1 src2));
15547
15548 ins_cost(3 * INSN_COST);
15549 format %{ "fcmps $src1, 0.0" %}
15550
15551 ins_encode %{
15552 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15553 %}
15554
15555 ins_pipe(pipe_class_compare);
15556 %}
15557 // FROM HERE
15558
15559 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15560 %{
15561 match(Set cr (CmpD src1 src2));
15562
15563 ins_cost(3 * INSN_COST);
15564 format %{ "fcmpd $src1, $src2" %}
15565
15566 ins_encode %{
15567 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15568 %}
15569
15570 ins_pipe(pipe_class_compare);
15571 %}
15572
15573 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15574 %{
15575 match(Set cr (CmpD src1 src2));
15576
15577 ins_cost(3 * INSN_COST);
15578 format %{ "fcmpd $src1, 0.0" %}
15579
15580 ins_encode %{
15581 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15582 %}
15583
15584 ins_pipe(pipe_class_compare);
15585 %}
15586
15587 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15588 %{
15589 match(Set dst (CmpF3 src1 src2));
15590 effect(KILL cr);
15591
15592 ins_cost(5 * INSN_COST);
15593 format %{ "fcmps $src1, $src2\n\t"
15594 "csinvw($dst, zr, zr, eq\n\t"
15595 "csnegw($dst, $dst, $dst, lt)"
15596 %}
15597
15598 ins_encode %{
15599 Label done;
15600 FloatRegister s1 = as_FloatRegister($src1$$reg);
15601 FloatRegister s2 = as_FloatRegister($src2$$reg);
15602 Register d = as_Register($dst$$reg);
15603 __ fcmps(s1, s2);
15604 // installs 0 if EQ else -1
15605 __ csinvw(d, zr, zr, Assembler::EQ);
15606 // keeps -1 if less or unordered else installs 1
15607 __ csnegw(d, d, d, Assembler::LT);
15608 __ bind(done);
15609 %}
15610
15611 ins_pipe(pipe_class_default);
15612
15613 %}
15614
15615 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15616 %{
15617 match(Set dst (CmpD3 src1 src2));
15618 effect(KILL cr);
15619
15620 ins_cost(5 * INSN_COST);
15621 format %{ "fcmpd $src1, $src2\n\t"
15622 "csinvw($dst, zr, zr, eq\n\t"
15623 "csnegw($dst, $dst, $dst, lt)"
15624 %}
15625
15626 ins_encode %{
15627 Label done;
15628 FloatRegister s1 = as_FloatRegister($src1$$reg);
15629 FloatRegister s2 = as_FloatRegister($src2$$reg);
15630 Register d = as_Register($dst$$reg);
15631 __ fcmpd(s1, s2);
15632 // installs 0 if EQ else -1
15633 __ csinvw(d, zr, zr, Assembler::EQ);
15634 // keeps -1 if less or unordered else installs 1
15635 __ csnegw(d, d, d, Assembler::LT);
15636 __ bind(done);
15637 %}
15638 ins_pipe(pipe_class_default);
15639
15640 %}
15641
15642 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15643 %{
15644 match(Set dst (CmpF3 src1 zero));
15645 effect(KILL cr);
15646
15647 ins_cost(5 * INSN_COST);
15648 format %{ "fcmps $src1, 0.0\n\t"
15649 "csinvw($dst, zr, zr, eq\n\t"
15650 "csnegw($dst, $dst, $dst, lt)"
15651 %}
15652
15653 ins_encode %{
15654 Label done;
15655 FloatRegister s1 = as_FloatRegister($src1$$reg);
15656 Register d = as_Register($dst$$reg);
15657 __ fcmps(s1, 0.0);
15658 // installs 0 if EQ else -1
15659 __ csinvw(d, zr, zr, Assembler::EQ);
15660 // keeps -1 if less or unordered else installs 1
15661 __ csnegw(d, d, d, Assembler::LT);
15662 __ bind(done);
15663 %}
15664
15665 ins_pipe(pipe_class_default);
15666
15667 %}
15668
15669 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15670 %{
15671 match(Set dst (CmpD3 src1 zero));
15672 effect(KILL cr);
15673
15674 ins_cost(5 * INSN_COST);
15675 format %{ "fcmpd $src1, 0.0\n\t"
15676 "csinvw($dst, zr, zr, eq\n\t"
15677 "csnegw($dst, $dst, $dst, lt)"
15678 %}
15679
15680 ins_encode %{
15681 Label done;
15682 FloatRegister s1 = as_FloatRegister($src1$$reg);
15683 Register d = as_Register($dst$$reg);
15684 __ fcmpd(s1, 0.0);
15685 // installs 0 if EQ else -1
15686 __ csinvw(d, zr, zr, Assembler::EQ);
15687 // keeps -1 if less or unordered else installs 1
15688 __ csnegw(d, d, d, Assembler::LT);
15689 __ bind(done);
15690 %}
15691 ins_pipe(pipe_class_default);
15692
15693 %}
15694
15695 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15696 %{
15697 match(Set dst (CmpLTMask p q));
15698 effect(KILL cr);
15699
15700 ins_cost(3 * INSN_COST);
15701
15702 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15703 "csetw $dst, lt\n\t"
15704 "subw $dst, zr, $dst"
15705 %}
15706
15707 ins_encode %{
15708 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15709 __ csetw(as_Register($dst$$reg), Assembler::LT);
15710 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15711 %}
15712
15713 ins_pipe(ialu_reg_reg);
15714 %}
15715
15716 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15717 %{
15718 match(Set dst (CmpLTMask src zero));
15719 effect(KILL cr);
15720
15721 ins_cost(INSN_COST);
15722
15723 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15724
15725 ins_encode %{
15726 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15727 %}
15728
15729 ins_pipe(ialu_reg_shift);
15730 %}
15731
15732 // ============================================================================
15733 // Max and Min
15734
15735 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15736
15737 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15738 %{
15739 effect(DEF cr, USE src);
15740 ins_cost(INSN_COST);
15741 format %{ "cmpw $src, 0" %}
15742
15743 ins_encode %{
15744 __ cmpw($src$$Register, 0);
15745 %}
15746 ins_pipe(icmp_reg_imm);
15747 %}
15748
15749 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15750 %{
15751 match(Set dst (MinI src1 src2));
15752 ins_cost(INSN_COST * 3);
15753
15754 expand %{
15755 rFlagsReg cr;
15756 compI_reg_reg(cr, src1, src2);
15757 cmovI_reg_reg_lt(dst, src1, src2, cr);
15758 %}
15759 %}
15760
15761 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15762 %{
15763 match(Set dst (MaxI src1 src2));
15764 ins_cost(INSN_COST * 3);
15765
15766 expand %{
15767 rFlagsReg cr;
15768 compI_reg_reg(cr, src1, src2);
15769 cmovI_reg_reg_gt(dst, src1, src2, cr);
15770 %}
15771 %}
15772
15773
15774 // ============================================================================
15775 // Branch Instructions
15776
15777 // Direct Branch.
15778 instruct branch(label lbl)
15779 %{
15780 match(Goto);
15781
15782 effect(USE lbl);
15783
15784 ins_cost(BRANCH_COST);
15785 format %{ "b $lbl" %}
15786
15787 ins_encode(aarch64_enc_b(lbl));
15788
15789 ins_pipe(pipe_branch);
15790 %}
15791
15792 // Conditional Near Branch
15793 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15794 %{
15795 // Same match rule as `branchConFar'.
15796 match(If cmp cr);
15797
15798 effect(USE lbl);
15799
15800 ins_cost(BRANCH_COST);
15801 // If set to 1 this indicates that the current instruction is a
15802 // short variant of a long branch. This avoids using this
15803 // instruction in first-pass matching. It will then only be used in
15804 // the `Shorten_branches' pass.
15805 // ins_short_branch(1);
15806 format %{ "b$cmp $lbl" %}
15807
15808 ins_encode(aarch64_enc_br_con(cmp, lbl));
15809
15810 ins_pipe(pipe_branch_cond);
15811 %}
15812
15813 // Conditional Near Branch Unsigned
15814 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15815 %{
15816 // Same match rule as `branchConFar'.
15817 match(If cmp cr);
15818
15819 effect(USE lbl);
15820
15821 ins_cost(BRANCH_COST);
15822 // If set to 1 this indicates that the current instruction is a
15823 // short variant of a long branch. This avoids using this
15824 // instruction in first-pass matching. It will then only be used in
15825 // the `Shorten_branches' pass.
15826 // ins_short_branch(1);
15827 format %{ "b$cmp $lbl\t# unsigned" %}
15828
15829 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15830
15831 ins_pipe(pipe_branch_cond);
15832 %}
15833
15834 // Make use of CBZ and CBNZ. These instructions, as well as being
15835 // shorter than (cmp; branch), have the additional benefit of not
15836 // killing the flags.
15837
15838 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15839 match(If cmp (CmpI op1 op2));
15840 effect(USE labl);
15841
15842 ins_cost(BRANCH_COST);
15843 format %{ "cbw$cmp $op1, $labl" %}
15844 ins_encode %{
15845 Label* L = $labl$$label;
15846 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15847 if (cond == Assembler::EQ)
15848 __ cbzw($op1$$Register, *L);
15849 else
15850 __ cbnzw($op1$$Register, *L);
15851 %}
15852 ins_pipe(pipe_cmp_branch);
15853 %}
15854
15855 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15856 match(If cmp (CmpL op1 op2));
15857 effect(USE labl);
15858
15859 ins_cost(BRANCH_COST);
15860 format %{ "cb$cmp $op1, $labl" %}
15861 ins_encode %{
15862 Label* L = $labl$$label;
15863 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15864 if (cond == Assembler::EQ)
15865 __ cbz($op1$$Register, *L);
15866 else
15867 __ cbnz($op1$$Register, *L);
15868 %}
15869 ins_pipe(pipe_cmp_branch);
15870 %}
15871
15872 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15873 match(If cmp (CmpP op1 op2));
15874 effect(USE labl);
15875
15876 ins_cost(BRANCH_COST);
15877 format %{ "cb$cmp $op1, $labl" %}
15878 ins_encode %{
15879 Label* L = $labl$$label;
15880 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15881 if (cond == Assembler::EQ)
15882 __ cbz($op1$$Register, *L);
15883 else
15884 __ cbnz($op1$$Register, *L);
15885 %}
15886 ins_pipe(pipe_cmp_branch);
15887 %}
15888
15889 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15890 match(If cmp (CmpN op1 op2));
15891 effect(USE labl);
15892
15893 ins_cost(BRANCH_COST);
15894 format %{ "cbw$cmp $op1, $labl" %}
15895 ins_encode %{
15896 Label* L = $labl$$label;
15897 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15898 if (cond == Assembler::EQ)
15899 __ cbzw($op1$$Register, *L);
15900 else
15901 __ cbnzw($op1$$Register, *L);
15902 %}
15903 ins_pipe(pipe_cmp_branch);
15904 %}
15905
15906 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15907 match(If cmp (CmpP (DecodeN oop) zero));
15908 effect(USE labl);
15909
15910 ins_cost(BRANCH_COST);
15911 format %{ "cb$cmp $oop, $labl" %}
15912 ins_encode %{
15913 Label* L = $labl$$label;
15914 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15915 if (cond == Assembler::EQ)
15916 __ cbzw($oop$$Register, *L);
15917 else
15918 __ cbnzw($oop$$Register, *L);
15919 %}
15920 ins_pipe(pipe_cmp_branch);
15921 %}
15922
15923 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15924 match(If cmp (CmpU op1 op2));
15925 effect(USE labl);
15926
15927 ins_cost(BRANCH_COST);
15928 format %{ "cbw$cmp $op1, $labl" %}
15929 ins_encode %{
15930 Label* L = $labl$$label;
15931 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15932 if (cond == Assembler::EQ || cond == Assembler::LS) {
15933 __ cbzw($op1$$Register, *L);
15934 } else {
15935 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15936 __ cbnzw($op1$$Register, *L);
15937 }
15938 %}
15939 ins_pipe(pipe_cmp_branch);
15940 %}
15941
15942 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15943 match(If cmp (CmpUL op1 op2));
15944 effect(USE labl);
15945
15946 ins_cost(BRANCH_COST);
15947 format %{ "cb$cmp $op1, $labl" %}
15948 ins_encode %{
15949 Label* L = $labl$$label;
15950 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15951 if (cond == Assembler::EQ || cond == Assembler::LS) {
15952 __ cbz($op1$$Register, *L);
15953 } else {
15954 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15955 __ cbnz($op1$$Register, *L);
15956 }
15957 %}
15958 ins_pipe(pipe_cmp_branch);
15959 %}
15960
15961 // Test bit and Branch
15962
15963 // Patterns for short (< 32KiB) variants
15964 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15965 match(If cmp (CmpL op1 op2));
15966 effect(USE labl);
15967
15968 ins_cost(BRANCH_COST);
15969 format %{ "cb$cmp $op1, $labl # long" %}
15970 ins_encode %{
15971 Label* L = $labl$$label;
15972 Assembler::Condition cond =
15973 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15974 __ tbr(cond, $op1$$Register, 63, *L);
15975 %}
15976 ins_pipe(pipe_cmp_branch);
15977 ins_short_branch(1);
15978 %}
15979
15980 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15981 match(If cmp (CmpI op1 op2));
15982 effect(USE labl);
15983
15984 ins_cost(BRANCH_COST);
15985 format %{ "cb$cmp $op1, $labl # int" %}
15986 ins_encode %{
15987 Label* L = $labl$$label;
15988 Assembler::Condition cond =
15989 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15990 __ tbr(cond, $op1$$Register, 31, *L);
15991 %}
15992 ins_pipe(pipe_cmp_branch);
15993 ins_short_branch(1);
15994 %}
15995
15996 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15997 match(If cmp (CmpL (AndL op1 op2) op3));
15998 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15999 effect(USE labl);
16000
16001 ins_cost(BRANCH_COST);
16002 format %{ "tb$cmp $op1, $op2, $labl" %}
16003 ins_encode %{
16004 Label* L = $labl$$label;
16005 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16006 int bit = exact_log2_long($op2$$constant);
16007 __ tbr(cond, $op1$$Register, bit, *L);
16008 %}
16009 ins_pipe(pipe_cmp_branch);
16010 ins_short_branch(1);
16011 %}
16012
16013 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16014 match(If cmp (CmpI (AndI op1 op2) op3));
16015 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16016 effect(USE labl);
16017
16018 ins_cost(BRANCH_COST);
16019 format %{ "tb$cmp $op1, $op2, $labl" %}
16020 ins_encode %{
16021 Label* L = $labl$$label;
16022 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16023 int bit = exact_log2((juint)$op2$$constant);
16024 __ tbr(cond, $op1$$Register, bit, *L);
16025 %}
16026 ins_pipe(pipe_cmp_branch);
16027 ins_short_branch(1);
16028 %}
16029
16030 // And far variants
16031 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16032 match(If cmp (CmpL op1 op2));
16033 effect(USE labl);
16034
16035 ins_cost(BRANCH_COST);
16036 format %{ "cb$cmp $op1, $labl # long" %}
16037 ins_encode %{
16038 Label* L = $labl$$label;
16039 Assembler::Condition cond =
16040 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16041 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16042 %}
16043 ins_pipe(pipe_cmp_branch);
16044 %}
16045
16046 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16047 match(If cmp (CmpI op1 op2));
16048 effect(USE labl);
16049
16050 ins_cost(BRANCH_COST);
16051 format %{ "cb$cmp $op1, $labl # int" %}
16052 ins_encode %{
16053 Label* L = $labl$$label;
16054 Assembler::Condition cond =
16055 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16056 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16057 %}
16058 ins_pipe(pipe_cmp_branch);
16059 %}
16060
16061 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16062 match(If cmp (CmpL (AndL op1 op2) op3));
16063 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16064 effect(USE labl);
16065
16066 ins_cost(BRANCH_COST);
16067 format %{ "tb$cmp $op1, $op2, $labl" %}
16068 ins_encode %{
16069 Label* L = $labl$$label;
16070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16071 int bit = exact_log2_long($op2$$constant);
16072 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16073 %}
16074 ins_pipe(pipe_cmp_branch);
16075 %}
16076
16077 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16078 match(If cmp (CmpI (AndI op1 op2) op3));
16079 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16080 effect(USE labl);
16081
16082 ins_cost(BRANCH_COST);
16083 format %{ "tb$cmp $op1, $op2, $labl" %}
16084 ins_encode %{
16085 Label* L = $labl$$label;
16086 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16087 int bit = exact_log2((juint)$op2$$constant);
16088 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16089 %}
16090 ins_pipe(pipe_cmp_branch);
16091 %}
16092
16093 // Test bits
16094
16095 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16096 match(Set cr (CmpL (AndL op1 op2) op3));
16097 predicate(Assembler::operand_valid_for_logical_immediate
16098 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16099
16100 ins_cost(INSN_COST);
16101 format %{ "tst $op1, $op2 # long" %}
16102 ins_encode %{
16103 __ tst($op1$$Register, $op2$$constant);
16104 %}
16105 ins_pipe(ialu_reg_reg);
16106 %}
16107
16108 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16109 match(Set cr (CmpI (AndI op1 op2) op3));
16110 predicate(Assembler::operand_valid_for_logical_immediate
16111 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16112
16113 ins_cost(INSN_COST);
16114 format %{ "tst $op1, $op2 # int" %}
16115 ins_encode %{
16116 __ tstw($op1$$Register, $op2$$constant);
16117 %}
16118 ins_pipe(ialu_reg_reg);
16119 %}
16120
16121 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16122 match(Set cr (CmpL (AndL op1 op2) op3));
16123
16124 ins_cost(INSN_COST);
16125 format %{ "tst $op1, $op2 # long" %}
16126 ins_encode %{
16127 __ tst($op1$$Register, $op2$$Register);
16128 %}
16129 ins_pipe(ialu_reg_reg);
16130 %}
16131
16132 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16133 match(Set cr (CmpI (AndI op1 op2) op3));
16134
16135 ins_cost(INSN_COST);
16136 format %{ "tstw $op1, $op2 # int" %}
16137 ins_encode %{
16138 __ tstw($op1$$Register, $op2$$Register);
16139 %}
16140 ins_pipe(ialu_reg_reg);
16141 %}
16142
16143
16144 // Conditional Far Branch
16145 // Conditional Far Branch Unsigned
16146 // TODO: fixme
16147
16148 // counted loop end branch near
16149 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16150 %{
16151 match(CountedLoopEnd cmp cr);
16152
16153 effect(USE lbl);
16154
16155 ins_cost(BRANCH_COST);
16156 // short variant.
16157 // ins_short_branch(1);
16158 format %{ "b$cmp $lbl \t// counted loop end" %}
16159
16160 ins_encode(aarch64_enc_br_con(cmp, lbl));
16161
16162 ins_pipe(pipe_branch);
16163 %}
16164
16165 // counted loop end branch far
16166 // TODO: fixme
16167
16168 // ============================================================================
16169 // inlined locking and unlocking
16170
16171 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16172 %{
16173 predicate(LockingMode != LM_LIGHTWEIGHT);
16174 match(Set cr (FastLock object box));
16175 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16176
16177 ins_cost(5 * INSN_COST);
16178 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16179
16180 ins_encode %{
16181 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16182 %}
16183
16184 ins_pipe(pipe_serial);
16185 %}
16186
16187 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16188 %{
16189 predicate(LockingMode != LM_LIGHTWEIGHT);
16190 match(Set cr (FastUnlock object box));
16191 effect(TEMP tmp, TEMP tmp2);
16192
16193 ins_cost(5 * INSN_COST);
16194 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16195
16196 ins_encode %{
16197 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16198 %}
16199
16200 ins_pipe(pipe_serial);
16201 %}
16202
16203 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16204 %{
16205 predicate(LockingMode == LM_LIGHTWEIGHT);
16206 match(Set cr (FastLock object box));
16207 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16208
16209 ins_cost(5 * INSN_COST);
16210 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16211
16212 ins_encode %{
16213 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16214 %}
16215
16216 ins_pipe(pipe_serial);
16217 %}
16218
16219 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16220 %{
16221 predicate(LockingMode == LM_LIGHTWEIGHT);
16222 match(Set cr (FastUnlock object box));
16223 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16224
16225 ins_cost(5 * INSN_COST);
16226 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16227
16228 ins_encode %{
16229 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16230 %}
16231
16232 ins_pipe(pipe_serial);
16233 %}
16234
16235 // ============================================================================
16236 // Safepoint Instructions
16237
16238 // TODO
16239 // provide a near and far version of this code
16240
16241 instruct safePoint(rFlagsReg cr, iRegP poll)
16242 %{
16243 match(SafePoint poll);
16244 effect(KILL cr);
16245
16246 format %{
16247 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16248 %}
16249 ins_encode %{
16250 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16251 %}
16252 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16253 %}
16254
16255
16256 // ============================================================================
16257 // Procedure Call/Return Instructions
16258
16259 // Call Java Static Instruction
16260
16261 instruct CallStaticJavaDirect(method meth)
16262 %{
16263 match(CallStaticJava);
16264
16265 effect(USE meth);
16266
16267 ins_cost(CALL_COST);
16268
16269 format %{ "call,static $meth \t// ==> " %}
16270
16271 ins_encode(aarch64_enc_java_static_call(meth),
16272 aarch64_enc_call_epilog);
16273
16274 ins_pipe(pipe_class_call);
16275 %}
16276
16277 // TO HERE
16278
16279 // Call Java Dynamic Instruction
16280 instruct CallDynamicJavaDirect(method meth)
16281 %{
16282 match(CallDynamicJava);
16283
16284 effect(USE meth);
16285
16286 ins_cost(CALL_COST);
16287
16288 format %{ "CALL,dynamic $meth \t// ==> " %}
16289
16290 ins_encode(aarch64_enc_java_dynamic_call(meth),
16291 aarch64_enc_call_epilog);
16292
16293 ins_pipe(pipe_class_call);
16294 %}
16295
16296 // Call Runtime Instruction
16297
16298 instruct CallRuntimeDirect(method meth)
16299 %{
16300 match(CallRuntime);
16301
16302 effect(USE meth);
16303
16304 ins_cost(CALL_COST);
16305
16306 format %{ "CALL, runtime $meth" %}
16307
16308 ins_encode( aarch64_enc_java_to_runtime(meth) );
16309
16310 ins_pipe(pipe_class_call);
16311 %}
16312
16313 // Call Runtime Instruction
16314
16315 instruct CallLeafDirect(method meth)
16316 %{
16317 match(CallLeaf);
16318
16319 effect(USE meth);
16320
16321 ins_cost(CALL_COST);
16322
16323 format %{ "CALL, runtime leaf $meth" %}
16324
16325 ins_encode( aarch64_enc_java_to_runtime(meth) );
16326
16327 ins_pipe(pipe_class_call);
16328 %}
16329
16330 // Call Runtime Instruction without safepoint and with vector arguments
16331 instruct CallLeafDirectVector(method meth)
16332 %{
16333 match(CallLeafVector);
16334
16335 effect(USE meth);
16336
16337 ins_cost(CALL_COST);
16338
16339 format %{ "CALL, runtime leaf vector $meth" %}
16340
16341 ins_encode(aarch64_enc_java_to_runtime(meth));
16342
16343 ins_pipe(pipe_class_call);
16344 %}
16345
16346 // Call Runtime Instruction
16347
16348 instruct CallLeafNoFPDirect(method meth)
16349 %{
16350 match(CallLeafNoFP);
16351
16352 effect(USE meth);
16353
16354 ins_cost(CALL_COST);
16355
16356 format %{ "CALL, runtime leaf nofp $meth" %}
16357
16358 ins_encode( aarch64_enc_java_to_runtime(meth) );
16359
16360 ins_pipe(pipe_class_call);
16361 %}
16362
16363 // Tail Call; Jump from runtime stub to Java code.
16364 // Also known as an 'interprocedural jump'.
16365 // Target of jump will eventually return to caller.
16366 // TailJump below removes the return address.
16367 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16368 // emitted just above the TailCall which has reset rfp to the caller state.
16369 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16370 %{
16371 match(TailCall jump_target method_ptr);
16372
16373 ins_cost(CALL_COST);
16374
16375 format %{ "br $jump_target\t# $method_ptr holds method" %}
16376
16377 ins_encode(aarch64_enc_tail_call(jump_target));
16378
16379 ins_pipe(pipe_class_call);
16380 %}
16381
16382 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16383 %{
16384 match(TailJump jump_target ex_oop);
16385
16386 ins_cost(CALL_COST);
16387
16388 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16389
16390 ins_encode(aarch64_enc_tail_jmp(jump_target));
16391
16392 ins_pipe(pipe_class_call);
16393 %}
16394
16395 // Forward exception.
16396 instruct ForwardExceptionjmp()
16397 %{
16398 match(ForwardException);
16399 ins_cost(CALL_COST);
16400
16401 format %{ "b forward_exception_stub" %}
16402 ins_encode %{
16403 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16404 %}
16405 ins_pipe(pipe_class_call);
16406 %}
16407
16408 // Create exception oop: created by stack-crawling runtime code.
16409 // Created exception is now available to this handler, and is setup
16410 // just prior to jumping to this handler. No code emitted.
16411 // TODO check
16412 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16413 instruct CreateException(iRegP_R0 ex_oop)
16414 %{
16415 match(Set ex_oop (CreateEx));
16416
16417 format %{ " -- \t// exception oop; no code emitted" %}
16418
16419 size(0);
16420
16421 ins_encode( /*empty*/ );
16422
16423 ins_pipe(pipe_class_empty);
16424 %}
16425
16426 // Rethrow exception: The exception oop will come in the first
16427 // argument position. Then JUMP (not call) to the rethrow stub code.
16428 instruct RethrowException() %{
16429 match(Rethrow);
16430 ins_cost(CALL_COST);
16431
16432 format %{ "b rethrow_stub" %}
16433
16434 ins_encode( aarch64_enc_rethrow() );
16435
16436 ins_pipe(pipe_class_call);
16437 %}
16438
16439
16440 // Return Instruction
16441 // epilog node loads ret address into lr as part of frame pop
16442 instruct Ret()
16443 %{
16444 match(Return);
16445
16446 format %{ "ret\t// return register" %}
16447
16448 ins_encode( aarch64_enc_ret() );
16449
16450 ins_pipe(pipe_branch);
16451 %}
16452
16453 // Die now.
16454 instruct ShouldNotReachHere() %{
16455 match(Halt);
16456
16457 ins_cost(CALL_COST);
16458 format %{ "ShouldNotReachHere" %}
16459
16460 ins_encode %{
16461 if (is_reachable()) {
16462 const char* str = __ code_string(_halt_reason);
16463 __ stop(str);
16464 }
16465 %}
16466
16467 ins_pipe(pipe_class_default);
16468 %}
16469
16470 // ============================================================================
16471 // Partial Subtype Check
16472 //
16473 // superklass array for an instance of the superklass. Set a hidden
16474 // internal cache on a hit (cache is checked with exposed code in
16475 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16476 // encoding ALSO sets flags.
16477
16478 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16479 %{
16480 match(Set result (PartialSubtypeCheck sub super));
16481 predicate(!UseSecondarySupersTable);
16482 effect(KILL cr, KILL temp);
16483
16484 ins_cost(20 * INSN_COST); // slightly larger than the next version
16485 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16486
16487 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16488
16489 opcode(0x1); // Force zero of result reg on hit
16490
16491 ins_pipe(pipe_class_memory);
16492 %}
16493
16494 // Two versions of partialSubtypeCheck, both used when we need to
16495 // search for a super class in the secondary supers array. The first
16496 // is used when we don't know _a priori_ the class being searched
16497 // for. The second, far more common, is used when we do know: this is
16498 // used for instanceof, checkcast, and any case where C2 can determine
16499 // it by constant propagation.
16500
16501 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16502 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16503 rFlagsReg cr)
16504 %{
16505 match(Set result (PartialSubtypeCheck sub super));
16506 predicate(UseSecondarySupersTable);
16507 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16508
16509 ins_cost(10 * INSN_COST); // slightly larger than the next version
16510 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16511
16512 ins_encode %{
16513 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16514 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16515 $vtemp$$FloatRegister,
16516 $result$$Register, /*L_success*/nullptr);
16517 %}
16518
16519 ins_pipe(pipe_class_memory);
16520 %}
16521
16522 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16523 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16524 rFlagsReg cr)
16525 %{
16526 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16527 predicate(UseSecondarySupersTable);
16528 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16529
16530 ins_cost(5 * INSN_COST); // smaller than the next version
16531 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16532
16533 ins_encode %{
16534 bool success = false;
16535 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16536 if (InlineSecondarySupersTest) {
16537 success =
16538 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16539 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16540 $vtemp$$FloatRegister,
16541 $result$$Register,
16542 super_klass_slot);
16543 } else {
16544 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16545 success = (call != nullptr);
16546 }
16547 if (!success) {
16548 ciEnv::current()->record_failure("CodeCache is full");
16549 return;
16550 }
16551 %}
16552
16553 ins_pipe(pipe_class_memory);
16554 %}
16555
16556 // Intrisics for String.compareTo()
16557
16558 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16559 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16560 %{
16561 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16562 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16563 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16564
16565 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16566 ins_encode %{
16567 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16568 __ string_compare($str1$$Register, $str2$$Register,
16569 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16570 $tmp1$$Register, $tmp2$$Register,
16571 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16572 %}
16573 ins_pipe(pipe_class_memory);
16574 %}
16575
16576 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16577 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16578 %{
16579 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16580 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16581 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16582
16583 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16584 ins_encode %{
16585 __ string_compare($str1$$Register, $str2$$Register,
16586 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16587 $tmp1$$Register, $tmp2$$Register,
16588 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16589 %}
16590 ins_pipe(pipe_class_memory);
16591 %}
16592
16593 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16594 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16595 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16596 %{
16597 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16598 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16599 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16600 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16601
16602 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16603 ins_encode %{
16604 __ string_compare($str1$$Register, $str2$$Register,
16605 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16606 $tmp1$$Register, $tmp2$$Register,
16607 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16608 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16609 %}
16610 ins_pipe(pipe_class_memory);
16611 %}
16612
16613 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16614 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16615 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16616 %{
16617 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16618 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16619 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16620 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16621
16622 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16623 ins_encode %{
16624 __ string_compare($str1$$Register, $str2$$Register,
16625 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16626 $tmp1$$Register, $tmp2$$Register,
16627 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16628 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16629 %}
16630 ins_pipe(pipe_class_memory);
16631 %}
16632
16633 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16634 // these string_compare variants as NEON register type for convenience so that the prototype of
16635 // string_compare can be shared with all variants.
16636
16637 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16638 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16639 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16640 pRegGov_P1 pgtmp2, rFlagsReg cr)
16641 %{
16642 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16643 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16644 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16645 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16646
16647 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16648 ins_encode %{
16649 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16650 __ string_compare($str1$$Register, $str2$$Register,
16651 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16652 $tmp1$$Register, $tmp2$$Register,
16653 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16654 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16655 StrIntrinsicNode::LL);
16656 %}
16657 ins_pipe(pipe_class_memory);
16658 %}
16659
16660 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16661 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16662 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16663 pRegGov_P1 pgtmp2, rFlagsReg cr)
16664 %{
16665 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16666 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16667 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16668 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16669
16670 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16671 ins_encode %{
16672 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16673 __ string_compare($str1$$Register, $str2$$Register,
16674 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16675 $tmp1$$Register, $tmp2$$Register,
16676 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16677 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16678 StrIntrinsicNode::LU);
16679 %}
16680 ins_pipe(pipe_class_memory);
16681 %}
16682
16683 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16684 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16685 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16686 pRegGov_P1 pgtmp2, rFlagsReg cr)
16687 %{
16688 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16689 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16690 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16691 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16692
16693 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16694 ins_encode %{
16695 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16696 __ string_compare($str1$$Register, $str2$$Register,
16697 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16698 $tmp1$$Register, $tmp2$$Register,
16699 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16700 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16701 StrIntrinsicNode::UL);
16702 %}
16703 ins_pipe(pipe_class_memory);
16704 %}
16705
16706 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16707 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16708 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16709 pRegGov_P1 pgtmp2, rFlagsReg cr)
16710 %{
16711 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16712 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16713 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16714 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16715
16716 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16717 ins_encode %{
16718 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16719 __ string_compare($str1$$Register, $str2$$Register,
16720 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16721 $tmp1$$Register, $tmp2$$Register,
16722 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16723 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16724 StrIntrinsicNode::UU);
16725 %}
16726 ins_pipe(pipe_class_memory);
16727 %}
16728
16729 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16730 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16731 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16732 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16733 %{
16734 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16735 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16736 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16737 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16738 TEMP vtmp0, TEMP vtmp1, KILL cr);
16739 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16740 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16741
16742 ins_encode %{
16743 __ string_indexof($str1$$Register, $str2$$Register,
16744 $cnt1$$Register, $cnt2$$Register,
16745 $tmp1$$Register, $tmp2$$Register,
16746 $tmp3$$Register, $tmp4$$Register,
16747 $tmp5$$Register, $tmp6$$Register,
16748 -1, $result$$Register, StrIntrinsicNode::UU);
16749 %}
16750 ins_pipe(pipe_class_memory);
16751 %}
16752
16753 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16754 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16755 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16756 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16757 %{
16758 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16759 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16760 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16761 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16762 TEMP vtmp0, TEMP vtmp1, KILL cr);
16763 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16764 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16765
16766 ins_encode %{
16767 __ string_indexof($str1$$Register, $str2$$Register,
16768 $cnt1$$Register, $cnt2$$Register,
16769 $tmp1$$Register, $tmp2$$Register,
16770 $tmp3$$Register, $tmp4$$Register,
16771 $tmp5$$Register, $tmp6$$Register,
16772 -1, $result$$Register, StrIntrinsicNode::LL);
16773 %}
16774 ins_pipe(pipe_class_memory);
16775 %}
16776
16777 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16778 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16779 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16780 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16781 %{
16782 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16783 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16784 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16785 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16786 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16787 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16788 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16789
16790 ins_encode %{
16791 __ string_indexof($str1$$Register, $str2$$Register,
16792 $cnt1$$Register, $cnt2$$Register,
16793 $tmp1$$Register, $tmp2$$Register,
16794 $tmp3$$Register, $tmp4$$Register,
16795 $tmp5$$Register, $tmp6$$Register,
16796 -1, $result$$Register, StrIntrinsicNode::UL);
16797 %}
16798 ins_pipe(pipe_class_memory);
16799 %}
16800
16801 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16802 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16803 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16804 %{
16805 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16806 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16807 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16808 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16809 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16810 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16811
16812 ins_encode %{
16813 int icnt2 = (int)$int_cnt2$$constant;
16814 __ string_indexof($str1$$Register, $str2$$Register,
16815 $cnt1$$Register, zr,
16816 $tmp1$$Register, $tmp2$$Register,
16817 $tmp3$$Register, $tmp4$$Register, zr, zr,
16818 icnt2, $result$$Register, StrIntrinsicNode::UU);
16819 %}
16820 ins_pipe(pipe_class_memory);
16821 %}
16822
16823 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16824 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16825 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16826 %{
16827 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16828 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16829 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16830 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16831 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16832 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16833
16834 ins_encode %{
16835 int icnt2 = (int)$int_cnt2$$constant;
16836 __ string_indexof($str1$$Register, $str2$$Register,
16837 $cnt1$$Register, zr,
16838 $tmp1$$Register, $tmp2$$Register,
16839 $tmp3$$Register, $tmp4$$Register, zr, zr,
16840 icnt2, $result$$Register, StrIntrinsicNode::LL);
16841 %}
16842 ins_pipe(pipe_class_memory);
16843 %}
16844
16845 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16846 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16847 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16848 %{
16849 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16850 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16851 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16852 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16853 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16854 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16855
16856 ins_encode %{
16857 int icnt2 = (int)$int_cnt2$$constant;
16858 __ string_indexof($str1$$Register, $str2$$Register,
16859 $cnt1$$Register, zr,
16860 $tmp1$$Register, $tmp2$$Register,
16861 $tmp3$$Register, $tmp4$$Register, zr, zr,
16862 icnt2, $result$$Register, StrIntrinsicNode::UL);
16863 %}
16864 ins_pipe(pipe_class_memory);
16865 %}
16866
16867 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16868 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16869 iRegINoSp tmp3, rFlagsReg cr)
16870 %{
16871 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16872 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16873 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16874 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16875
16876 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16877
16878 ins_encode %{
16879 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16880 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16881 $tmp3$$Register);
16882 %}
16883 ins_pipe(pipe_class_memory);
16884 %}
16885
16886 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16887 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16888 iRegINoSp tmp3, rFlagsReg cr)
16889 %{
16890 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16891 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16892 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16893 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16894
16895 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16896
16897 ins_encode %{
16898 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16899 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16900 $tmp3$$Register);
16901 %}
16902 ins_pipe(pipe_class_memory);
16903 %}
16904
16905 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16906 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16907 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16908 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16909 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16910 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16911 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16912 ins_encode %{
16913 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16914 $result$$Register, $ztmp1$$FloatRegister,
16915 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16916 $ptmp$$PRegister, true /* isL */);
16917 %}
16918 ins_pipe(pipe_class_memory);
16919 %}
16920
16921 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16922 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16923 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16924 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16925 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16926 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16927 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16928 ins_encode %{
16929 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16930 $result$$Register, $ztmp1$$FloatRegister,
16931 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16932 $ptmp$$PRegister, false /* isL */);
16933 %}
16934 ins_pipe(pipe_class_memory);
16935 %}
16936
16937 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16938 iRegI_R0 result, rFlagsReg cr)
16939 %{
16940 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16941 match(Set result (StrEquals (Binary str1 str2) cnt));
16942 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16943
16944 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16945 ins_encode %{
16946 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16947 __ string_equals($str1$$Register, $str2$$Register,
16948 $result$$Register, $cnt$$Register);
16949 %}
16950 ins_pipe(pipe_class_memory);
16951 %}
16952
16953 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16954 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16955 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16956 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16957 iRegP_R10 tmp, rFlagsReg cr)
16958 %{
16959 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16960 match(Set result (AryEq ary1 ary2));
16961 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16962 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16963 TEMP vtmp6, TEMP vtmp7, KILL cr);
16964
16965 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16966 ins_encode %{
16967 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16968 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16969 $result$$Register, $tmp$$Register, 1);
16970 if (tpc == nullptr) {
16971 ciEnv::current()->record_failure("CodeCache is full");
16972 return;
16973 }
16974 %}
16975 ins_pipe(pipe_class_memory);
16976 %}
16977
16978 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16979 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16980 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16981 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16982 iRegP_R10 tmp, rFlagsReg cr)
16983 %{
16984 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16985 match(Set result (AryEq ary1 ary2));
16986 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16987 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16988 TEMP vtmp6, TEMP vtmp7, KILL cr);
16989
16990 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16991 ins_encode %{
16992 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16993 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16994 $result$$Register, $tmp$$Register, 2);
16995 if (tpc == nullptr) {
16996 ciEnv::current()->record_failure("CodeCache is full");
16997 return;
16998 }
16999 %}
17000 ins_pipe(pipe_class_memory);
17001 %}
17002
17003 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17004 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17005 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17006 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17007 %{
17008 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17009 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17010 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17011
17012 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17013 ins_encode %{
17014 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17015 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17016 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17017 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17018 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17019 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17020 (BasicType)$basic_type$$constant);
17021 if (tpc == nullptr) {
17022 ciEnv::current()->record_failure("CodeCache is full");
17023 return;
17024 }
17025 %}
17026 ins_pipe(pipe_class_memory);
17027 %}
17028
17029 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17030 %{
17031 match(Set result (CountPositives ary1 len));
17032 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17033 format %{ "count positives byte[] $ary1,$len -> $result" %}
17034 ins_encode %{
17035 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17036 if (tpc == nullptr) {
17037 ciEnv::current()->record_failure("CodeCache is full");
17038 return;
17039 }
17040 %}
17041 ins_pipe( pipe_slow );
17042 %}
17043
17044 // fast char[] to byte[] compression
17045 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17046 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17047 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17048 iRegI_R0 result, rFlagsReg cr)
17049 %{
17050 match(Set result (StrCompressedCopy src (Binary dst len)));
17051 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17052 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17053
17054 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17055 ins_encode %{
17056 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17057 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17058 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17059 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17060 %}
17061 ins_pipe(pipe_slow);
17062 %}
17063
17064 // fast byte[] to char[] inflation
17065 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17066 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17067 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17068 %{
17069 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17070 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17071 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17072 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17073
17074 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17075 ins_encode %{
17076 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17077 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17078 $vtmp2$$FloatRegister, $tmp$$Register);
17079 if (tpc == nullptr) {
17080 ciEnv::current()->record_failure("CodeCache is full");
17081 return;
17082 }
17083 %}
17084 ins_pipe(pipe_class_memory);
17085 %}
17086
17087 // encode char[] to byte[] in ISO_8859_1
17088 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17089 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17090 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17091 iRegI_R0 result, rFlagsReg cr)
17092 %{
17093 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17094 match(Set result (EncodeISOArray src (Binary dst len)));
17095 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17096 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17097
17098 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17099 ins_encode %{
17100 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17101 $result$$Register, false,
17102 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17103 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17104 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17105 %}
17106 ins_pipe(pipe_class_memory);
17107 %}
17108
17109 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17110 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17111 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17112 iRegI_R0 result, rFlagsReg cr)
17113 %{
17114 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17115 match(Set result (EncodeISOArray src (Binary dst len)));
17116 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17117 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17118
17119 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17120 ins_encode %{
17121 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17122 $result$$Register, true,
17123 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17124 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17125 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17126 %}
17127 ins_pipe(pipe_class_memory);
17128 %}
17129
17130 //----------------------------- CompressBits/ExpandBits ------------------------
17131
17132 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17133 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17134 match(Set dst (CompressBits src mask));
17135 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17136 format %{ "mov $tsrc, $src\n\t"
17137 "mov $tmask, $mask\n\t"
17138 "bext $tdst, $tsrc, $tmask\n\t"
17139 "mov $dst, $tdst"
17140 %}
17141 ins_encode %{
17142 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17143 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17144 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17145 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17146 %}
17147 ins_pipe(pipe_slow);
17148 %}
17149
17150 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17151 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17152 match(Set dst (CompressBits (LoadI mem) mask));
17153 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17154 format %{ "ldrs $tsrc, $mem\n\t"
17155 "ldrs $tmask, $mask\n\t"
17156 "bext $tdst, $tsrc, $tmask\n\t"
17157 "mov $dst, $tdst"
17158 %}
17159 ins_encode %{
17160 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17161 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17162 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17163 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17164 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17165 %}
17166 ins_pipe(pipe_slow);
17167 %}
17168
17169 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17170 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17171 match(Set dst (CompressBits src mask));
17172 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17173 format %{ "mov $tsrc, $src\n\t"
17174 "mov $tmask, $mask\n\t"
17175 "bext $tdst, $tsrc, $tmask\n\t"
17176 "mov $dst, $tdst"
17177 %}
17178 ins_encode %{
17179 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17180 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17181 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17182 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17183 %}
17184 ins_pipe(pipe_slow);
17185 %}
17186
17187 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17188 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17189 match(Set dst (CompressBits (LoadL mem) mask));
17190 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17191 format %{ "ldrd $tsrc, $mem\n\t"
17192 "ldrd $tmask, $mask\n\t"
17193 "bext $tdst, $tsrc, $tmask\n\t"
17194 "mov $dst, $tdst"
17195 %}
17196 ins_encode %{
17197 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17198 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17199 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17200 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17201 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17202 %}
17203 ins_pipe(pipe_slow);
17204 %}
17205
17206 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17207 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17208 match(Set dst (ExpandBits src mask));
17209 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17210 format %{ "mov $tsrc, $src\n\t"
17211 "mov $tmask, $mask\n\t"
17212 "bdep $tdst, $tsrc, $tmask\n\t"
17213 "mov $dst, $tdst"
17214 %}
17215 ins_encode %{
17216 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17217 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17218 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17219 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17220 %}
17221 ins_pipe(pipe_slow);
17222 %}
17223
17224 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17225 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17226 match(Set dst (ExpandBits (LoadI mem) mask));
17227 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17228 format %{ "ldrs $tsrc, $mem\n\t"
17229 "ldrs $tmask, $mask\n\t"
17230 "bdep $tdst, $tsrc, $tmask\n\t"
17231 "mov $dst, $tdst"
17232 %}
17233 ins_encode %{
17234 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17235 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17236 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17237 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17238 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17239 %}
17240 ins_pipe(pipe_slow);
17241 %}
17242
17243 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17244 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17245 match(Set dst (ExpandBits src mask));
17246 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17247 format %{ "mov $tsrc, $src\n\t"
17248 "mov $tmask, $mask\n\t"
17249 "bdep $tdst, $tsrc, $tmask\n\t"
17250 "mov $dst, $tdst"
17251 %}
17252 ins_encode %{
17253 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17254 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17255 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17256 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17257 %}
17258 ins_pipe(pipe_slow);
17259 %}
17260
17261
17262 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17263 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17264 match(Set dst (ExpandBits (LoadL mem) mask));
17265 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17266 format %{ "ldrd $tsrc, $mem\n\t"
17267 "ldrd $tmask, $mask\n\t"
17268 "bdep $tdst, $tsrc, $tmask\n\t"
17269 "mov $dst, $tdst"
17270 %}
17271 ins_encode %{
17272 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17273 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17274 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17275 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17276 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17277 %}
17278 ins_pipe(pipe_slow);
17279 %}
17280
17281 //----------------------------- Reinterpret ----------------------------------
17282 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17283 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17284 match(Set dst (ReinterpretHF2S src));
17285 format %{ "reinterpretHF2S $dst, $src" %}
17286 ins_encode %{
17287 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17288 %}
17289 ins_pipe(pipe_slow);
17290 %}
17291
17292 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17293 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17294 match(Set dst (ReinterpretS2HF src));
17295 format %{ "reinterpretS2HF $dst, $src" %}
17296 ins_encode %{
17297 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17298 %}
17299 ins_pipe(pipe_slow);
17300 %}
17301
17302 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17303 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17304 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17305 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17306 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17307 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17308 // can be omitted in this pattern, resulting in -
17309 // fcvt $dst, $src // Convert float to half-precision float
17310 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17311 %{
17312 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17313 format %{ "convF2HFAndS2HF $dst, $src" %}
17314 ins_encode %{
17315 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17316 %}
17317 ins_pipe(pipe_slow);
17318 %}
17319
17320 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17321 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17322 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17323 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17324 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17325 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17326 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17327 // resulting in -
17328 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17329 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17330 %{
17331 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17332 format %{ "convHF2SAndHF2F $dst, $src" %}
17333 ins_encode %{
17334 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17335 %}
17336 ins_pipe(pipe_slow);
17337 %}
17338
17339 // ============================================================================
17340 // This name is KNOWN by the ADLC and cannot be changed.
17341 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17342 // for this guy.
17343 instruct tlsLoadP(thread_RegP dst)
17344 %{
17345 match(Set dst (ThreadLocal));
17346
17347 ins_cost(0);
17348
17349 format %{ " -- \t// $dst=Thread::current(), empty" %}
17350
17351 size(0);
17352
17353 ins_encode( /*empty*/ );
17354
17355 ins_pipe(pipe_class_empty);
17356 %}
17357
17358 //----------PEEPHOLE RULES-----------------------------------------------------
17359 // These must follow all instruction definitions as they use the names
17360 // defined in the instructions definitions.
17361 //
17362 // peepmatch ( root_instr_name [preceding_instruction]* );
17363 //
17364 // peepconstraint %{
17365 // (instruction_number.operand_name relational_op instruction_number.operand_name
17366 // [, ...] );
17367 // // instruction numbers are zero-based using left to right order in peepmatch
17368 //
17369 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17370 // // provide an instruction_number.operand_name for each operand that appears
17371 // // in the replacement instruction's match rule
17372 //
17373 // ---------VM FLAGS---------------------------------------------------------
17374 //
17375 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17376 //
17377 // Each peephole rule is given an identifying number starting with zero and
17378 // increasing by one in the order seen by the parser. An individual peephole
17379 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17380 // on the command-line.
17381 //
17382 // ---------CURRENT LIMITATIONS----------------------------------------------
17383 //
17384 // Only match adjacent instructions in same basic block
17385 // Only equality constraints
17386 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17387 // Only one replacement instruction
17388 //
17389 // ---------EXAMPLE----------------------------------------------------------
17390 //
17391 // // pertinent parts of existing instructions in architecture description
17392 // instruct movI(iRegINoSp dst, iRegI src)
17393 // %{
17394 // match(Set dst (CopyI src));
17395 // %}
17396 //
17397 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17398 // %{
17399 // match(Set dst (AddI dst src));
17400 // effect(KILL cr);
17401 // %}
17402 //
17403 // // Change (inc mov) to lea
17404 // peephole %{
17405 // // increment preceded by register-register move
17406 // peepmatch ( incI_iReg movI );
17407 // // require that the destination register of the increment
17408 // // match the destination register of the move
17409 // peepconstraint ( 0.dst == 1.dst );
17410 // // construct a replacement instruction that sets
17411 // // the destination to ( move's source register + one )
17412 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17413 // %}
17414 //
17415
17416 // Implementation no longer uses movX instructions since
17417 // machine-independent system no longer uses CopyX nodes.
17418 //
17419 // peephole
17420 // %{
17421 // peepmatch (incI_iReg movI);
17422 // peepconstraint (0.dst == 1.dst);
17423 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17424 // %}
17425
17426 // peephole
17427 // %{
17428 // peepmatch (decI_iReg movI);
17429 // peepconstraint (0.dst == 1.dst);
17430 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17431 // %}
17432
17433 // peephole
17434 // %{
17435 // peepmatch (addI_iReg_imm movI);
17436 // peepconstraint (0.dst == 1.dst);
17437 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17438 // %}
17439
17440 // peephole
17441 // %{
17442 // peepmatch (incL_iReg movL);
17443 // peepconstraint (0.dst == 1.dst);
17444 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17445 // %}
17446
17447 // peephole
17448 // %{
17449 // peepmatch (decL_iReg movL);
17450 // peepconstraint (0.dst == 1.dst);
17451 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17452 // %}
17453
17454 // peephole
17455 // %{
17456 // peepmatch (addL_iReg_imm movL);
17457 // peepconstraint (0.dst == 1.dst);
17458 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17459 // %}
17460
17461 // peephole
17462 // %{
17463 // peepmatch (addP_iReg_imm movP);
17464 // peepconstraint (0.dst == 1.dst);
17465 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17466 // %}
17467
17468 // // Change load of spilled value to only a spill
17469 // instruct storeI(memory mem, iRegI src)
17470 // %{
17471 // match(Set mem (StoreI mem src));
17472 // %}
17473 //
17474 // instruct loadI(iRegINoSp dst, memory mem)
17475 // %{
17476 // match(Set dst (LoadI mem));
17477 // %}
17478 //
17479
17480 //----------SMARTSPILL RULES---------------------------------------------------
17481 // These must follow all instruction definitions as they use the names
17482 // defined in the instructions definitions.
17483
17484 // Local Variables:
17485 // mode: c++
17486 // End: