1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 bool Matcher::supports_vector_calling_convention(void) {
2311 return EnableVectorSupport && UseVectorStubs;
2312 }
2313
2314 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2315 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2316 int lo = V0_num;
2317 int hi = V0_H_num;
2318 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2319 hi = V0_K_num;
2320 }
2321 return OptoRegPair(hi, lo);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2636 return true;
2637 }
2638
2639 Node *off = m->in(AddPNode::Offset);
2640 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2641 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2642 // Are there other uses besides address expressions?
2643 !is_visited(off)) {
2644 address_visited.set(off->_idx); // Flag as address_visited
2645 mstack.push(off->in(2), Visit);
2646 Node *conv = off->in(1);
2647 if (conv->Opcode() == Op_ConvI2L &&
2648 // Are there other uses besides address expressions?
2649 !is_visited(conv)) {
2650 address_visited.set(conv->_idx); // Flag as address_visited
2651 mstack.push(conv->in(1), Pre_Visit);
2652 } else {
2653 mstack.push(conv, Pre_Visit);
2654 }
2655 address_visited.test_set(m->_idx); // Flag as address_visited
2656 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2657 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2658 return true;
2659 } else if (off->Opcode() == Op_ConvI2L &&
2660 // Are there other uses besides address expressions?
2661 !is_visited(off)) {
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 address_visited.set(off->_idx); // Flag as address_visited
2664 mstack.push(off->in(1), Pre_Visit);
2665 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2666 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2667 return true;
2668 }
2669 return false;
2670 }
2671
2672 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2673 { \
2674 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2675 guarantee(DISP == 0, "mode not permitted for volatile"); \
2676 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2677 __ INSN(REG, as_Register(BASE)); \
2678 }
2679
2680
2681 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2682 {
2683 Address::extend scale;
2684
2685 // Hooboy, this is fugly. We need a way to communicate to the
2686 // encoder that the index needs to be sign extended, so we have to
2687 // enumerate all the cases.
2688 switch (opcode) {
2689 case INDINDEXSCALEDI2L:
2690 case INDINDEXSCALEDI2LN:
2691 case INDINDEXI2L:
2692 case INDINDEXI2LN:
2693 scale = Address::sxtw(size);
2694 break;
2695 default:
2696 scale = Address::lsl(size);
2697 }
2698
2699 if (index == -1) {
2700 return Address(base, disp);
2701 } else {
2702 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2703 return Address(base, as_Register(index), scale);
2704 }
2705 }
2706
2707
2708 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2709 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2710 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2711 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2712 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2713
2714 // Used for all non-volatile memory accesses. The use of
2715 // $mem->opcode() to discover whether this pattern uses sign-extended
2716 // offsets is something of a kludge.
2717 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2718 Register reg, int opcode,
2719 Register base, int index, int scale, int disp,
2720 int size_in_memory)
2721 {
2722 Address addr = mem2address(opcode, base, index, scale, disp);
2723 if (addr.getMode() == Address::base_plus_offset) {
2724 /* Fix up any out-of-range offsets. */
2725 assert_different_registers(rscratch1, base);
2726 assert_different_registers(rscratch1, reg);
2727 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2728 }
2729 (masm->*insn)(reg, addr);
2730 }
2731
2732 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2733 FloatRegister reg, int opcode,
2734 Register base, int index, int size, int disp,
2735 int size_in_memory)
2736 {
2737 Address::extend scale;
2738
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 scale = Address::sxtw(size);
2743 break;
2744 default:
2745 scale = Address::lsl(size);
2746 }
2747
2748 if (index == -1) {
2749 // Fix up any out-of-range offsets.
2750 assert_different_registers(rscratch1, base);
2751 Address addr = Address(base, disp);
2752 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2753 (masm->*insn)(reg, addr);
2754 } else {
2755 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2756 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2757 }
2758 }
2759
2760 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2761 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2762 int opcode, Register base, int index, int size, int disp)
2763 {
2764 if (index == -1) {
2765 (masm->*insn)(reg, T, Address(base, disp));
2766 } else {
2767 assert(disp == 0, "unsupported address mode");
2768 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2769 }
2770 }
2771
2772 %}
2773
2774
2775
2776 //----------ENCODING BLOCK-----------------------------------------------------
2777 // This block specifies the encoding classes used by the compiler to
2778 // output byte streams. Encoding classes are parameterized macros
2779 // used by Machine Instruction Nodes in order to generate the bit
2780 // encoding of the instruction. Operands specify their base encoding
2781 // interface with the interface keyword. There are currently
2782 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2783 // COND_INTER. REG_INTER causes an operand to generate a function
2784 // which returns its register number when queried. CONST_INTER causes
2785 // an operand to generate a function which returns the value of the
2786 // constant when queried. MEMORY_INTER causes an operand to generate
2787 // four functions which return the Base Register, the Index Register,
2788 // the Scale Value, and the Offset Value of the operand when queried.
2789 // COND_INTER causes an operand to generate six functions which return
2790 // the encoding code (ie - encoding bits for the instruction)
2791 // associated with each basic boolean condition for a conditional
2792 // instruction.
2793 //
2794 // Instructions specify two basic values for encoding. Again, a
2795 // function is available to check if the constant displacement is an
2796 // oop. They use the ins_encode keyword to specify their encoding
2797 // classes (which must be a sequence of enc_class names, and their
2798 // parameters, specified in the encoding block), and they use the
2799 // opcode keyword to specify, in order, their primary, secondary, and
2800 // tertiary opcode. Only the opcode sections which a particular
2801 // instruction needs for encoding need to be specified.
2802 encode %{
2803 // Build emit functions for each basic byte or larger field in the
2804 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2805 // from C++ code in the enc_class source block. Emit functions will
2806 // live in the main source block for now. In future, we can
2807 // generalize this by adding a syntax that specifies the sizes of
2808 // fields in an order, so that the adlc can build the emit functions
2809 // automagically
2810
2811 // catch all for unimplemented encodings
2812 enc_class enc_unimplemented %{
2813 __ unimplemented("C2 catch all");
2814 %}
2815
2816 // BEGIN Non-volatile memory access
2817
2818 // This encoding class is generated automatically from ad_encode.m4.
2819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2820 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2821 Register dst_reg = as_Register($dst$$reg);
2822 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2823 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2824 %}
2825
2826 // This encoding class is generated automatically from ad_encode.m4.
2827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2828 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2829 Register dst_reg = as_Register($dst$$reg);
2830 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2831 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2832 %}
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2917 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2925 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2933 Register src_reg = as_Register($src$$reg);
2934 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_strb0(memory1 mem) %{
2941 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2948 Register src_reg = as_Register($src$$reg);
2949 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strh0(memory2 mem) %{
2956 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2963 Register src_reg = as_Register($src$$reg);
2964 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_strw0(memory4 mem) %{
2971 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2978 Register src_reg = as_Register($src$$reg);
2979 // we sometimes get asked to store the stack pointer into the
2980 // current thread -- we cannot do that directly on AArch64
2981 if (src_reg == r31_sp) {
2982 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2983 __ mov(rscratch2, sp);
2984 src_reg = rscratch2;
2985 }
2986 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_str0(memory8 mem) %{
2993 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3000 FloatRegister src_reg = as_FloatRegister($src$$reg);
3001 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3008 FloatRegister src_reg = as_FloatRegister($src$$reg);
3009 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3016 __ membar(Assembler::StoreStore);
3017 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3019 %}
3020
3021 // END Non-volatile memory access
3022
3023 // Vector loads and stores
3024 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3025 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3026 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3031 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3032 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3037 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3038 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3039 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3040 %}
3041
3042 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3043 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3044 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3045 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3046 %}
3047
3048 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3049 FloatRegister src_reg = as_FloatRegister($src$$reg);
3050 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3051 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3052 %}
3053
3054 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3055 FloatRegister src_reg = as_FloatRegister($src$$reg);
3056 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3057 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3058 %}
3059
3060 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3061 FloatRegister src_reg = as_FloatRegister($src$$reg);
3062 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3063 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3064 %}
3065
3066 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3069 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3070 %}
3071
3072 // volatile loads and stores
3073
3074 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3075 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3076 rscratch1, stlrb);
3077 %}
3078
3079 enc_class aarch64_enc_stlrb0(memory mem) %{
3080 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3081 rscratch1, stlrb);
3082 %}
3083
3084 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3085 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3086 rscratch1, stlrh);
3087 %}
3088
3089 enc_class aarch64_enc_stlrh0(memory mem) %{
3090 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3091 rscratch1, stlrh);
3092 %}
3093
3094 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3095 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3096 rscratch1, stlrw);
3097 %}
3098
3099 enc_class aarch64_enc_stlrw0(memory mem) %{
3100 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, stlrw);
3102 %}
3103
3104 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3105 Register dst_reg = as_Register($dst$$reg);
3106 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3107 rscratch1, ldarb);
3108 __ sxtbw(dst_reg, dst_reg);
3109 %}
3110
3111 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3112 Register dst_reg = as_Register($dst$$reg);
3113 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3114 rscratch1, ldarb);
3115 __ sxtb(dst_reg, dst_reg);
3116 %}
3117
3118 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3119 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, ldarb);
3121 %}
3122
3123 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3124 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, ldarb);
3126 %}
3127
3128 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarh);
3132 __ sxthw(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3136 Register dst_reg = as_Register($dst$$reg);
3137 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, ldarh);
3139 __ sxth(dst_reg, dst_reg);
3140 %}
3141
3142 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3143 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, ldarh);
3145 %}
3146
3147 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarh);
3150 %}
3151
3152 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarw);
3155 %}
3156
3157 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3158 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, ldarw);
3160 %}
3161
3162 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3163 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldar);
3165 %}
3166
3167 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3168 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarw);
3170 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3171 %}
3172
3173 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3174 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldar);
3176 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3177 %}
3178
3179 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3180 Register src_reg = as_Register($src$$reg);
3181 // we sometimes get asked to store the stack pointer into the
3182 // current thread -- we cannot do that directly on AArch64
3183 if (src_reg == r31_sp) {
3184 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3185 __ mov(rscratch2, sp);
3186 src_reg = rscratch2;
3187 }
3188 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, stlr);
3190 %}
3191
3192 enc_class aarch64_enc_stlr0(memory mem) %{
3193 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, stlr);
3195 %}
3196
3197 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3198 {
3199 FloatRegister src_reg = as_FloatRegister($src$$reg);
3200 __ fmovs(rscratch2, src_reg);
3201 }
3202 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, stlrw);
3204 %}
3205
3206 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3207 {
3208 FloatRegister src_reg = as_FloatRegister($src$$reg);
3209 __ fmovd(rscratch2, src_reg);
3210 }
3211 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 // synchronized read/update encodings
3216
3217 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3218 Register dst_reg = as_Register($dst$$reg);
3219 Register base = as_Register($mem$$base);
3220 int index = $mem$$index;
3221 int scale = $mem$$scale;
3222 int disp = $mem$$disp;
3223 if (index == -1) {
3224 if (disp != 0) {
3225 __ lea(rscratch1, Address(base, disp));
3226 __ ldaxr(dst_reg, rscratch1);
3227 } else {
3228 // TODO
3229 // should we ever get anything other than this case?
3230 __ ldaxr(dst_reg, base);
3231 }
3232 } else {
3233 Register index_reg = as_Register(index);
3234 if (disp == 0) {
3235 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3236 __ ldaxr(dst_reg, rscratch1);
3237 } else {
3238 __ lea(rscratch1, Address(base, disp));
3239 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3240 __ ldaxr(dst_reg, rscratch1);
3241 }
3242 }
3243 %}
3244
3245 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3246 Register src_reg = as_Register($src$$reg);
3247 Register base = as_Register($mem$$base);
3248 int index = $mem$$index;
3249 int scale = $mem$$scale;
3250 int disp = $mem$$disp;
3251 if (index == -1) {
3252 if (disp != 0) {
3253 __ lea(rscratch2, Address(base, disp));
3254 __ stlxr(rscratch1, src_reg, rscratch2);
3255 } else {
3256 // TODO
3257 // should we ever get anything other than this case?
3258 __ stlxr(rscratch1, src_reg, base);
3259 }
3260 } else {
3261 Register index_reg = as_Register(index);
3262 if (disp == 0) {
3263 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3264 __ stlxr(rscratch1, src_reg, rscratch2);
3265 } else {
3266 __ lea(rscratch2, Address(base, disp));
3267 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3268 __ stlxr(rscratch1, src_reg, rscratch2);
3269 }
3270 }
3271 __ cmpw(rscratch1, zr);
3272 %}
3273
3274 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3275 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3276 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3277 Assembler::xword, /*acquire*/ false, /*release*/ true,
3278 /*weak*/ false, noreg);
3279 %}
3280
3281 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3282 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3283 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3284 Assembler::word, /*acquire*/ false, /*release*/ true,
3285 /*weak*/ false, noreg);
3286 %}
3287
3288 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3289 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3290 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3291 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3292 /*weak*/ false, noreg);
3293 %}
3294
3295 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::byte, /*acquire*/ false, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302
3303 // The only difference between aarch64_enc_cmpxchg and
3304 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3305 // CompareAndSwap sequence to serve as a barrier on acquiring a
3306 // lock.
3307 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3308 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3309 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3310 Assembler::xword, /*acquire*/ true, /*release*/ true,
3311 /*weak*/ false, noreg);
3312 %}
3313
3314 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3315 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3316 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3317 Assembler::word, /*acquire*/ true, /*release*/ true,
3318 /*weak*/ false, noreg);
3319 %}
3320
3321 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3322 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3323 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3324 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3325 /*weak*/ false, noreg);
3326 %}
3327
3328 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3329 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3330 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3331 Assembler::byte, /*acquire*/ true, /*release*/ true,
3332 /*weak*/ false, noreg);
3333 %}
3334
3335 // auxiliary used for CompareAndSwapX to set result register
3336 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3337 Register res_reg = as_Register($res$$reg);
3338 __ cset(res_reg, Assembler::EQ);
3339 %}
3340
3341 // prefetch encodings
3342
3343 enc_class aarch64_enc_prefetchw(memory mem) %{
3344 Register base = as_Register($mem$$base);
3345 int index = $mem$$index;
3346 int scale = $mem$$scale;
3347 int disp = $mem$$disp;
3348 if (index == -1) {
3349 // Fix up any out-of-range offsets.
3350 assert_different_registers(rscratch1, base);
3351 Address addr = Address(base, disp);
3352 addr = __ legitimize_address(addr, 8, rscratch1);
3353 __ prfm(addr, PSTL1KEEP);
3354 } else {
3355 Register index_reg = as_Register(index);
3356 if (disp == 0) {
3357 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3358 } else {
3359 __ lea(rscratch1, Address(base, disp));
3360 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3361 }
3362 }
3363 %}
3364
3365 // mov encodings
3366
3367 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3368 uint32_t con = (uint32_t)$src$$constant;
3369 Register dst_reg = as_Register($dst$$reg);
3370 if (con == 0) {
3371 __ movw(dst_reg, zr);
3372 } else {
3373 __ movw(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 uint64_t con = (uint64_t)$src$$constant;
3380 if (con == 0) {
3381 __ mov(dst_reg, zr);
3382 } else {
3383 __ mov(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 address con = (address)$src$$constant;
3390 if (con == nullptr || con == (address)1) {
3391 ShouldNotReachHere();
3392 } else {
3393 relocInfo::relocType rtype = $src->constant_reloc();
3394 if (rtype == relocInfo::oop_type) {
3395 __ movoop(dst_reg, (jobject)con);
3396 } else if (rtype == relocInfo::metadata_type) {
3397 __ mov_metadata(dst_reg, (Metadata*)con);
3398 } else {
3399 assert(rtype == relocInfo::none, "unexpected reloc type");
3400 if (! __ is_valid_AArch64_address(con) ||
3401 con < (address)(uintptr_t)os::vm_page_size()) {
3402 __ mov(dst_reg, con);
3403 } else {
3404 uint64_t offset;
3405 __ adrp(dst_reg, con, offset);
3406 __ add(dst_reg, dst_reg, offset);
3407 }
3408 }
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 __ mov(dst_reg, zr);
3415 %}
3416
3417 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, (uint64_t)1);
3420 %}
3421
3422 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3423 __ load_byte_map_base($dst$$Register);
3424 %}
3425
3426 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 address con = (address)$src$$constant;
3429 if (con == nullptr) {
3430 ShouldNotReachHere();
3431 } else {
3432 relocInfo::relocType rtype = $src->constant_reloc();
3433 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3434 __ set_narrow_oop(dst_reg, (jobject)con);
3435 }
3436 %}
3437
3438 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3439 Register dst_reg = as_Register($dst$$reg);
3440 __ mov(dst_reg, zr);
3441 %}
3442
3443 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 address con = (address)$src$$constant;
3446 if (con == nullptr) {
3447 ShouldNotReachHere();
3448 } else {
3449 relocInfo::relocType rtype = $src->constant_reloc();
3450 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3451 __ set_narrow_klass(dst_reg, (Klass *)con);
3452 }
3453 %}
3454
3455 // arithmetic encodings
3456
3457 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3458 Register dst_reg = as_Register($dst$$reg);
3459 Register src_reg = as_Register($src1$$reg);
3460 int32_t con = (int32_t)$src2$$constant;
3461 // add has primary == 0, subtract has primary == 1
3462 if ($primary) { con = -con; }
3463 if (con < 0) {
3464 __ subw(dst_reg, src_reg, -con);
3465 } else {
3466 __ addw(dst_reg, src_reg, con);
3467 }
3468 %}
3469
3470 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3471 Register dst_reg = as_Register($dst$$reg);
3472 Register src_reg = as_Register($src1$$reg);
3473 int32_t con = (int32_t)$src2$$constant;
3474 // add has primary == 0, subtract has primary == 1
3475 if ($primary) { con = -con; }
3476 if (con < 0) {
3477 __ sub(dst_reg, src_reg, -con);
3478 } else {
3479 __ add(dst_reg, src_reg, con);
3480 }
3481 %}
3482
3483 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src1_reg = as_Register($src1$$reg);
3486 Register src2_reg = as_Register($src2$$reg);
3487 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3488 %}
3489
3490 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3491 Register dst_reg = as_Register($dst$$reg);
3492 Register src1_reg = as_Register($src1$$reg);
3493 Register src2_reg = as_Register($src2$$reg);
3494 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3495 %}
3496
3497 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3498 Register dst_reg = as_Register($dst$$reg);
3499 Register src1_reg = as_Register($src1$$reg);
3500 Register src2_reg = as_Register($src2$$reg);
3501 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3502 %}
3503
3504 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3505 Register dst_reg = as_Register($dst$$reg);
3506 Register src1_reg = as_Register($src1$$reg);
3507 Register src2_reg = as_Register($src2$$reg);
3508 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3509 %}
3510
3511 // compare instruction encodings
3512
3513 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3514 Register reg1 = as_Register($src1$$reg);
3515 Register reg2 = as_Register($src2$$reg);
3516 __ cmpw(reg1, reg2);
3517 %}
3518
3519 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3520 Register reg = as_Register($src1$$reg);
3521 int32_t val = $src2$$constant;
3522 if (val >= 0) {
3523 __ subsw(zr, reg, val);
3524 } else {
3525 __ addsw(zr, reg, -val);
3526 }
3527 %}
3528
3529 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3530 Register reg1 = as_Register($src1$$reg);
3531 uint32_t val = (uint32_t)$src2$$constant;
3532 __ movw(rscratch1, val);
3533 __ cmpw(reg1, rscratch1);
3534 %}
3535
3536 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3537 Register reg1 = as_Register($src1$$reg);
3538 Register reg2 = as_Register($src2$$reg);
3539 __ cmp(reg1, reg2);
3540 %}
3541
3542 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3543 Register reg = as_Register($src1$$reg);
3544 int64_t val = $src2$$constant;
3545 if (val >= 0) {
3546 __ subs(zr, reg, val);
3547 } else if (val != -val) {
3548 __ adds(zr, reg, -val);
3549 } else {
3550 // aargh, Long.MIN_VALUE is a special case
3551 __ orr(rscratch1, zr, (uint64_t)val);
3552 __ subs(zr, reg, rscratch1);
3553 }
3554 %}
3555
3556 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3557 Register reg1 = as_Register($src1$$reg);
3558 uint64_t val = (uint64_t)$src2$$constant;
3559 __ mov(rscratch1, val);
3560 __ cmp(reg1, rscratch1);
3561 %}
3562
3563 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3564 Register reg1 = as_Register($src1$$reg);
3565 Register reg2 = as_Register($src2$$reg);
3566 __ cmp(reg1, reg2);
3567 %}
3568
3569 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3570 Register reg1 = as_Register($src1$$reg);
3571 Register reg2 = as_Register($src2$$reg);
3572 __ cmpw(reg1, reg2);
3573 %}
3574
3575 enc_class aarch64_enc_testp(iRegP src) %{
3576 Register reg = as_Register($src$$reg);
3577 __ cmp(reg, zr);
3578 %}
3579
3580 enc_class aarch64_enc_testn(iRegN src) %{
3581 Register reg = as_Register($src$$reg);
3582 __ cmpw(reg, zr);
3583 %}
3584
3585 enc_class aarch64_enc_b(label lbl) %{
3586 Label *L = $lbl$$label;
3587 __ b(*L);
3588 %}
3589
3590 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3591 Label *L = $lbl$$label;
3592 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3593 %}
3594
3595 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3596 Label *L = $lbl$$label;
3597 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3598 %}
3599
3600 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3601 %{
3602 Register sub_reg = as_Register($sub$$reg);
3603 Register super_reg = as_Register($super$$reg);
3604 Register temp_reg = as_Register($temp$$reg);
3605 Register result_reg = as_Register($result$$reg);
3606
3607 Label miss;
3608 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3609 nullptr, &miss,
3610 /*set_cond_codes:*/ true);
3611 if ($primary) {
3612 __ mov(result_reg, zr);
3613 }
3614 __ bind(miss);
3615 %}
3616
3617 enc_class aarch64_enc_java_static_call(method meth) %{
3618 address addr = (address)$meth$$method;
3619 address call;
3620 if (!_method) {
3621 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3622 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3623 if (call == nullptr) {
3624 ciEnv::current()->record_failure("CodeCache is full");
3625 return;
3626 }
3627 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3628 // The NOP here is purely to ensure that eliding a call to
3629 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3630 __ nop();
3631 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3632 } else {
3633 int method_index = resolved_method_index(masm);
3634 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3635 : static_call_Relocation::spec(method_index);
3636 call = __ trampoline_call(Address(addr, rspec));
3637 if (call == nullptr) {
3638 ciEnv::current()->record_failure("CodeCache is full");
3639 return;
3640 }
3641 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3642 // Calls of the same statically bound method can share
3643 // a stub to the interpreter.
3644 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3645 } else {
3646 // Emit stub for static call
3647 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3648 if (stub == nullptr) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 }
3653 }
3654
3655 __ post_call_nop();
3656
3657 // Only non uncommon_trap calls need to reinitialize ptrue.
3658 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3659 __ reinitialize_ptrue();
3660 }
3661 %}
3662
3663 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3664 int method_index = resolved_method_index(masm);
3665 address call = __ ic_call((address)$meth$$method, method_index);
3666 if (call == nullptr) {
3667 ciEnv::current()->record_failure("CodeCache is full");
3668 return;
3669 }
3670 __ post_call_nop();
3671 if (Compile::current()->max_vector_size() > 0) {
3672 __ reinitialize_ptrue();
3673 }
3674 %}
3675
3676 enc_class aarch64_enc_call_epilog() %{
3677 if (VerifyStackAtCalls) {
3678 // Check that stack depth is unchanged: find majik cookie on stack
3679 __ call_Unimplemented();
3680 }
3681 %}
3682
3683 enc_class aarch64_enc_java_to_runtime(method meth) %{
3684 // some calls to generated routines (arraycopy code) are scheduled
3685 // by C2 as runtime calls. if so we can call them using a br (they
3686 // will be in a reachable segment) otherwise we have to use a blr
3687 // which loads the absolute address into a register.
3688 address entry = (address)$meth$$method;
3689 CodeBlob *cb = CodeCache::find_blob(entry);
3690 if (cb) {
3691 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3692 if (call == nullptr) {
3693 ciEnv::current()->record_failure("CodeCache is full");
3694 return;
3695 }
3696 __ post_call_nop();
3697 } else {
3698 Label retaddr;
3699 // Make the anchor frame walkable
3700 __ adr(rscratch2, retaddr);
3701 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3702 __ lea(rscratch1, RuntimeAddress(entry));
3703 __ blr(rscratch1);
3704 __ bind(retaddr);
3705 __ post_call_nop();
3706 }
3707 if (Compile::current()->max_vector_size() > 0) {
3708 __ reinitialize_ptrue();
3709 }
3710 %}
3711
3712 enc_class aarch64_enc_rethrow() %{
3713 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3714 %}
3715
3716 enc_class aarch64_enc_ret() %{
3717 #ifdef ASSERT
3718 if (Compile::current()->max_vector_size() > 0) {
3719 __ verify_ptrue();
3720 }
3721 #endif
3722 __ ret(lr);
3723 %}
3724
3725 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3726 Register target_reg = as_Register($jump_target$$reg);
3727 __ br(target_reg);
3728 %}
3729
3730 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3731 Register target_reg = as_Register($jump_target$$reg);
3732 // exception oop should be in r0
3733 // ret addr has been popped into lr
3734 // callee expects it in r3
3735 __ mov(r3, lr);
3736 __ br(target_reg);
3737 %}
3738
3739 %}
3740
3741 //----------FRAME--------------------------------------------------------------
3742 // Definition of frame structure and management information.
3743 //
3744 // S T A C K L A Y O U T Allocators stack-slot number
3745 // | (to get allocators register number
3746 // G Owned by | | v add OptoReg::stack0())
3747 // r CALLER | |
3748 // o | +--------+ pad to even-align allocators stack-slot
3749 // w V | pad0 | numbers; owned by CALLER
3750 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3751 // h ^ | in | 5
3752 // | | args | 4 Holes in incoming args owned by SELF
3753 // | | | | 3
3754 // | | +--------+
3755 // V | | old out| Empty on Intel, window on Sparc
3756 // | old |preserve| Must be even aligned.
3757 // | SP-+--------+----> Matcher::_old_SP, even aligned
3758 // | | in | 3 area for Intel ret address
3759 // Owned by |preserve| Empty on Sparc.
3760 // SELF +--------+
3761 // | | pad2 | 2 pad to align old SP
3762 // | +--------+ 1
3763 // | | locks | 0
3764 // | +--------+----> OptoReg::stack0(), even aligned
3765 // | | pad1 | 11 pad to align new SP
3766 // | +--------+
3767 // | | | 10
3768 // | | spills | 9 spills
3769 // V | | 8 (pad0 slot for callee)
3770 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3771 // ^ | out | 7
3772 // | | args | 6 Holes in outgoing args owned by CALLEE
3773 // Owned by +--------+
3774 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3775 // | new |preserve| Must be even-aligned.
3776 // | SP-+--------+----> Matcher::_new_SP, even aligned
3777 // | | |
3778 //
3779 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3780 // known from SELF's arguments and the Java calling convention.
3781 // Region 6-7 is determined per call site.
3782 // Note 2: If the calling convention leaves holes in the incoming argument
3783 // area, those holes are owned by SELF. Holes in the outgoing area
3784 // are owned by the CALLEE. Holes should not be necessary in the
3785 // incoming area, as the Java calling convention is completely under
3786 // the control of the AD file. Doubles can be sorted and packed to
3787 // avoid holes. Holes in the outgoing arguments may be necessary for
3788 // varargs C calling conventions.
3789 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3790 // even aligned with pad0 as needed.
3791 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3792 // (the latter is true on Intel but is it false on AArch64?)
3793 // region 6-11 is even aligned; it may be padded out more so that
3794 // the region from SP to FP meets the minimum stack alignment.
3795 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3796 // alignment. Region 11, pad1, may be dynamically extended so that
3797 // SP meets the minimum alignment.
3798
3799 frame %{
3800 // These three registers define part of the calling convention
3801 // between compiled code and the interpreter.
3802
3803 // Inline Cache Register or Method for I2C.
3804 inline_cache_reg(R12);
3805
3806 // Number of stack slots consumed by locking an object
3807 sync_stack_slots(2);
3808
3809 // Compiled code's Frame Pointer
3810 frame_pointer(R31);
3811
3812 // Interpreter stores its frame pointer in a register which is
3813 // stored to the stack by I2CAdaptors.
3814 // I2CAdaptors convert from interpreted java to compiled java.
3815 interpreter_frame_pointer(R29);
3816
3817 // Stack alignment requirement
3818 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3819
3820 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3821 // for calls to C. Supports the var-args backing area for register parms.
3822 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3823
3824 // The after-PROLOG location of the return address. Location of
3825 // return address specifies a type (REG or STACK) and a number
3826 // representing the register number (i.e. - use a register name) or
3827 // stack slot.
3828 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3829 // Otherwise, it is above the locks and verification slot and alignment word
3830 // TODO this may well be correct but need to check why that - 2 is there
3831 // ppc port uses 0 but we definitely need to allow for fixed_slots
3832 // which folds in the space used for monitors
3833 return_addr(STACK - 2 +
3834 align_up((Compile::current()->in_preserve_stack_slots() +
3835 Compile::current()->fixed_slots()),
3836 stack_alignment_in_slots()));
3837
3838 // Location of compiled Java return values. Same as C for now.
3839 return_value
3840 %{
3841 // TODO do we allow ideal_reg == Op_RegN???
3842 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3843 "only return normal values");
3844
3845 static const int lo[Op_RegL + 1] = { // enum name
3846 0, // Op_Node
3847 0, // Op_Set
3848 R0_num, // Op_RegN
3849 R0_num, // Op_RegI
3850 R0_num, // Op_RegP
3851 V0_num, // Op_RegF
3852 V0_num, // Op_RegD
3853 R0_num // Op_RegL
3854 };
3855
3856 static const int hi[Op_RegL + 1] = { // enum name
3857 0, // Op_Node
3858 0, // Op_Set
3859 OptoReg::Bad, // Op_RegN
3860 OptoReg::Bad, // Op_RegI
3861 R0_H_num, // Op_RegP
3862 OptoReg::Bad, // Op_RegF
3863 V0_H_num, // Op_RegD
3864 R0_H_num // Op_RegL
3865 };
3866
3867 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3868 %}
3869 %}
3870
3871 //----------ATTRIBUTES---------------------------------------------------------
3872 //----------Operand Attributes-------------------------------------------------
3873 op_attrib op_cost(1); // Required cost attribute
3874
3875 //----------Instruction Attributes---------------------------------------------
3876 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3877 ins_attrib ins_size(32); // Required size attribute (in bits)
3878 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3879 // a non-matching short branch variant
3880 // of some long branch?
3881 ins_attrib ins_alignment(4); // Required alignment attribute (must
3882 // be a power of 2) specifies the
3883 // alignment that some part of the
3884 // instruction (not necessarily the
3885 // start) requires. If > 1, a
3886 // compute_padding() function must be
3887 // provided for the instruction
3888
3889 //----------OPERANDS-----------------------------------------------------------
3890 // Operand definitions must precede instruction definitions for correct parsing
3891 // in the ADLC because operands constitute user defined types which are used in
3892 // instruction definitions.
3893
3894 //----------Simple Operands----------------------------------------------------
3895
3896 // Integer operands 32 bit
3897 // 32 bit immediate
3898 operand immI()
3899 %{
3900 match(ConI);
3901
3902 op_cost(0);
3903 format %{ %}
3904 interface(CONST_INTER);
3905 %}
3906
3907 // 32 bit zero
3908 operand immI0()
3909 %{
3910 predicate(n->get_int() == 0);
3911 match(ConI);
3912
3913 op_cost(0);
3914 format %{ %}
3915 interface(CONST_INTER);
3916 %}
3917
3918 // 32 bit unit increment
3919 operand immI_1()
3920 %{
3921 predicate(n->get_int() == 1);
3922 match(ConI);
3923
3924 op_cost(0);
3925 format %{ %}
3926 interface(CONST_INTER);
3927 %}
3928
3929 // 32 bit unit decrement
3930 operand immI_M1()
3931 %{
3932 predicate(n->get_int() == -1);
3933 match(ConI);
3934
3935 op_cost(0);
3936 format %{ %}
3937 interface(CONST_INTER);
3938 %}
3939
3940 // Shift values for add/sub extension shift
3941 operand immIExt()
3942 %{
3943 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3944 match(ConI);
3945
3946 op_cost(0);
3947 format %{ %}
3948 interface(CONST_INTER);
3949 %}
3950
3951 operand immI_gt_1()
3952 %{
3953 predicate(n->get_int() > 1);
3954 match(ConI);
3955
3956 op_cost(0);
3957 format %{ %}
3958 interface(CONST_INTER);
3959 %}
3960
3961 operand immI_le_4()
3962 %{
3963 predicate(n->get_int() <= 4);
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 operand immI_16()
3972 %{
3973 predicate(n->get_int() == 16);
3974 match(ConI);
3975
3976 op_cost(0);
3977 format %{ %}
3978 interface(CONST_INTER);
3979 %}
3980
3981 operand immI_24()
3982 %{
3983 predicate(n->get_int() == 24);
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 operand immI_32()
3992 %{
3993 predicate(n->get_int() == 32);
3994 match(ConI);
3995
3996 op_cost(0);
3997 format %{ %}
3998 interface(CONST_INTER);
3999 %}
4000
4001 operand immI_48()
4002 %{
4003 predicate(n->get_int() == 48);
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_56()
4012 %{
4013 predicate(n->get_int() == 56);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_255()
4022 %{
4023 predicate(n->get_int() == 255);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_65535()
4032 %{
4033 predicate(n->get_int() == 65535);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_positive()
4042 %{
4043 predicate(n->get_int() > 0);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 // BoolTest condition for signed compare
4052 operand immI_cmp_cond()
4053 %{
4054 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4055 match(ConI);
4056
4057 op_cost(0);
4058 format %{ %}
4059 interface(CONST_INTER);
4060 %}
4061
4062 // BoolTest condition for unsigned compare
4063 operand immI_cmpU_cond()
4064 %{
4065 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4066 match(ConI);
4067
4068 op_cost(0);
4069 format %{ %}
4070 interface(CONST_INTER);
4071 %}
4072
4073 operand immL_255()
4074 %{
4075 predicate(n->get_long() == 255L);
4076 match(ConL);
4077
4078 op_cost(0);
4079 format %{ %}
4080 interface(CONST_INTER);
4081 %}
4082
4083 operand immL_65535()
4084 %{
4085 predicate(n->get_long() == 65535L);
4086 match(ConL);
4087
4088 op_cost(0);
4089 format %{ %}
4090 interface(CONST_INTER);
4091 %}
4092
4093 operand immL_4294967295()
4094 %{
4095 predicate(n->get_long() == 4294967295L);
4096 match(ConL);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 operand immL_bitmask()
4104 %{
4105 predicate((n->get_long() != 0)
4106 && ((n->get_long() & 0xc000000000000000l) == 0)
4107 && is_power_of_2(n->get_long() + 1));
4108 match(ConL);
4109
4110 op_cost(0);
4111 format %{ %}
4112 interface(CONST_INTER);
4113 %}
4114
4115 operand immI_bitmask()
4116 %{
4117 predicate((n->get_int() != 0)
4118 && ((n->get_int() & 0xc0000000) == 0)
4119 && is_power_of_2(n->get_int() + 1));
4120 match(ConI);
4121
4122 op_cost(0);
4123 format %{ %}
4124 interface(CONST_INTER);
4125 %}
4126
4127 operand immL_positive_bitmaskI()
4128 %{
4129 predicate((n->get_long() != 0)
4130 && ((julong)n->get_long() < 0x80000000ULL)
4131 && is_power_of_2(n->get_long() + 1));
4132 match(ConL);
4133
4134 op_cost(0);
4135 format %{ %}
4136 interface(CONST_INTER);
4137 %}
4138
4139 // Scale values for scaled offset addressing modes (up to long but not quad)
4140 operand immIScale()
4141 %{
4142 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4143 match(ConI);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 // 5 bit signed integer
4151 operand immI5()
4152 %{
4153 predicate(Assembler::is_simm(n->get_int(), 5));
4154 match(ConI);
4155
4156 op_cost(0);
4157 format %{ %}
4158 interface(CONST_INTER);
4159 %}
4160
4161 // 7 bit unsigned integer
4162 operand immIU7()
4163 %{
4164 predicate(Assembler::is_uimm(n->get_int(), 7));
4165 match(ConI);
4166
4167 op_cost(0);
4168 format %{ %}
4169 interface(CONST_INTER);
4170 %}
4171
4172 // Offset for scaled or unscaled immediate loads and stores
4173 operand immIOffset()
4174 %{
4175 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4176 match(ConI);
4177
4178 op_cost(0);
4179 format %{ %}
4180 interface(CONST_INTER);
4181 %}
4182
4183 operand immIOffset1()
4184 %{
4185 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4186 match(ConI);
4187
4188 op_cost(0);
4189 format %{ %}
4190 interface(CONST_INTER);
4191 %}
4192
4193 operand immIOffset2()
4194 %{
4195 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4196 match(ConI);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 operand immIOffset4()
4204 %{
4205 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4206 match(ConI);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 operand immIOffset8()
4214 %{
4215 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4216 match(ConI);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 operand immIOffset16()
4224 %{
4225 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4226 match(ConI);
4227
4228 op_cost(0);
4229 format %{ %}
4230 interface(CONST_INTER);
4231 %}
4232
4233 operand immLOffset()
4234 %{
4235 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4236 match(ConL);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immLoffset1()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4246 match(ConL);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immLoffset2()
4254 %{
4255 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4256 match(ConL);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immLoffset4()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4266 match(ConL);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immLoffset8()
4274 %{
4275 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4276 match(ConL);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immLoffset16()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4286 match(ConL);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 // 5 bit signed long integer
4294 operand immL5()
4295 %{
4296 predicate(Assembler::is_simm(n->get_long(), 5));
4297 match(ConL);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 // 7 bit unsigned long integer
4305 operand immLU7()
4306 %{
4307 predicate(Assembler::is_uimm(n->get_long(), 7));
4308 match(ConL);
4309
4310 op_cost(0);
4311 format %{ %}
4312 interface(CONST_INTER);
4313 %}
4314
4315 // 8 bit signed value.
4316 operand immI8()
4317 %{
4318 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4319 match(ConI);
4320
4321 op_cost(0);
4322 format %{ %}
4323 interface(CONST_INTER);
4324 %}
4325
4326 // 8 bit signed value (simm8), or #simm8 LSL 8.
4327 operand immI8_shift8()
4328 %{
4329 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4330 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4331 match(ConI);
4332
4333 op_cost(0);
4334 format %{ %}
4335 interface(CONST_INTER);
4336 %}
4337
4338 // 8 bit signed value (simm8), or #simm8 LSL 8.
4339 operand immL8_shift8()
4340 %{
4341 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4342 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 // 8 bit integer valid for vector add sub immediate
4351 operand immBAddSubV()
4352 %{
4353 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4354 match(ConI);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 // 32 bit integer valid for add sub immediate
4362 operand immIAddSub()
4363 %{
4364 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4365 match(ConI);
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 32 bit integer valid for vector add sub immediate
4372 operand immIAddSubV()
4373 %{
4374 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4375 match(ConI);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 32 bit unsigned integer valid for logical immediate
4383
4384 operand immBLog()
4385 %{
4386 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4387 match(ConI);
4388
4389 op_cost(0);
4390 format %{ %}
4391 interface(CONST_INTER);
4392 %}
4393
4394 operand immSLog()
4395 %{
4396 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 operand immILog()
4405 %{
4406 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 // Integer operands 64 bit
4415 // 64 bit immediate
4416 operand immL()
4417 %{
4418 match(ConL);
4419
4420 op_cost(0);
4421 format %{ %}
4422 interface(CONST_INTER);
4423 %}
4424
4425 // 64 bit zero
4426 operand immL0()
4427 %{
4428 predicate(n->get_long() == 0);
4429 match(ConL);
4430
4431 op_cost(0);
4432 format %{ %}
4433 interface(CONST_INTER);
4434 %}
4435
4436 // 64 bit unit decrement
4437 operand immL_M1()
4438 %{
4439 predicate(n->get_long() == -1);
4440 match(ConL);
4441
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 64 bit integer valid for add sub immediate
4448 operand immLAddSub()
4449 %{
4450 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4451 match(ConL);
4452 op_cost(0);
4453 format %{ %}
4454 interface(CONST_INTER);
4455 %}
4456
4457 // 64 bit integer valid for addv subv immediate
4458 operand immLAddSubV()
4459 %{
4460 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4461 match(ConL);
4462
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 // 64 bit integer valid for logical immediate
4469 operand immLLog()
4470 %{
4471 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4472 match(ConL);
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // Long Immediate: low 32-bit mask
4479 operand immL_32bits()
4480 %{
4481 predicate(n->get_long() == 0xFFFFFFFFL);
4482 match(ConL);
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // Pointer operands
4489 // Pointer Immediate
4490 operand immP()
4491 %{
4492 match(ConP);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // nullptr Pointer Immediate
4500 operand immP0()
4501 %{
4502 predicate(n->get_ptr() == 0);
4503 match(ConP);
4504
4505 op_cost(0);
4506 format %{ %}
4507 interface(CONST_INTER);
4508 %}
4509
4510 // Pointer Immediate One
4511 // this is used in object initialization (initial object header)
4512 operand immP_1()
4513 %{
4514 predicate(n->get_ptr() == 1);
4515 match(ConP);
4516
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 // Card Table Byte Map Base
4523 operand immByteMapBase()
4524 %{
4525 // Get base of card map
4526 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4527 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4528 match(ConP);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 // Float and Double operands
4536 // Double Immediate
4537 operand immD()
4538 %{
4539 match(ConD);
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // Double Immediate: +0.0d
4546 operand immD0()
4547 %{
4548 predicate(jlong_cast(n->getd()) == 0);
4549 match(ConD);
4550
4551 op_cost(0);
4552 format %{ %}
4553 interface(CONST_INTER);
4554 %}
4555
4556 // constant 'double +0.0'.
4557 operand immDPacked()
4558 %{
4559 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4560 match(ConD);
4561 op_cost(0);
4562 format %{ %}
4563 interface(CONST_INTER);
4564 %}
4565
4566 // Float Immediate
4567 operand immF()
4568 %{
4569 match(ConF);
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // Float Immediate: +0.0f.
4576 operand immF0()
4577 %{
4578 predicate(jint_cast(n->getf()) == 0);
4579 match(ConF);
4580
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 //
4587 operand immFPacked()
4588 %{
4589 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4590 match(ConF);
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // Narrow pointer operands
4597 // Narrow Pointer Immediate
4598 operand immN()
4599 %{
4600 match(ConN);
4601
4602 op_cost(0);
4603 format %{ %}
4604 interface(CONST_INTER);
4605 %}
4606
4607 // Narrow nullptr Pointer Immediate
4608 operand immN0()
4609 %{
4610 predicate(n->get_narrowcon() == 0);
4611 match(ConN);
4612
4613 op_cost(0);
4614 format %{ %}
4615 interface(CONST_INTER);
4616 %}
4617
4618 operand immNKlass()
4619 %{
4620 match(ConNKlass);
4621
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Integer 32 bit Register Operands
4628 // Integer 32 bitRegister (excludes SP)
4629 operand iRegI()
4630 %{
4631 constraint(ALLOC_IN_RC(any_reg32));
4632 match(RegI);
4633 match(iRegINoSp);
4634 op_cost(0);
4635 format %{ %}
4636 interface(REG_INTER);
4637 %}
4638
4639 // Integer 32 bit Register not Special
4640 operand iRegINoSp()
4641 %{
4642 constraint(ALLOC_IN_RC(no_special_reg32));
4643 match(RegI);
4644 op_cost(0);
4645 format %{ %}
4646 interface(REG_INTER);
4647 %}
4648
4649 // Integer 64 bit Register Operands
4650 // Integer 64 bit Register (includes SP)
4651 operand iRegL()
4652 %{
4653 constraint(ALLOC_IN_RC(any_reg));
4654 match(RegL);
4655 match(iRegLNoSp);
4656 op_cost(0);
4657 format %{ %}
4658 interface(REG_INTER);
4659 %}
4660
4661 // Integer 64 bit Register not Special
4662 operand iRegLNoSp()
4663 %{
4664 constraint(ALLOC_IN_RC(no_special_reg));
4665 match(RegL);
4666 match(iRegL_R0);
4667 format %{ %}
4668 interface(REG_INTER);
4669 %}
4670
4671 // Pointer Register Operands
4672 // Pointer Register
4673 operand iRegP()
4674 %{
4675 constraint(ALLOC_IN_RC(ptr_reg));
4676 match(RegP);
4677 match(iRegPNoSp);
4678 match(iRegP_R0);
4679 //match(iRegP_R2);
4680 //match(iRegP_R4);
4681 match(iRegP_R5);
4682 match(thread_RegP);
4683 op_cost(0);
4684 format %{ %}
4685 interface(REG_INTER);
4686 %}
4687
4688 // Pointer 64 bit Register not Special
4689 operand iRegPNoSp()
4690 %{
4691 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4692 match(RegP);
4693 // match(iRegP);
4694 // match(iRegP_R0);
4695 // match(iRegP_R2);
4696 // match(iRegP_R4);
4697 // match(iRegP_R5);
4698 // match(thread_RegP);
4699 op_cost(0);
4700 format %{ %}
4701 interface(REG_INTER);
4702 %}
4703
4704 // This operand is not allowed to use rfp even if
4705 // rfp is not used to hold the frame pointer.
4706 operand iRegPNoSpNoRfp()
4707 %{
4708 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4709 match(RegP);
4710 match(iRegPNoSp);
4711 op_cost(0);
4712 format %{ %}
4713 interface(REG_INTER);
4714 %}
4715
4716 // Pointer 64 bit Register R0 only
4717 operand iRegP_R0()
4718 %{
4719 constraint(ALLOC_IN_RC(r0_reg));
4720 match(RegP);
4721 // match(iRegP);
4722 match(iRegPNoSp);
4723 op_cost(0);
4724 format %{ %}
4725 interface(REG_INTER);
4726 %}
4727
4728 // Pointer 64 bit Register R1 only
4729 operand iRegP_R1()
4730 %{
4731 constraint(ALLOC_IN_RC(r1_reg));
4732 match(RegP);
4733 // match(iRegP);
4734 match(iRegPNoSp);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Pointer 64 bit Register R2 only
4741 operand iRegP_R2()
4742 %{
4743 constraint(ALLOC_IN_RC(r2_reg));
4744 match(RegP);
4745 // match(iRegP);
4746 match(iRegPNoSp);
4747 op_cost(0);
4748 format %{ %}
4749 interface(REG_INTER);
4750 %}
4751
4752 // Pointer 64 bit Register R3 only
4753 operand iRegP_R3()
4754 %{
4755 constraint(ALLOC_IN_RC(r3_reg));
4756 match(RegP);
4757 // match(iRegP);
4758 match(iRegPNoSp);
4759 op_cost(0);
4760 format %{ %}
4761 interface(REG_INTER);
4762 %}
4763
4764 // Pointer 64 bit Register R4 only
4765 operand iRegP_R4()
4766 %{
4767 constraint(ALLOC_IN_RC(r4_reg));
4768 match(RegP);
4769 // match(iRegP);
4770 match(iRegPNoSp);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776 // Pointer 64 bit Register R5 only
4777 operand iRegP_R5()
4778 %{
4779 constraint(ALLOC_IN_RC(r5_reg));
4780 match(RegP);
4781 // match(iRegP);
4782 match(iRegPNoSp);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // Pointer 64 bit Register R10 only
4789 operand iRegP_R10()
4790 %{
4791 constraint(ALLOC_IN_RC(r10_reg));
4792 match(RegP);
4793 // match(iRegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Long 64 bit Register R0 only
4801 operand iRegL_R0()
4802 %{
4803 constraint(ALLOC_IN_RC(r0_reg));
4804 match(RegL);
4805 match(iRegLNoSp);
4806 op_cost(0);
4807 format %{ %}
4808 interface(REG_INTER);
4809 %}
4810
4811 // Long 64 bit Register R11 only
4812 operand iRegL_R11()
4813 %{
4814 constraint(ALLOC_IN_RC(r11_reg));
4815 match(RegL);
4816 match(iRegLNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Register R0 only
4823 operand iRegI_R0()
4824 %{
4825 constraint(ALLOC_IN_RC(int_r0_reg));
4826 match(RegI);
4827 match(iRegINoSp);
4828 op_cost(0);
4829 format %{ %}
4830 interface(REG_INTER);
4831 %}
4832
4833 // Register R2 only
4834 operand iRegI_R2()
4835 %{
4836 constraint(ALLOC_IN_RC(int_r2_reg));
4837 match(RegI);
4838 match(iRegINoSp);
4839 op_cost(0);
4840 format %{ %}
4841 interface(REG_INTER);
4842 %}
4843
4844 // Register R3 only
4845 operand iRegI_R3()
4846 %{
4847 constraint(ALLOC_IN_RC(int_r3_reg));
4848 match(RegI);
4849 match(iRegINoSp);
4850 op_cost(0);
4851 format %{ %}
4852 interface(REG_INTER);
4853 %}
4854
4855
4856 // Register R4 only
4857 operand iRegI_R4()
4858 %{
4859 constraint(ALLOC_IN_RC(int_r4_reg));
4860 match(RegI);
4861 match(iRegINoSp);
4862 op_cost(0);
4863 format %{ %}
4864 interface(REG_INTER);
4865 %}
4866
4867
4868 // Pointer Register Operands
4869 // Narrow Pointer Register
4870 operand iRegN()
4871 %{
4872 constraint(ALLOC_IN_RC(any_reg32));
4873 match(RegN);
4874 match(iRegNNoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880 // Integer 64 bit Register not Special
4881 operand iRegNNoSp()
4882 %{
4883 constraint(ALLOC_IN_RC(no_special_reg32));
4884 match(RegN);
4885 op_cost(0);
4886 format %{ %}
4887 interface(REG_INTER);
4888 %}
4889
4890 // Float Register
4891 // Float register operands
4892 operand vRegF()
4893 %{
4894 constraint(ALLOC_IN_RC(float_reg));
4895 match(RegF);
4896
4897 op_cost(0);
4898 format %{ %}
4899 interface(REG_INTER);
4900 %}
4901
4902 // Double Register
4903 // Double register operands
4904 operand vRegD()
4905 %{
4906 constraint(ALLOC_IN_RC(double_reg));
4907 match(RegD);
4908
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Generic vector class. This will be used for
4915 // all vector operands, including NEON and SVE.
4916 operand vReg()
4917 %{
4918 constraint(ALLOC_IN_RC(dynamic));
4919 match(VecA);
4920 match(VecD);
4921 match(VecX);
4922
4923 op_cost(0);
4924 format %{ %}
4925 interface(REG_INTER);
4926 %}
4927
4928 operand vecA()
4929 %{
4930 constraint(ALLOC_IN_RC(vectora_reg));
4931 match(VecA);
4932
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 operand vecD()
4939 %{
4940 constraint(ALLOC_IN_RC(vectord_reg));
4941 match(VecD);
4942
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 operand vecX()
4949 %{
4950 constraint(ALLOC_IN_RC(vectorx_reg));
4951 match(VecX);
4952
4953 op_cost(0);
4954 format %{ %}
4955 interface(REG_INTER);
4956 %}
4957
4958 operand vRegD_V0()
4959 %{
4960 constraint(ALLOC_IN_RC(v0_reg));
4961 match(RegD);
4962 op_cost(0);
4963 format %{ %}
4964 interface(REG_INTER);
4965 %}
4966
4967 operand vRegD_V1()
4968 %{
4969 constraint(ALLOC_IN_RC(v1_reg));
4970 match(RegD);
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976 operand vRegD_V2()
4977 %{
4978 constraint(ALLOC_IN_RC(v2_reg));
4979 match(RegD);
4980 op_cost(0);
4981 format %{ %}
4982 interface(REG_INTER);
4983 %}
4984
4985 operand vRegD_V3()
4986 %{
4987 constraint(ALLOC_IN_RC(v3_reg));
4988 match(RegD);
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 operand vRegD_V4()
4995 %{
4996 constraint(ALLOC_IN_RC(v4_reg));
4997 match(RegD);
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vRegD_V5()
5004 %{
5005 constraint(ALLOC_IN_RC(v5_reg));
5006 match(RegD);
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vRegD_V6()
5013 %{
5014 constraint(ALLOC_IN_RC(v6_reg));
5015 match(RegD);
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 operand vRegD_V7()
5022 %{
5023 constraint(ALLOC_IN_RC(v7_reg));
5024 match(RegD);
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vRegD_V12()
5031 %{
5032 constraint(ALLOC_IN_RC(v12_reg));
5033 match(RegD);
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand vRegD_V13()
5040 %{
5041 constraint(ALLOC_IN_RC(v13_reg));
5042 match(RegD);
5043 op_cost(0);
5044 format %{ %}
5045 interface(REG_INTER);
5046 %}
5047
5048 operand pReg()
5049 %{
5050 constraint(ALLOC_IN_RC(pr_reg));
5051 match(RegVectMask);
5052 match(pRegGov);
5053 op_cost(0);
5054 format %{ %}
5055 interface(REG_INTER);
5056 %}
5057
5058 operand pRegGov()
5059 %{
5060 constraint(ALLOC_IN_RC(gov_pr));
5061 match(RegVectMask);
5062 match(pReg);
5063 op_cost(0);
5064 format %{ %}
5065 interface(REG_INTER);
5066 %}
5067
5068 operand pRegGov_P0()
5069 %{
5070 constraint(ALLOC_IN_RC(p0_reg));
5071 match(RegVectMask);
5072 op_cost(0);
5073 format %{ %}
5074 interface(REG_INTER);
5075 %}
5076
5077 operand pRegGov_P1()
5078 %{
5079 constraint(ALLOC_IN_RC(p1_reg));
5080 match(RegVectMask);
5081 op_cost(0);
5082 format %{ %}
5083 interface(REG_INTER);
5084 %}
5085
5086 // Flags register, used as output of signed compare instructions
5087
5088 // note that on AArch64 we also use this register as the output for
5089 // for floating point compare instructions (CmpF CmpD). this ensures
5090 // that ordered inequality tests use GT, GE, LT or LE none of which
5091 // pass through cases where the result is unordered i.e. one or both
5092 // inputs to the compare is a NaN. this means that the ideal code can
5093 // replace e.g. a GT with an LE and not end up capturing the NaN case
5094 // (where the comparison should always fail). EQ and NE tests are
5095 // always generated in ideal code so that unordered folds into the NE
5096 // case, matching the behaviour of AArch64 NE.
5097 //
5098 // This differs from x86 where the outputs of FP compares use a
5099 // special FP flags registers and where compares based on this
5100 // register are distinguished into ordered inequalities (cmpOpUCF) and
5101 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5102 // to explicitly handle the unordered case in branches. x86 also has
5103 // to include extra CMoveX rules to accept a cmpOpUCF input.
5104
5105 operand rFlagsReg()
5106 %{
5107 constraint(ALLOC_IN_RC(int_flags));
5108 match(RegFlags);
5109
5110 op_cost(0);
5111 format %{ "RFLAGS" %}
5112 interface(REG_INTER);
5113 %}
5114
5115 // Flags register, used as output of unsigned compare instructions
5116 operand rFlagsRegU()
5117 %{
5118 constraint(ALLOC_IN_RC(int_flags));
5119 match(RegFlags);
5120
5121 op_cost(0);
5122 format %{ "RFLAGSU" %}
5123 interface(REG_INTER);
5124 %}
5125
5126 // Special Registers
5127
5128 // Method Register
5129 operand inline_cache_RegP(iRegP reg)
5130 %{
5131 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5132 match(reg);
5133 match(iRegPNoSp);
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 // Thread Register
5140 operand thread_RegP(iRegP reg)
5141 %{
5142 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5143 match(reg);
5144 op_cost(0);
5145 format %{ %}
5146 interface(REG_INTER);
5147 %}
5148
5149 //----------Memory Operands----------------------------------------------------
5150
5151 operand indirect(iRegP reg)
5152 %{
5153 constraint(ALLOC_IN_RC(ptr_reg));
5154 match(reg);
5155 op_cost(0);
5156 format %{ "[$reg]" %}
5157 interface(MEMORY_INTER) %{
5158 base($reg);
5159 index(0xffffffff);
5160 scale(0x0);
5161 disp(0x0);
5162 %}
5163 %}
5164
5165 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5166 %{
5167 constraint(ALLOC_IN_RC(ptr_reg));
5168 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5169 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5170 op_cost(0);
5171 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5172 interface(MEMORY_INTER) %{
5173 base($reg);
5174 index($ireg);
5175 scale($scale);
5176 disp(0x0);
5177 %}
5178 %}
5179
5180 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5181 %{
5182 constraint(ALLOC_IN_RC(ptr_reg));
5183 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5184 match(AddP reg (LShiftL lreg scale));
5185 op_cost(0);
5186 format %{ "$reg, $lreg lsl($scale)" %}
5187 interface(MEMORY_INTER) %{
5188 base($reg);
5189 index($lreg);
5190 scale($scale);
5191 disp(0x0);
5192 %}
5193 %}
5194
5195 operand indIndexI2L(iRegP reg, iRegI ireg)
5196 %{
5197 constraint(ALLOC_IN_RC(ptr_reg));
5198 match(AddP reg (ConvI2L ireg));
5199 op_cost(0);
5200 format %{ "$reg, $ireg, 0, I2L" %}
5201 interface(MEMORY_INTER) %{
5202 base($reg);
5203 index($ireg);
5204 scale(0x0);
5205 disp(0x0);
5206 %}
5207 %}
5208
5209 operand indIndex(iRegP reg, iRegL lreg)
5210 %{
5211 constraint(ALLOC_IN_RC(ptr_reg));
5212 match(AddP reg lreg);
5213 op_cost(0);
5214 format %{ "$reg, $lreg" %}
5215 interface(MEMORY_INTER) %{
5216 base($reg);
5217 index($lreg);
5218 scale(0x0);
5219 disp(0x0);
5220 %}
5221 %}
5222
5223 operand indOffI1(iRegP reg, immIOffset1 off)
5224 %{
5225 constraint(ALLOC_IN_RC(ptr_reg));
5226 match(AddP reg off);
5227 op_cost(0);
5228 format %{ "[$reg, $off]" %}
5229 interface(MEMORY_INTER) %{
5230 base($reg);
5231 index(0xffffffff);
5232 scale(0x0);
5233 disp($off);
5234 %}
5235 %}
5236
5237 operand indOffI2(iRegP reg, immIOffset2 off)
5238 %{
5239 constraint(ALLOC_IN_RC(ptr_reg));
5240 match(AddP reg off);
5241 op_cost(0);
5242 format %{ "[$reg, $off]" %}
5243 interface(MEMORY_INTER) %{
5244 base($reg);
5245 index(0xffffffff);
5246 scale(0x0);
5247 disp($off);
5248 %}
5249 %}
5250
5251 operand indOffI4(iRegP reg, immIOffset4 off)
5252 %{
5253 constraint(ALLOC_IN_RC(ptr_reg));
5254 match(AddP reg off);
5255 op_cost(0);
5256 format %{ "[$reg, $off]" %}
5257 interface(MEMORY_INTER) %{
5258 base($reg);
5259 index(0xffffffff);
5260 scale(0x0);
5261 disp($off);
5262 %}
5263 %}
5264
5265 operand indOffI8(iRegP reg, immIOffset8 off)
5266 %{
5267 constraint(ALLOC_IN_RC(ptr_reg));
5268 match(AddP reg off);
5269 op_cost(0);
5270 format %{ "[$reg, $off]" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index(0xffffffff);
5274 scale(0x0);
5275 disp($off);
5276 %}
5277 %}
5278
5279 operand indOffI16(iRegP reg, immIOffset16 off)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 match(AddP reg off);
5283 op_cost(0);
5284 format %{ "[$reg, $off]" %}
5285 interface(MEMORY_INTER) %{
5286 base($reg);
5287 index(0xffffffff);
5288 scale(0x0);
5289 disp($off);
5290 %}
5291 %}
5292
5293 operand indOffL1(iRegP reg, immLoffset1 off)
5294 %{
5295 constraint(ALLOC_IN_RC(ptr_reg));
5296 match(AddP reg off);
5297 op_cost(0);
5298 format %{ "[$reg, $off]" %}
5299 interface(MEMORY_INTER) %{
5300 base($reg);
5301 index(0xffffffff);
5302 scale(0x0);
5303 disp($off);
5304 %}
5305 %}
5306
5307 operand indOffL2(iRegP reg, immLoffset2 off)
5308 %{
5309 constraint(ALLOC_IN_RC(ptr_reg));
5310 match(AddP reg off);
5311 op_cost(0);
5312 format %{ "[$reg, $off]" %}
5313 interface(MEMORY_INTER) %{
5314 base($reg);
5315 index(0xffffffff);
5316 scale(0x0);
5317 disp($off);
5318 %}
5319 %}
5320
5321 operand indOffL4(iRegP reg, immLoffset4 off)
5322 %{
5323 constraint(ALLOC_IN_RC(ptr_reg));
5324 match(AddP reg off);
5325 op_cost(0);
5326 format %{ "[$reg, $off]" %}
5327 interface(MEMORY_INTER) %{
5328 base($reg);
5329 index(0xffffffff);
5330 scale(0x0);
5331 disp($off);
5332 %}
5333 %}
5334
5335 operand indOffL8(iRegP reg, immLoffset8 off)
5336 %{
5337 constraint(ALLOC_IN_RC(ptr_reg));
5338 match(AddP reg off);
5339 op_cost(0);
5340 format %{ "[$reg, $off]" %}
5341 interface(MEMORY_INTER) %{
5342 base($reg);
5343 index(0xffffffff);
5344 scale(0x0);
5345 disp($off);
5346 %}
5347 %}
5348
5349 operand indOffL16(iRegP reg, immLoffset16 off)
5350 %{
5351 constraint(ALLOC_IN_RC(ptr_reg));
5352 match(AddP reg off);
5353 op_cost(0);
5354 format %{ "[$reg, $off]" %}
5355 interface(MEMORY_INTER) %{
5356 base($reg);
5357 index(0xffffffff);
5358 scale(0x0);
5359 disp($off);
5360 %}
5361 %}
5362
5363 operand indirectX2P(iRegL reg)
5364 %{
5365 constraint(ALLOC_IN_RC(ptr_reg));
5366 match(CastX2P reg);
5367 op_cost(0);
5368 format %{ "[$reg]\t# long -> ptr" %}
5369 interface(MEMORY_INTER) %{
5370 base($reg);
5371 index(0xffffffff);
5372 scale(0x0);
5373 disp(0x0);
5374 %}
5375 %}
5376
5377 operand indOffX2P(iRegL reg, immLOffset off)
5378 %{
5379 constraint(ALLOC_IN_RC(ptr_reg));
5380 match(AddP (CastX2P reg) off);
5381 op_cost(0);
5382 format %{ "[$reg, $off]\t# long -> ptr" %}
5383 interface(MEMORY_INTER) %{
5384 base($reg);
5385 index(0xffffffff);
5386 scale(0x0);
5387 disp($off);
5388 %}
5389 %}
5390
5391 operand indirectN(iRegN reg)
5392 %{
5393 predicate(CompressedOops::shift() == 0);
5394 constraint(ALLOC_IN_RC(ptr_reg));
5395 match(DecodeN reg);
5396 op_cost(0);
5397 format %{ "[$reg]\t# narrow" %}
5398 interface(MEMORY_INTER) %{
5399 base($reg);
5400 index(0xffffffff);
5401 scale(0x0);
5402 disp(0x0);
5403 %}
5404 %}
5405
5406 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5407 %{
5408 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5409 constraint(ALLOC_IN_RC(ptr_reg));
5410 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5411 op_cost(0);
5412 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5413 interface(MEMORY_INTER) %{
5414 base($reg);
5415 index($ireg);
5416 scale($scale);
5417 disp(0x0);
5418 %}
5419 %}
5420
5421 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5422 %{
5423 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5424 constraint(ALLOC_IN_RC(ptr_reg));
5425 match(AddP (DecodeN reg) (LShiftL lreg scale));
5426 op_cost(0);
5427 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5428 interface(MEMORY_INTER) %{
5429 base($reg);
5430 index($lreg);
5431 scale($scale);
5432 disp(0x0);
5433 %}
5434 %}
5435
5436 operand indIndexI2LN(iRegN reg, iRegI ireg)
5437 %{
5438 predicate(CompressedOops::shift() == 0);
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(AddP (DecodeN reg) (ConvI2L ireg));
5441 op_cost(0);
5442 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index($ireg);
5446 scale(0x0);
5447 disp(0x0);
5448 %}
5449 %}
5450
5451 operand indIndexN(iRegN reg, iRegL lreg)
5452 %{
5453 predicate(CompressedOops::shift() == 0);
5454 constraint(ALLOC_IN_RC(ptr_reg));
5455 match(AddP (DecodeN reg) lreg);
5456 op_cost(0);
5457 format %{ "$reg, $lreg\t# narrow" %}
5458 interface(MEMORY_INTER) %{
5459 base($reg);
5460 index($lreg);
5461 scale(0x0);
5462 disp(0x0);
5463 %}
5464 %}
5465
5466 operand indOffIN(iRegN reg, immIOffset off)
5467 %{
5468 predicate(CompressedOops::shift() == 0);
5469 constraint(ALLOC_IN_RC(ptr_reg));
5470 match(AddP (DecodeN reg) off);
5471 op_cost(0);
5472 format %{ "[$reg, $off]\t# narrow" %}
5473 interface(MEMORY_INTER) %{
5474 base($reg);
5475 index(0xffffffff);
5476 scale(0x0);
5477 disp($off);
5478 %}
5479 %}
5480
5481 operand indOffLN(iRegN reg, immLOffset off)
5482 %{
5483 predicate(CompressedOops::shift() == 0);
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(AddP (DecodeN reg) off);
5486 op_cost(0);
5487 format %{ "[$reg, $off]\t# narrow" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index(0xffffffff);
5491 scale(0x0);
5492 disp($off);
5493 %}
5494 %}
5495
5496
5497 //----------Special Memory Operands--------------------------------------------
5498 // Stack Slot Operand - This operand is used for loading and storing temporary
5499 // values on the stack where a match requires a value to
5500 // flow through memory.
5501 operand stackSlotP(sRegP reg)
5502 %{
5503 constraint(ALLOC_IN_RC(stack_slots));
5504 op_cost(100);
5505 // No match rule because this operand is only generated in matching
5506 // match(RegP);
5507 format %{ "[$reg]" %}
5508 interface(MEMORY_INTER) %{
5509 base(0x1e); // RSP
5510 index(0x0); // No Index
5511 scale(0x0); // No Scale
5512 disp($reg); // Stack Offset
5513 %}
5514 %}
5515
5516 operand stackSlotI(sRegI reg)
5517 %{
5518 constraint(ALLOC_IN_RC(stack_slots));
5519 // No match rule because this operand is only generated in matching
5520 // match(RegI);
5521 format %{ "[$reg]" %}
5522 interface(MEMORY_INTER) %{
5523 base(0x1e); // RSP
5524 index(0x0); // No Index
5525 scale(0x0); // No Scale
5526 disp($reg); // Stack Offset
5527 %}
5528 %}
5529
5530 operand stackSlotF(sRegF reg)
5531 %{
5532 constraint(ALLOC_IN_RC(stack_slots));
5533 // No match rule because this operand is only generated in matching
5534 // match(RegF);
5535 format %{ "[$reg]" %}
5536 interface(MEMORY_INTER) %{
5537 base(0x1e); // RSP
5538 index(0x0); // No Index
5539 scale(0x0); // No Scale
5540 disp($reg); // Stack Offset
5541 %}
5542 %}
5543
5544 operand stackSlotD(sRegD reg)
5545 %{
5546 constraint(ALLOC_IN_RC(stack_slots));
5547 // No match rule because this operand is only generated in matching
5548 // match(RegD);
5549 format %{ "[$reg]" %}
5550 interface(MEMORY_INTER) %{
5551 base(0x1e); // RSP
5552 index(0x0); // No Index
5553 scale(0x0); // No Scale
5554 disp($reg); // Stack Offset
5555 %}
5556 %}
5557
5558 operand stackSlotL(sRegL reg)
5559 %{
5560 constraint(ALLOC_IN_RC(stack_slots));
5561 // No match rule because this operand is only generated in matching
5562 // match(RegL);
5563 format %{ "[$reg]" %}
5564 interface(MEMORY_INTER) %{
5565 base(0x1e); // RSP
5566 index(0x0); // No Index
5567 scale(0x0); // No Scale
5568 disp($reg); // Stack Offset
5569 %}
5570 %}
5571
5572 // Operands for expressing Control Flow
5573 // NOTE: Label is a predefined operand which should not be redefined in
5574 // the AD file. It is generically handled within the ADLC.
5575
5576 //----------Conditional Branch Operands----------------------------------------
5577 // Comparison Op - This is the operation of the comparison, and is limited to
5578 // the following set of codes:
5579 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5580 //
5581 // Other attributes of the comparison, such as unsignedness, are specified
5582 // by the comparison instruction that sets a condition code flags register.
5583 // That result is represented by a flags operand whose subtype is appropriate
5584 // to the unsignedness (etc.) of the comparison.
5585 //
5586 // Later, the instruction which matches both the Comparison Op (a Bool) and
5587 // the flags (produced by the Cmp) specifies the coding of the comparison op
5588 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5589
5590 // used for signed integral comparisons and fp comparisons
5591
5592 operand cmpOp()
5593 %{
5594 match(Bool);
5595
5596 format %{ "" %}
5597 interface(COND_INTER) %{
5598 equal(0x0, "eq");
5599 not_equal(0x1, "ne");
5600 less(0xb, "lt");
5601 greater_equal(0xa, "ge");
5602 less_equal(0xd, "le");
5603 greater(0xc, "gt");
5604 overflow(0x6, "vs");
5605 no_overflow(0x7, "vc");
5606 %}
5607 %}
5608
5609 // used for unsigned integral comparisons
5610
5611 operand cmpOpU()
5612 %{
5613 match(Bool);
5614
5615 format %{ "" %}
5616 interface(COND_INTER) %{
5617 equal(0x0, "eq");
5618 not_equal(0x1, "ne");
5619 less(0x3, "lo");
5620 greater_equal(0x2, "hs");
5621 less_equal(0x9, "ls");
5622 greater(0x8, "hi");
5623 overflow(0x6, "vs");
5624 no_overflow(0x7, "vc");
5625 %}
5626 %}
5627
5628 // used for certain integral comparisons which can be
5629 // converted to cbxx or tbxx instructions
5630
5631 operand cmpOpEqNe()
5632 %{
5633 match(Bool);
5634 op_cost(0);
5635 predicate(n->as_Bool()->_test._test == BoolTest::ne
5636 || n->as_Bool()->_test._test == BoolTest::eq);
5637
5638 format %{ "" %}
5639 interface(COND_INTER) %{
5640 equal(0x0, "eq");
5641 not_equal(0x1, "ne");
5642 less(0xb, "lt");
5643 greater_equal(0xa, "ge");
5644 less_equal(0xd, "le");
5645 greater(0xc, "gt");
5646 overflow(0x6, "vs");
5647 no_overflow(0x7, "vc");
5648 %}
5649 %}
5650
5651 // used for certain integral comparisons which can be
5652 // converted to cbxx or tbxx instructions
5653
5654 operand cmpOpLtGe()
5655 %{
5656 match(Bool);
5657 op_cost(0);
5658
5659 predicate(n->as_Bool()->_test._test == BoolTest::lt
5660 || n->as_Bool()->_test._test == BoolTest::ge);
5661
5662 format %{ "" %}
5663 interface(COND_INTER) %{
5664 equal(0x0, "eq");
5665 not_equal(0x1, "ne");
5666 less(0xb, "lt");
5667 greater_equal(0xa, "ge");
5668 less_equal(0xd, "le");
5669 greater(0xc, "gt");
5670 overflow(0x6, "vs");
5671 no_overflow(0x7, "vc");
5672 %}
5673 %}
5674
5675 // used for certain unsigned integral comparisons which can be
5676 // converted to cbxx or tbxx instructions
5677
5678 operand cmpOpUEqNeLeGt()
5679 %{
5680 match(Bool);
5681 op_cost(0);
5682
5683 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5684 n->as_Bool()->_test._test == BoolTest::ne ||
5685 n->as_Bool()->_test._test == BoolTest::le ||
5686 n->as_Bool()->_test._test == BoolTest::gt);
5687
5688 format %{ "" %}
5689 interface(COND_INTER) %{
5690 equal(0x0, "eq");
5691 not_equal(0x1, "ne");
5692 less(0x3, "lo");
5693 greater_equal(0x2, "hs");
5694 less_equal(0x9, "ls");
5695 greater(0x8, "hi");
5696 overflow(0x6, "vs");
5697 no_overflow(0x7, "vc");
5698 %}
5699 %}
5700
5701 // Special operand allowing long args to int ops to be truncated for free
5702
5703 operand iRegL2I(iRegL reg) %{
5704
5705 op_cost(0);
5706
5707 match(ConvL2I reg);
5708
5709 format %{ "l2i($reg)" %}
5710
5711 interface(REG_INTER)
5712 %}
5713
5714 operand iRegL2P(iRegL reg) %{
5715
5716 op_cost(0);
5717
5718 match(CastX2P reg);
5719
5720 format %{ "l2p($reg)" %}
5721
5722 interface(REG_INTER)
5723 %}
5724
5725 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5726 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5727 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5728 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5729
5730 //----------OPERAND CLASSES----------------------------------------------------
5731 // Operand Classes are groups of operands that are used as to simplify
5732 // instruction definitions by not requiring the AD writer to specify
5733 // separate instructions for every form of operand when the
5734 // instruction accepts multiple operand types with the same basic
5735 // encoding and format. The classic case of this is memory operands.
5736
5737 // memory is used to define read/write location for load/store
5738 // instruction defs. we can turn a memory op into an Address
5739
5740 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5741 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5742
5743 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5744 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5745
5746 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5747 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5748
5749 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5750 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5751
5752 // All of the memory operands. For the pipeline description.
5753 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5754 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5755 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5756
5757
5758 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5759 // operations. it allows the src to be either an iRegI or a (ConvL2I
5760 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5761 // can be elided because the 32-bit instruction will just employ the
5762 // lower 32 bits anyway.
5763 //
5764 // n.b. this does not elide all L2I conversions. if the truncated
5765 // value is consumed by more than one operation then the ConvL2I
5766 // cannot be bundled into the consuming nodes so an l2i gets planted
5767 // (actually a movw $dst $src) and the downstream instructions consume
5768 // the result of the l2i as an iRegI input. That's a shame since the
5769 // movw is actually redundant but its not too costly.
5770
5771 opclass iRegIorL2I(iRegI, iRegL2I);
5772 opclass iRegPorL2P(iRegP, iRegL2P);
5773
5774 //----------PIPELINE-----------------------------------------------------------
5775 // Rules which define the behavior of the target architectures pipeline.
5776
5777 // For specific pipelines, eg A53, define the stages of that pipeline
5778 //pipe_desc(ISS, EX1, EX2, WR);
5779 #define ISS S0
5780 #define EX1 S1
5781 #define EX2 S2
5782 #define WR S3
5783
5784 // Integer ALU reg operation
5785 pipeline %{
5786
5787 attributes %{
5788 // ARM instructions are of fixed length
5789 fixed_size_instructions; // Fixed size instructions TODO does
5790 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5791 // ARM instructions come in 32-bit word units
5792 instruction_unit_size = 4; // An instruction is 4 bytes long
5793 instruction_fetch_unit_size = 64; // The processor fetches one line
5794 instruction_fetch_units = 1; // of 64 bytes
5795
5796 // List of nop instructions
5797 nops( MachNop );
5798 %}
5799
5800 // We don't use an actual pipeline model so don't care about resources
5801 // or description. we do use pipeline classes to introduce fixed
5802 // latencies
5803
5804 //----------RESOURCES----------------------------------------------------------
5805 // Resources are the functional units available to the machine
5806
5807 resources( INS0, INS1, INS01 = INS0 | INS1,
5808 ALU0, ALU1, ALU = ALU0 | ALU1,
5809 MAC,
5810 DIV,
5811 BRANCH,
5812 LDST,
5813 NEON_FP);
5814
5815 //----------PIPELINE DESCRIPTION-----------------------------------------------
5816 // Pipeline Description specifies the stages in the machine's pipeline
5817
5818 // Define the pipeline as a generic 6 stage pipeline
5819 pipe_desc(S0, S1, S2, S3, S4, S5);
5820
5821 //----------PIPELINE CLASSES---------------------------------------------------
5822 // Pipeline Classes describe the stages in which input and output are
5823 // referenced by the hardware pipeline.
5824
5825 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5826 %{
5827 single_instruction;
5828 src1 : S1(read);
5829 src2 : S2(read);
5830 dst : S5(write);
5831 INS01 : ISS;
5832 NEON_FP : S5;
5833 %}
5834
5835 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5836 %{
5837 single_instruction;
5838 src1 : S1(read);
5839 src2 : S2(read);
5840 dst : S5(write);
5841 INS01 : ISS;
5842 NEON_FP : S5;
5843 %}
5844
5845 pipe_class fp_uop_s(vRegF dst, vRegF src)
5846 %{
5847 single_instruction;
5848 src : S1(read);
5849 dst : S5(write);
5850 INS01 : ISS;
5851 NEON_FP : S5;
5852 %}
5853
5854 pipe_class fp_uop_d(vRegD dst, vRegD src)
5855 %{
5856 single_instruction;
5857 src : S1(read);
5858 dst : S5(write);
5859 INS01 : ISS;
5860 NEON_FP : S5;
5861 %}
5862
5863 pipe_class fp_d2f(vRegF dst, vRegD src)
5864 %{
5865 single_instruction;
5866 src : S1(read);
5867 dst : S5(write);
5868 INS01 : ISS;
5869 NEON_FP : S5;
5870 %}
5871
5872 pipe_class fp_f2d(vRegD dst, vRegF src)
5873 %{
5874 single_instruction;
5875 src : S1(read);
5876 dst : S5(write);
5877 INS01 : ISS;
5878 NEON_FP : S5;
5879 %}
5880
5881 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5882 %{
5883 single_instruction;
5884 src : S1(read);
5885 dst : S5(write);
5886 INS01 : ISS;
5887 NEON_FP : S5;
5888 %}
5889
5890 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5891 %{
5892 single_instruction;
5893 src : S1(read);
5894 dst : S5(write);
5895 INS01 : ISS;
5896 NEON_FP : S5;
5897 %}
5898
5899 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5900 %{
5901 single_instruction;
5902 src : S1(read);
5903 dst : S5(write);
5904 INS01 : ISS;
5905 NEON_FP : S5;
5906 %}
5907
5908 pipe_class fp_l2f(vRegF dst, iRegL src)
5909 %{
5910 single_instruction;
5911 src : S1(read);
5912 dst : S5(write);
5913 INS01 : ISS;
5914 NEON_FP : S5;
5915 %}
5916
5917 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5918 %{
5919 single_instruction;
5920 src : S1(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5927 %{
5928 single_instruction;
5929 src : S1(read);
5930 dst : S5(write);
5931 INS01 : ISS;
5932 NEON_FP : S5;
5933 %}
5934
5935 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5936 %{
5937 single_instruction;
5938 src : S1(read);
5939 dst : S5(write);
5940 INS01 : ISS;
5941 NEON_FP : S5;
5942 %}
5943
5944 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5945 %{
5946 single_instruction;
5947 src : S1(read);
5948 dst : S5(write);
5949 INS01 : ISS;
5950 NEON_FP : S5;
5951 %}
5952
5953 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5954 %{
5955 single_instruction;
5956 src1 : S1(read);
5957 src2 : S2(read);
5958 dst : S5(write);
5959 INS0 : ISS;
5960 NEON_FP : S5;
5961 %}
5962
5963 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5964 %{
5965 single_instruction;
5966 src1 : S1(read);
5967 src2 : S2(read);
5968 dst : S5(write);
5969 INS0 : ISS;
5970 NEON_FP : S5;
5971 %}
5972
5973 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5974 %{
5975 single_instruction;
5976 cr : S1(read);
5977 src1 : S1(read);
5978 src2 : S1(read);
5979 dst : S3(write);
5980 INS01 : ISS;
5981 NEON_FP : S3;
5982 %}
5983
5984 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5985 %{
5986 single_instruction;
5987 cr : S1(read);
5988 src1 : S1(read);
5989 src2 : S1(read);
5990 dst : S3(write);
5991 INS01 : ISS;
5992 NEON_FP : S3;
5993 %}
5994
5995 pipe_class fp_imm_s(vRegF dst)
5996 %{
5997 single_instruction;
5998 dst : S3(write);
5999 INS01 : ISS;
6000 NEON_FP : S3;
6001 %}
6002
6003 pipe_class fp_imm_d(vRegD dst)
6004 %{
6005 single_instruction;
6006 dst : S3(write);
6007 INS01 : ISS;
6008 NEON_FP : S3;
6009 %}
6010
6011 pipe_class fp_load_constant_s(vRegF dst)
6012 %{
6013 single_instruction;
6014 dst : S4(write);
6015 INS01 : ISS;
6016 NEON_FP : S4;
6017 %}
6018
6019 pipe_class fp_load_constant_d(vRegD dst)
6020 %{
6021 single_instruction;
6022 dst : S4(write);
6023 INS01 : ISS;
6024 NEON_FP : S4;
6025 %}
6026
6027 //------- Integer ALU operations --------------------------
6028
6029 // Integer ALU reg-reg operation
6030 // Operands needed in EX1, result generated in EX2
6031 // Eg. ADD x0, x1, x2
6032 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6033 %{
6034 single_instruction;
6035 dst : EX2(write);
6036 src1 : EX1(read);
6037 src2 : EX1(read);
6038 INS01 : ISS; // Dual issue as instruction 0 or 1
6039 ALU : EX2;
6040 %}
6041
6042 // Integer ALU reg-reg operation with constant shift
6043 // Shifted register must be available in LATE_ISS instead of EX1
6044 // Eg. ADD x0, x1, x2, LSL #2
6045 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6046 %{
6047 single_instruction;
6048 dst : EX2(write);
6049 src1 : EX1(read);
6050 src2 : ISS(read);
6051 INS01 : ISS;
6052 ALU : EX2;
6053 %}
6054
6055 // Integer ALU reg operation with constant shift
6056 // Eg. LSL x0, x1, #shift
6057 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6058 %{
6059 single_instruction;
6060 dst : EX2(write);
6061 src1 : ISS(read);
6062 INS01 : ISS;
6063 ALU : EX2;
6064 %}
6065
6066 // Integer ALU reg-reg operation with variable shift
6067 // Both operands must be available in LATE_ISS instead of EX1
6068 // Result is available in EX1 instead of EX2
6069 // Eg. LSLV x0, x1, x2
6070 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6071 %{
6072 single_instruction;
6073 dst : EX1(write);
6074 src1 : ISS(read);
6075 src2 : ISS(read);
6076 INS01 : ISS;
6077 ALU : EX1;
6078 %}
6079
6080 // Integer ALU reg-reg operation with extract
6081 // As for _vshift above, but result generated in EX2
6082 // Eg. EXTR x0, x1, x2, #N
6083 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6084 %{
6085 single_instruction;
6086 dst : EX2(write);
6087 src1 : ISS(read);
6088 src2 : ISS(read);
6089 INS1 : ISS; // Can only dual issue as Instruction 1
6090 ALU : EX1;
6091 %}
6092
6093 // Integer ALU reg operation
6094 // Eg. NEG x0, x1
6095 pipe_class ialu_reg(iRegI dst, iRegI src)
6096 %{
6097 single_instruction;
6098 dst : EX2(write);
6099 src : EX1(read);
6100 INS01 : ISS;
6101 ALU : EX2;
6102 %}
6103
6104 // Integer ALU reg mmediate operation
6105 // Eg. ADD x0, x1, #N
6106 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6107 %{
6108 single_instruction;
6109 dst : EX2(write);
6110 src1 : EX1(read);
6111 INS01 : ISS;
6112 ALU : EX2;
6113 %}
6114
6115 // Integer ALU immediate operation (no source operands)
6116 // Eg. MOV x0, #N
6117 pipe_class ialu_imm(iRegI dst)
6118 %{
6119 single_instruction;
6120 dst : EX1(write);
6121 INS01 : ISS;
6122 ALU : EX1;
6123 %}
6124
6125 //------- Compare operation -------------------------------
6126
6127 // Compare reg-reg
6128 // Eg. CMP x0, x1
6129 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6130 %{
6131 single_instruction;
6132 // fixed_latency(16);
6133 cr : EX2(write);
6134 op1 : EX1(read);
6135 op2 : EX1(read);
6136 INS01 : ISS;
6137 ALU : EX2;
6138 %}
6139
6140 // Compare reg-reg
6141 // Eg. CMP x0, #N
6142 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6143 %{
6144 single_instruction;
6145 // fixed_latency(16);
6146 cr : EX2(write);
6147 op1 : EX1(read);
6148 INS01 : ISS;
6149 ALU : EX2;
6150 %}
6151
6152 //------- Conditional instructions ------------------------
6153
6154 // Conditional no operands
6155 // Eg. CSINC x0, zr, zr, <cond>
6156 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6157 %{
6158 single_instruction;
6159 cr : EX1(read);
6160 dst : EX2(write);
6161 INS01 : ISS;
6162 ALU : EX2;
6163 %}
6164
6165 // Conditional 2 operand
6166 // EG. CSEL X0, X1, X2, <cond>
6167 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6168 %{
6169 single_instruction;
6170 cr : EX1(read);
6171 src1 : EX1(read);
6172 src2 : EX1(read);
6173 dst : EX2(write);
6174 INS01 : ISS;
6175 ALU : EX2;
6176 %}
6177
6178 // Conditional 2 operand
6179 // EG. CSEL X0, X1, X2, <cond>
6180 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6181 %{
6182 single_instruction;
6183 cr : EX1(read);
6184 src : EX1(read);
6185 dst : EX2(write);
6186 INS01 : ISS;
6187 ALU : EX2;
6188 %}
6189
6190 //------- Multiply pipeline operations --------------------
6191
6192 // Multiply reg-reg
6193 // Eg. MUL w0, w1, w2
6194 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6195 %{
6196 single_instruction;
6197 dst : WR(write);
6198 src1 : ISS(read);
6199 src2 : ISS(read);
6200 INS01 : ISS;
6201 MAC : WR;
6202 %}
6203
6204 // Multiply accumulate
6205 // Eg. MADD w0, w1, w2, w3
6206 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6207 %{
6208 single_instruction;
6209 dst : WR(write);
6210 src1 : ISS(read);
6211 src2 : ISS(read);
6212 src3 : ISS(read);
6213 INS01 : ISS;
6214 MAC : WR;
6215 %}
6216
6217 // Eg. MUL w0, w1, w2
6218 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6219 %{
6220 single_instruction;
6221 fixed_latency(3); // Maximum latency for 64 bit mul
6222 dst : WR(write);
6223 src1 : ISS(read);
6224 src2 : ISS(read);
6225 INS01 : ISS;
6226 MAC : WR;
6227 %}
6228
6229 // Multiply accumulate
6230 // Eg. MADD w0, w1, w2, w3
6231 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6232 %{
6233 single_instruction;
6234 fixed_latency(3); // Maximum latency for 64 bit mul
6235 dst : WR(write);
6236 src1 : ISS(read);
6237 src2 : ISS(read);
6238 src3 : ISS(read);
6239 INS01 : ISS;
6240 MAC : WR;
6241 %}
6242
6243 //------- Divide pipeline operations --------------------
6244
6245 // Eg. SDIV w0, w1, w2
6246 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6247 %{
6248 single_instruction;
6249 fixed_latency(8); // Maximum latency for 32 bit divide
6250 dst : WR(write);
6251 src1 : ISS(read);
6252 src2 : ISS(read);
6253 INS0 : ISS; // Can only dual issue as instruction 0
6254 DIV : WR;
6255 %}
6256
6257 // Eg. SDIV x0, x1, x2
6258 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6259 %{
6260 single_instruction;
6261 fixed_latency(16); // Maximum latency for 64 bit divide
6262 dst : WR(write);
6263 src1 : ISS(read);
6264 src2 : ISS(read);
6265 INS0 : ISS; // Can only dual issue as instruction 0
6266 DIV : WR;
6267 %}
6268
6269 //------- Load pipeline operations ------------------------
6270
6271 // Load - prefetch
6272 // Eg. PFRM <mem>
6273 pipe_class iload_prefetch(memory mem)
6274 %{
6275 single_instruction;
6276 mem : ISS(read);
6277 INS01 : ISS;
6278 LDST : WR;
6279 %}
6280
6281 // Load - reg, mem
6282 // Eg. LDR x0, <mem>
6283 pipe_class iload_reg_mem(iRegI dst, memory mem)
6284 %{
6285 single_instruction;
6286 dst : WR(write);
6287 mem : ISS(read);
6288 INS01 : ISS;
6289 LDST : WR;
6290 %}
6291
6292 // Load - reg, reg
6293 // Eg. LDR x0, [sp, x1]
6294 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6295 %{
6296 single_instruction;
6297 dst : WR(write);
6298 src : ISS(read);
6299 INS01 : ISS;
6300 LDST : WR;
6301 %}
6302
6303 //------- Store pipeline operations -----------------------
6304
6305 // Store - zr, mem
6306 // Eg. STR zr, <mem>
6307 pipe_class istore_mem(memory mem)
6308 %{
6309 single_instruction;
6310 mem : ISS(read);
6311 INS01 : ISS;
6312 LDST : WR;
6313 %}
6314
6315 // Store - reg, mem
6316 // Eg. STR x0, <mem>
6317 pipe_class istore_reg_mem(iRegI src, memory mem)
6318 %{
6319 single_instruction;
6320 mem : ISS(read);
6321 src : EX2(read);
6322 INS01 : ISS;
6323 LDST : WR;
6324 %}
6325
6326 // Store - reg, reg
6327 // Eg. STR x0, [sp, x1]
6328 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6329 %{
6330 single_instruction;
6331 dst : ISS(read);
6332 src : EX2(read);
6333 INS01 : ISS;
6334 LDST : WR;
6335 %}
6336
6337 //------- Store pipeline operations -----------------------
6338
6339 // Branch
6340 pipe_class pipe_branch()
6341 %{
6342 single_instruction;
6343 INS01 : ISS;
6344 BRANCH : EX1;
6345 %}
6346
6347 // Conditional branch
6348 pipe_class pipe_branch_cond(rFlagsReg cr)
6349 %{
6350 single_instruction;
6351 cr : EX1(read);
6352 INS01 : ISS;
6353 BRANCH : EX1;
6354 %}
6355
6356 // Compare & Branch
6357 // EG. CBZ/CBNZ
6358 pipe_class pipe_cmp_branch(iRegI op1)
6359 %{
6360 single_instruction;
6361 op1 : EX1(read);
6362 INS01 : ISS;
6363 BRANCH : EX1;
6364 %}
6365
6366 //------- Synchronisation operations ----------------------
6367
6368 // Any operation requiring serialization.
6369 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6370 pipe_class pipe_serial()
6371 %{
6372 single_instruction;
6373 force_serialization;
6374 fixed_latency(16);
6375 INS01 : ISS(2); // Cannot dual issue with any other instruction
6376 LDST : WR;
6377 %}
6378
6379 // Generic big/slow expanded idiom - also serialized
6380 pipe_class pipe_slow()
6381 %{
6382 instruction_count(10);
6383 multiple_bundles;
6384 force_serialization;
6385 fixed_latency(16);
6386 INS01 : ISS(2); // Cannot dual issue with any other instruction
6387 LDST : WR;
6388 %}
6389
6390 // Empty pipeline class
6391 pipe_class pipe_class_empty()
6392 %{
6393 single_instruction;
6394 fixed_latency(0);
6395 %}
6396
6397 // Default pipeline class.
6398 pipe_class pipe_class_default()
6399 %{
6400 single_instruction;
6401 fixed_latency(2);
6402 %}
6403
6404 // Pipeline class for compares.
6405 pipe_class pipe_class_compare()
6406 %{
6407 single_instruction;
6408 fixed_latency(16);
6409 %}
6410
6411 // Pipeline class for memory operations.
6412 pipe_class pipe_class_memory()
6413 %{
6414 single_instruction;
6415 fixed_latency(16);
6416 %}
6417
6418 // Pipeline class for call.
6419 pipe_class pipe_class_call()
6420 %{
6421 single_instruction;
6422 fixed_latency(100);
6423 %}
6424
6425 // Define the class for the Nop node.
6426 define %{
6427 MachNop = pipe_class_empty;
6428 %}
6429
6430 %}
6431 //----------INSTRUCTIONS-------------------------------------------------------
6432 //
6433 // match -- States which machine-independent subtree may be replaced
6434 // by this instruction.
6435 // ins_cost -- The estimated cost of this instruction is used by instruction
6436 // selection to identify a minimum cost tree of machine
6437 // instructions that matches a tree of machine-independent
6438 // instructions.
6439 // format -- A string providing the disassembly for this instruction.
6440 // The value of an instruction's operand may be inserted
6441 // by referring to it with a '$' prefix.
6442 // opcode -- Three instruction opcodes may be provided. These are referred
6443 // to within an encode class as $primary, $secondary, and $tertiary
6444 // rrspectively. The primary opcode is commonly used to
6445 // indicate the type of machine instruction, while secondary
6446 // and tertiary are often used for prefix options or addressing
6447 // modes.
6448 // ins_encode -- A list of encode classes with parameters. The encode class
6449 // name must have been defined in an 'enc_class' specification
6450 // in the encode section of the architecture description.
6451
6452 // ============================================================================
6453 // Memory (Load/Store) Instructions
6454
6455 // Load Instructions
6456
6457 // Load Byte (8 bit signed)
6458 instruct loadB(iRegINoSp dst, memory1 mem)
6459 %{
6460 match(Set dst (LoadB mem));
6461 predicate(!needs_acquiring_load(n));
6462
6463 ins_cost(4 * INSN_COST);
6464 format %{ "ldrsbw $dst, $mem\t# byte" %}
6465
6466 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6467
6468 ins_pipe(iload_reg_mem);
6469 %}
6470
6471 // Load Byte (8 bit signed) into long
6472 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6473 %{
6474 match(Set dst (ConvI2L (LoadB mem)));
6475 predicate(!needs_acquiring_load(n->in(1)));
6476
6477 ins_cost(4 * INSN_COST);
6478 format %{ "ldrsb $dst, $mem\t# byte" %}
6479
6480 ins_encode(aarch64_enc_ldrsb(dst, mem));
6481
6482 ins_pipe(iload_reg_mem);
6483 %}
6484
6485 // Load Byte (8 bit unsigned)
6486 instruct loadUB(iRegINoSp dst, memory1 mem)
6487 %{
6488 match(Set dst (LoadUB mem));
6489 predicate(!needs_acquiring_load(n));
6490
6491 ins_cost(4 * INSN_COST);
6492 format %{ "ldrbw $dst, $mem\t# byte" %}
6493
6494 ins_encode(aarch64_enc_ldrb(dst, mem));
6495
6496 ins_pipe(iload_reg_mem);
6497 %}
6498
6499 // Load Byte (8 bit unsigned) into long
6500 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6501 %{
6502 match(Set dst (ConvI2L (LoadUB mem)));
6503 predicate(!needs_acquiring_load(n->in(1)));
6504
6505 ins_cost(4 * INSN_COST);
6506 format %{ "ldrb $dst, $mem\t# byte" %}
6507
6508 ins_encode(aarch64_enc_ldrb(dst, mem));
6509
6510 ins_pipe(iload_reg_mem);
6511 %}
6512
6513 // Load Short (16 bit signed)
6514 instruct loadS(iRegINoSp dst, memory2 mem)
6515 %{
6516 match(Set dst (LoadS mem));
6517 predicate(!needs_acquiring_load(n));
6518
6519 ins_cost(4 * INSN_COST);
6520 format %{ "ldrshw $dst, $mem\t# short" %}
6521
6522 ins_encode(aarch64_enc_ldrshw(dst, mem));
6523
6524 ins_pipe(iload_reg_mem);
6525 %}
6526
6527 // Load Short (16 bit signed) into long
6528 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6529 %{
6530 match(Set dst (ConvI2L (LoadS mem)));
6531 predicate(!needs_acquiring_load(n->in(1)));
6532
6533 ins_cost(4 * INSN_COST);
6534 format %{ "ldrsh $dst, $mem\t# short" %}
6535
6536 ins_encode(aarch64_enc_ldrsh(dst, mem));
6537
6538 ins_pipe(iload_reg_mem);
6539 %}
6540
6541 // Load Char (16 bit unsigned)
6542 instruct loadUS(iRegINoSp dst, memory2 mem)
6543 %{
6544 match(Set dst (LoadUS mem));
6545 predicate(!needs_acquiring_load(n));
6546
6547 ins_cost(4 * INSN_COST);
6548 format %{ "ldrh $dst, $mem\t# short" %}
6549
6550 ins_encode(aarch64_enc_ldrh(dst, mem));
6551
6552 ins_pipe(iload_reg_mem);
6553 %}
6554
6555 // Load Short/Char (16 bit unsigned) into long
6556 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6557 %{
6558 match(Set dst (ConvI2L (LoadUS mem)));
6559 predicate(!needs_acquiring_load(n->in(1)));
6560
6561 ins_cost(4 * INSN_COST);
6562 format %{ "ldrh $dst, $mem\t# short" %}
6563
6564 ins_encode(aarch64_enc_ldrh(dst, mem));
6565
6566 ins_pipe(iload_reg_mem);
6567 %}
6568
6569 // Load Integer (32 bit signed)
6570 instruct loadI(iRegINoSp dst, memory4 mem)
6571 %{
6572 match(Set dst (LoadI mem));
6573 predicate(!needs_acquiring_load(n));
6574
6575 ins_cost(4 * INSN_COST);
6576 format %{ "ldrw $dst, $mem\t# int" %}
6577
6578 ins_encode(aarch64_enc_ldrw(dst, mem));
6579
6580 ins_pipe(iload_reg_mem);
6581 %}
6582
6583 // Load Integer (32 bit signed) into long
6584 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6585 %{
6586 match(Set dst (ConvI2L (LoadI mem)));
6587 predicate(!needs_acquiring_load(n->in(1)));
6588
6589 ins_cost(4 * INSN_COST);
6590 format %{ "ldrsw $dst, $mem\t# int" %}
6591
6592 ins_encode(aarch64_enc_ldrsw(dst, mem));
6593
6594 ins_pipe(iload_reg_mem);
6595 %}
6596
6597 // Load Integer (32 bit unsigned) into long
6598 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6599 %{
6600 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6601 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6602
6603 ins_cost(4 * INSN_COST);
6604 format %{ "ldrw $dst, $mem\t# int" %}
6605
6606 ins_encode(aarch64_enc_ldrw(dst, mem));
6607
6608 ins_pipe(iload_reg_mem);
6609 %}
6610
6611 // Load Long (64 bit signed)
6612 instruct loadL(iRegLNoSp dst, memory8 mem)
6613 %{
6614 match(Set dst (LoadL mem));
6615 predicate(!needs_acquiring_load(n));
6616
6617 ins_cost(4 * INSN_COST);
6618 format %{ "ldr $dst, $mem\t# int" %}
6619
6620 ins_encode(aarch64_enc_ldr(dst, mem));
6621
6622 ins_pipe(iload_reg_mem);
6623 %}
6624
6625 // Load Range
6626 instruct loadRange(iRegINoSp dst, memory4 mem)
6627 %{
6628 match(Set dst (LoadRange mem));
6629
6630 ins_cost(4 * INSN_COST);
6631 format %{ "ldrw $dst, $mem\t# range" %}
6632
6633 ins_encode(aarch64_enc_ldrw(dst, mem));
6634
6635 ins_pipe(iload_reg_mem);
6636 %}
6637
6638 // Load Pointer
6639 instruct loadP(iRegPNoSp dst, memory8 mem)
6640 %{
6641 match(Set dst (LoadP mem));
6642 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6643
6644 ins_cost(4 * INSN_COST);
6645 format %{ "ldr $dst, $mem\t# ptr" %}
6646
6647 ins_encode(aarch64_enc_ldr(dst, mem));
6648
6649 ins_pipe(iload_reg_mem);
6650 %}
6651
6652 // Load Compressed Pointer
6653 instruct loadN(iRegNNoSp dst, memory4 mem)
6654 %{
6655 match(Set dst (LoadN mem));
6656 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6657
6658 ins_cost(4 * INSN_COST);
6659 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6660
6661 ins_encode(aarch64_enc_ldrw(dst, mem));
6662
6663 ins_pipe(iload_reg_mem);
6664 %}
6665
6666 // Load Klass Pointer
6667 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6668 %{
6669 match(Set dst (LoadKlass mem));
6670 predicate(!needs_acquiring_load(n));
6671
6672 ins_cost(4 * INSN_COST);
6673 format %{ "ldr $dst, $mem\t# class" %}
6674
6675 ins_encode(aarch64_enc_ldr(dst, mem));
6676
6677 ins_pipe(iload_reg_mem);
6678 %}
6679
6680 // Load Narrow Klass Pointer
6681 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6682 %{
6683 match(Set dst (LoadNKlass mem));
6684 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6685
6686 ins_cost(4 * INSN_COST);
6687 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6688
6689 ins_encode(aarch64_enc_ldrw(dst, mem));
6690
6691 ins_pipe(iload_reg_mem);
6692 %}
6693
6694 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6695 %{
6696 match(Set dst (LoadNKlass mem));
6697 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6698
6699 ins_cost(4 * INSN_COST);
6700 format %{
6701 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6702 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6703 %}
6704 ins_encode %{
6705 // inlined aarch64_enc_ldrw
6706 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6707 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6708 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6709 %}
6710 ins_pipe(iload_reg_mem);
6711 %}
6712
6713 // Load Float
6714 instruct loadF(vRegF dst, memory4 mem)
6715 %{
6716 match(Set dst (LoadF mem));
6717 predicate(!needs_acquiring_load(n));
6718
6719 ins_cost(4 * INSN_COST);
6720 format %{ "ldrs $dst, $mem\t# float" %}
6721
6722 ins_encode( aarch64_enc_ldrs(dst, mem) );
6723
6724 ins_pipe(pipe_class_memory);
6725 %}
6726
6727 // Load Double
6728 instruct loadD(vRegD dst, memory8 mem)
6729 %{
6730 match(Set dst (LoadD mem));
6731 predicate(!needs_acquiring_load(n));
6732
6733 ins_cost(4 * INSN_COST);
6734 format %{ "ldrd $dst, $mem\t# double" %}
6735
6736 ins_encode( aarch64_enc_ldrd(dst, mem) );
6737
6738 ins_pipe(pipe_class_memory);
6739 %}
6740
6741
6742 // Load Int Constant
6743 instruct loadConI(iRegINoSp dst, immI src)
6744 %{
6745 match(Set dst src);
6746
6747 ins_cost(INSN_COST);
6748 format %{ "mov $dst, $src\t# int" %}
6749
6750 ins_encode( aarch64_enc_movw_imm(dst, src) );
6751
6752 ins_pipe(ialu_imm);
6753 %}
6754
6755 // Load Long Constant
6756 instruct loadConL(iRegLNoSp dst, immL src)
6757 %{
6758 match(Set dst src);
6759
6760 ins_cost(INSN_COST);
6761 format %{ "mov $dst, $src\t# long" %}
6762
6763 ins_encode( aarch64_enc_mov_imm(dst, src) );
6764
6765 ins_pipe(ialu_imm);
6766 %}
6767
6768 // Load Pointer Constant
6769
6770 instruct loadConP(iRegPNoSp dst, immP con)
6771 %{
6772 match(Set dst con);
6773
6774 ins_cost(INSN_COST * 4);
6775 format %{
6776 "mov $dst, $con\t# ptr\n\t"
6777 %}
6778
6779 ins_encode(aarch64_enc_mov_p(dst, con));
6780
6781 ins_pipe(ialu_imm);
6782 %}
6783
6784 // Load Null Pointer Constant
6785
6786 instruct loadConP0(iRegPNoSp dst, immP0 con)
6787 %{
6788 match(Set dst con);
6789
6790 ins_cost(INSN_COST);
6791 format %{ "mov $dst, $con\t# nullptr ptr" %}
6792
6793 ins_encode(aarch64_enc_mov_p0(dst, con));
6794
6795 ins_pipe(ialu_imm);
6796 %}
6797
6798 // Load Pointer Constant One
6799
6800 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6801 %{
6802 match(Set dst con);
6803
6804 ins_cost(INSN_COST);
6805 format %{ "mov $dst, $con\t# nullptr ptr" %}
6806
6807 ins_encode(aarch64_enc_mov_p1(dst, con));
6808
6809 ins_pipe(ialu_imm);
6810 %}
6811
6812 // Load Byte Map Base Constant
6813
6814 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6815 %{
6816 match(Set dst con);
6817
6818 ins_cost(INSN_COST);
6819 format %{ "adr $dst, $con\t# Byte Map Base" %}
6820
6821 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6822
6823 ins_pipe(ialu_imm);
6824 %}
6825
6826 // Load Narrow Pointer Constant
6827
6828 instruct loadConN(iRegNNoSp dst, immN con)
6829 %{
6830 match(Set dst con);
6831
6832 ins_cost(INSN_COST * 4);
6833 format %{ "mov $dst, $con\t# compressed ptr" %}
6834
6835 ins_encode(aarch64_enc_mov_n(dst, con));
6836
6837 ins_pipe(ialu_imm);
6838 %}
6839
6840 // Load Narrow Null Pointer Constant
6841
6842 instruct loadConN0(iRegNNoSp dst, immN0 con)
6843 %{
6844 match(Set dst con);
6845
6846 ins_cost(INSN_COST);
6847 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6848
6849 ins_encode(aarch64_enc_mov_n0(dst, con));
6850
6851 ins_pipe(ialu_imm);
6852 %}
6853
6854 // Load Narrow Klass Constant
6855
6856 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6857 %{
6858 match(Set dst con);
6859
6860 ins_cost(INSN_COST);
6861 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6862
6863 ins_encode(aarch64_enc_mov_nk(dst, con));
6864
6865 ins_pipe(ialu_imm);
6866 %}
6867
6868 // Load Packed Float Constant
6869
6870 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6871 match(Set dst con);
6872 ins_cost(INSN_COST * 4);
6873 format %{ "fmovs $dst, $con"%}
6874 ins_encode %{
6875 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6876 %}
6877
6878 ins_pipe(fp_imm_s);
6879 %}
6880
6881 // Load Float Constant
6882
6883 instruct loadConF(vRegF dst, immF con) %{
6884 match(Set dst con);
6885
6886 ins_cost(INSN_COST * 4);
6887
6888 format %{
6889 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6890 %}
6891
6892 ins_encode %{
6893 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6894 %}
6895
6896 ins_pipe(fp_load_constant_s);
6897 %}
6898
6899 // Load Packed Double Constant
6900
6901 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6902 match(Set dst con);
6903 ins_cost(INSN_COST);
6904 format %{ "fmovd $dst, $con"%}
6905 ins_encode %{
6906 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6907 %}
6908
6909 ins_pipe(fp_imm_d);
6910 %}
6911
6912 // Load Double Constant
6913
6914 instruct loadConD(vRegD dst, immD con) %{
6915 match(Set dst con);
6916
6917 ins_cost(INSN_COST * 5);
6918 format %{
6919 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6920 %}
6921
6922 ins_encode %{
6923 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6924 %}
6925
6926 ins_pipe(fp_load_constant_d);
6927 %}
6928
6929 // Store Instructions
6930
6931 // Store Byte
6932 instruct storeB(iRegIorL2I src, memory1 mem)
6933 %{
6934 match(Set mem (StoreB mem src));
6935 predicate(!needs_releasing_store(n));
6936
6937 ins_cost(INSN_COST);
6938 format %{ "strb $src, $mem\t# byte" %}
6939
6940 ins_encode(aarch64_enc_strb(src, mem));
6941
6942 ins_pipe(istore_reg_mem);
6943 %}
6944
6945
6946 instruct storeimmB0(immI0 zero, memory1 mem)
6947 %{
6948 match(Set mem (StoreB mem zero));
6949 predicate(!needs_releasing_store(n));
6950
6951 ins_cost(INSN_COST);
6952 format %{ "strb rscractch2, $mem\t# byte" %}
6953
6954 ins_encode(aarch64_enc_strb0(mem));
6955
6956 ins_pipe(istore_mem);
6957 %}
6958
6959 // Store Char/Short
6960 instruct storeC(iRegIorL2I src, memory2 mem)
6961 %{
6962 match(Set mem (StoreC mem src));
6963 predicate(!needs_releasing_store(n));
6964
6965 ins_cost(INSN_COST);
6966 format %{ "strh $src, $mem\t# short" %}
6967
6968 ins_encode(aarch64_enc_strh(src, mem));
6969
6970 ins_pipe(istore_reg_mem);
6971 %}
6972
6973 instruct storeimmC0(immI0 zero, memory2 mem)
6974 %{
6975 match(Set mem (StoreC mem zero));
6976 predicate(!needs_releasing_store(n));
6977
6978 ins_cost(INSN_COST);
6979 format %{ "strh zr, $mem\t# short" %}
6980
6981 ins_encode(aarch64_enc_strh0(mem));
6982
6983 ins_pipe(istore_mem);
6984 %}
6985
6986 // Store Integer
6987
6988 instruct storeI(iRegIorL2I src, memory4 mem)
6989 %{
6990 match(Set mem(StoreI mem src));
6991 predicate(!needs_releasing_store(n));
6992
6993 ins_cost(INSN_COST);
6994 format %{ "strw $src, $mem\t# int" %}
6995
6996 ins_encode(aarch64_enc_strw(src, mem));
6997
6998 ins_pipe(istore_reg_mem);
6999 %}
7000
7001 instruct storeimmI0(immI0 zero, memory4 mem)
7002 %{
7003 match(Set mem(StoreI mem zero));
7004 predicate(!needs_releasing_store(n));
7005
7006 ins_cost(INSN_COST);
7007 format %{ "strw zr, $mem\t# int" %}
7008
7009 ins_encode(aarch64_enc_strw0(mem));
7010
7011 ins_pipe(istore_mem);
7012 %}
7013
7014 // Store Long (64 bit signed)
7015 instruct storeL(iRegL src, memory8 mem)
7016 %{
7017 match(Set mem (StoreL mem src));
7018 predicate(!needs_releasing_store(n));
7019
7020 ins_cost(INSN_COST);
7021 format %{ "str $src, $mem\t# int" %}
7022
7023 ins_encode(aarch64_enc_str(src, mem));
7024
7025 ins_pipe(istore_reg_mem);
7026 %}
7027
7028 // Store Long (64 bit signed)
7029 instruct storeimmL0(immL0 zero, memory8 mem)
7030 %{
7031 match(Set mem (StoreL mem zero));
7032 predicate(!needs_releasing_store(n));
7033
7034 ins_cost(INSN_COST);
7035 format %{ "str zr, $mem\t# int" %}
7036
7037 ins_encode(aarch64_enc_str0(mem));
7038
7039 ins_pipe(istore_mem);
7040 %}
7041
7042 // Store Pointer
7043 instruct storeP(iRegP src, memory8 mem)
7044 %{
7045 match(Set mem (StoreP mem src));
7046 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7047
7048 ins_cost(INSN_COST);
7049 format %{ "str $src, $mem\t# ptr" %}
7050
7051 ins_encode(aarch64_enc_str(src, mem));
7052
7053 ins_pipe(istore_reg_mem);
7054 %}
7055
7056 // Store Pointer
7057 instruct storeimmP0(immP0 zero, memory8 mem)
7058 %{
7059 match(Set mem (StoreP mem zero));
7060 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7061
7062 ins_cost(INSN_COST);
7063 format %{ "str zr, $mem\t# ptr" %}
7064
7065 ins_encode(aarch64_enc_str0(mem));
7066
7067 ins_pipe(istore_mem);
7068 %}
7069
7070 // Store Compressed Pointer
7071 instruct storeN(iRegN src, memory4 mem)
7072 %{
7073 match(Set mem (StoreN mem src));
7074 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7075
7076 ins_cost(INSN_COST);
7077 format %{ "strw $src, $mem\t# compressed ptr" %}
7078
7079 ins_encode(aarch64_enc_strw(src, mem));
7080
7081 ins_pipe(istore_reg_mem);
7082 %}
7083
7084 instruct storeImmN0(immN0 zero, memory4 mem)
7085 %{
7086 match(Set mem (StoreN mem zero));
7087 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7088
7089 ins_cost(INSN_COST);
7090 format %{ "strw zr, $mem\t# compressed ptr" %}
7091
7092 ins_encode(aarch64_enc_strw0(mem));
7093
7094 ins_pipe(istore_mem);
7095 %}
7096
7097 // Store Float
7098 instruct storeF(vRegF src, memory4 mem)
7099 %{
7100 match(Set mem (StoreF mem src));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "strs $src, $mem\t# float" %}
7105
7106 ins_encode( aarch64_enc_strs(src, mem) );
7107
7108 ins_pipe(pipe_class_memory);
7109 %}
7110
7111 // TODO
7112 // implement storeImmF0 and storeFImmPacked
7113
7114 // Store Double
7115 instruct storeD(vRegD src, memory8 mem)
7116 %{
7117 match(Set mem (StoreD mem src));
7118 predicate(!needs_releasing_store(n));
7119
7120 ins_cost(INSN_COST);
7121 format %{ "strd $src, $mem\t# double" %}
7122
7123 ins_encode( aarch64_enc_strd(src, mem) );
7124
7125 ins_pipe(pipe_class_memory);
7126 %}
7127
7128 // Store Compressed Klass Pointer
7129 instruct storeNKlass(iRegN src, memory4 mem)
7130 %{
7131 predicate(!needs_releasing_store(n));
7132 match(Set mem (StoreNKlass mem src));
7133
7134 ins_cost(INSN_COST);
7135 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7136
7137 ins_encode(aarch64_enc_strw(src, mem));
7138
7139 ins_pipe(istore_reg_mem);
7140 %}
7141
7142 // TODO
7143 // implement storeImmD0 and storeDImmPacked
7144
7145 // prefetch instructions
7146 // Must be safe to execute with invalid address (cannot fault).
7147
7148 instruct prefetchalloc( memory8 mem ) %{
7149 match(PrefetchAllocation mem);
7150
7151 ins_cost(INSN_COST);
7152 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7153
7154 ins_encode( aarch64_enc_prefetchw(mem) );
7155
7156 ins_pipe(iload_prefetch);
7157 %}
7158
7159 // ---------------- volatile loads and stores ----------------
7160
7161 // Load Byte (8 bit signed)
7162 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7163 %{
7164 match(Set dst (LoadB mem));
7165
7166 ins_cost(VOLATILE_REF_COST);
7167 format %{ "ldarsb $dst, $mem\t# byte" %}
7168
7169 ins_encode(aarch64_enc_ldarsb(dst, mem));
7170
7171 ins_pipe(pipe_serial);
7172 %}
7173
7174 // Load Byte (8 bit signed) into long
7175 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7176 %{
7177 match(Set dst (ConvI2L (LoadB mem)));
7178
7179 ins_cost(VOLATILE_REF_COST);
7180 format %{ "ldarsb $dst, $mem\t# byte" %}
7181
7182 ins_encode(aarch64_enc_ldarsb(dst, mem));
7183
7184 ins_pipe(pipe_serial);
7185 %}
7186
7187 // Load Byte (8 bit unsigned)
7188 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7189 %{
7190 match(Set dst (LoadUB mem));
7191
7192 ins_cost(VOLATILE_REF_COST);
7193 format %{ "ldarb $dst, $mem\t# byte" %}
7194
7195 ins_encode(aarch64_enc_ldarb(dst, mem));
7196
7197 ins_pipe(pipe_serial);
7198 %}
7199
7200 // Load Byte (8 bit unsigned) into long
7201 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7202 %{
7203 match(Set dst (ConvI2L (LoadUB mem)));
7204
7205 ins_cost(VOLATILE_REF_COST);
7206 format %{ "ldarb $dst, $mem\t# byte" %}
7207
7208 ins_encode(aarch64_enc_ldarb(dst, mem));
7209
7210 ins_pipe(pipe_serial);
7211 %}
7212
7213 // Load Short (16 bit signed)
7214 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7215 %{
7216 match(Set dst (LoadS mem));
7217
7218 ins_cost(VOLATILE_REF_COST);
7219 format %{ "ldarshw $dst, $mem\t# short" %}
7220
7221 ins_encode(aarch64_enc_ldarshw(dst, mem));
7222
7223 ins_pipe(pipe_serial);
7224 %}
7225
7226 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7227 %{
7228 match(Set dst (LoadUS mem));
7229
7230 ins_cost(VOLATILE_REF_COST);
7231 format %{ "ldarhw $dst, $mem\t# short" %}
7232
7233 ins_encode(aarch64_enc_ldarhw(dst, mem));
7234
7235 ins_pipe(pipe_serial);
7236 %}
7237
7238 // Load Short/Char (16 bit unsigned) into long
7239 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7240 %{
7241 match(Set dst (ConvI2L (LoadUS mem)));
7242
7243 ins_cost(VOLATILE_REF_COST);
7244 format %{ "ldarh $dst, $mem\t# short" %}
7245
7246 ins_encode(aarch64_enc_ldarh(dst, mem));
7247
7248 ins_pipe(pipe_serial);
7249 %}
7250
7251 // Load Short/Char (16 bit signed) into long
7252 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7253 %{
7254 match(Set dst (ConvI2L (LoadS mem)));
7255
7256 ins_cost(VOLATILE_REF_COST);
7257 format %{ "ldarh $dst, $mem\t# short" %}
7258
7259 ins_encode(aarch64_enc_ldarsh(dst, mem));
7260
7261 ins_pipe(pipe_serial);
7262 %}
7263
7264 // Load Integer (32 bit signed)
7265 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7266 %{
7267 match(Set dst (LoadI mem));
7268
7269 ins_cost(VOLATILE_REF_COST);
7270 format %{ "ldarw $dst, $mem\t# int" %}
7271
7272 ins_encode(aarch64_enc_ldarw(dst, mem));
7273
7274 ins_pipe(pipe_serial);
7275 %}
7276
7277 // Load Integer (32 bit unsigned) into long
7278 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7279 %{
7280 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7281
7282 ins_cost(VOLATILE_REF_COST);
7283 format %{ "ldarw $dst, $mem\t# int" %}
7284
7285 ins_encode(aarch64_enc_ldarw(dst, mem));
7286
7287 ins_pipe(pipe_serial);
7288 %}
7289
7290 // Load Long (64 bit signed)
7291 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7292 %{
7293 match(Set dst (LoadL mem));
7294
7295 ins_cost(VOLATILE_REF_COST);
7296 format %{ "ldar $dst, $mem\t# int" %}
7297
7298 ins_encode(aarch64_enc_ldar(dst, mem));
7299
7300 ins_pipe(pipe_serial);
7301 %}
7302
7303 // Load Pointer
7304 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7305 %{
7306 match(Set dst (LoadP mem));
7307 predicate(n->as_Load()->barrier_data() == 0);
7308
7309 ins_cost(VOLATILE_REF_COST);
7310 format %{ "ldar $dst, $mem\t# ptr" %}
7311
7312 ins_encode(aarch64_enc_ldar(dst, mem));
7313
7314 ins_pipe(pipe_serial);
7315 %}
7316
7317 // Load Compressed Pointer
7318 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7319 %{
7320 match(Set dst (LoadN mem));
7321 predicate(n->as_Load()->barrier_data() == 0);
7322
7323 ins_cost(VOLATILE_REF_COST);
7324 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7325
7326 ins_encode(aarch64_enc_ldarw(dst, mem));
7327
7328 ins_pipe(pipe_serial);
7329 %}
7330
7331 // Load Float
7332 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7333 %{
7334 match(Set dst (LoadF mem));
7335
7336 ins_cost(VOLATILE_REF_COST);
7337 format %{ "ldars $dst, $mem\t# float" %}
7338
7339 ins_encode( aarch64_enc_fldars(dst, mem) );
7340
7341 ins_pipe(pipe_serial);
7342 %}
7343
7344 // Load Double
7345 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7346 %{
7347 match(Set dst (LoadD mem));
7348
7349 ins_cost(VOLATILE_REF_COST);
7350 format %{ "ldard $dst, $mem\t# double" %}
7351
7352 ins_encode( aarch64_enc_fldard(dst, mem) );
7353
7354 ins_pipe(pipe_serial);
7355 %}
7356
7357 // Store Byte
7358 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7359 %{
7360 match(Set mem (StoreB mem src));
7361
7362 ins_cost(VOLATILE_REF_COST);
7363 format %{ "stlrb $src, $mem\t# byte" %}
7364
7365 ins_encode(aarch64_enc_stlrb(src, mem));
7366
7367 ins_pipe(pipe_class_memory);
7368 %}
7369
7370 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7371 %{
7372 match(Set mem (StoreB mem zero));
7373
7374 ins_cost(VOLATILE_REF_COST);
7375 format %{ "stlrb zr, $mem\t# byte" %}
7376
7377 ins_encode(aarch64_enc_stlrb0(mem));
7378
7379 ins_pipe(pipe_class_memory);
7380 %}
7381
7382 // Store Char/Short
7383 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7384 %{
7385 match(Set mem (StoreC mem src));
7386
7387 ins_cost(VOLATILE_REF_COST);
7388 format %{ "stlrh $src, $mem\t# short" %}
7389
7390 ins_encode(aarch64_enc_stlrh(src, mem));
7391
7392 ins_pipe(pipe_class_memory);
7393 %}
7394
7395 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7396 %{
7397 match(Set mem (StoreC mem zero));
7398
7399 ins_cost(VOLATILE_REF_COST);
7400 format %{ "stlrh zr, $mem\t# short" %}
7401
7402 ins_encode(aarch64_enc_stlrh0(mem));
7403
7404 ins_pipe(pipe_class_memory);
7405 %}
7406
7407 // Store Integer
7408
7409 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7410 %{
7411 match(Set mem(StoreI mem src));
7412
7413 ins_cost(VOLATILE_REF_COST);
7414 format %{ "stlrw $src, $mem\t# int" %}
7415
7416 ins_encode(aarch64_enc_stlrw(src, mem));
7417
7418 ins_pipe(pipe_class_memory);
7419 %}
7420
7421 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7422 %{
7423 match(Set mem(StoreI mem zero));
7424
7425 ins_cost(VOLATILE_REF_COST);
7426 format %{ "stlrw zr, $mem\t# int" %}
7427
7428 ins_encode(aarch64_enc_stlrw0(mem));
7429
7430 ins_pipe(pipe_class_memory);
7431 %}
7432
7433 // Store Long (64 bit signed)
7434 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7435 %{
7436 match(Set mem (StoreL mem src));
7437
7438 ins_cost(VOLATILE_REF_COST);
7439 format %{ "stlr $src, $mem\t# int" %}
7440
7441 ins_encode(aarch64_enc_stlr(src, mem));
7442
7443 ins_pipe(pipe_class_memory);
7444 %}
7445
7446 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7447 %{
7448 match(Set mem (StoreL mem zero));
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "stlr zr, $mem\t# int" %}
7452
7453 ins_encode(aarch64_enc_stlr0(mem));
7454
7455 ins_pipe(pipe_class_memory);
7456 %}
7457
7458 // Store Pointer
7459 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set mem (StoreP mem src));
7462 predicate(n->as_Store()->barrier_data() == 0);
7463
7464 ins_cost(VOLATILE_REF_COST);
7465 format %{ "stlr $src, $mem\t# ptr" %}
7466
7467 ins_encode(aarch64_enc_stlr(src, mem));
7468
7469 ins_pipe(pipe_class_memory);
7470 %}
7471
7472 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set mem (StoreP mem zero));
7475 predicate(n->as_Store()->barrier_data() == 0);
7476
7477 ins_cost(VOLATILE_REF_COST);
7478 format %{ "stlr zr, $mem\t# ptr" %}
7479
7480 ins_encode(aarch64_enc_stlr0(mem));
7481
7482 ins_pipe(pipe_class_memory);
7483 %}
7484
7485 // Store Compressed Pointer
7486 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7487 %{
7488 match(Set mem (StoreN mem src));
7489 predicate(n->as_Store()->barrier_data() == 0);
7490
7491 ins_cost(VOLATILE_REF_COST);
7492 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7493
7494 ins_encode(aarch64_enc_stlrw(src, mem));
7495
7496 ins_pipe(pipe_class_memory);
7497 %}
7498
7499 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7500 %{
7501 match(Set mem (StoreN mem zero));
7502 predicate(n->as_Store()->barrier_data() == 0);
7503
7504 ins_cost(VOLATILE_REF_COST);
7505 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7506
7507 ins_encode(aarch64_enc_stlrw0(mem));
7508
7509 ins_pipe(pipe_class_memory);
7510 %}
7511
7512 // Store Float
7513 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7514 %{
7515 match(Set mem (StoreF mem src));
7516
7517 ins_cost(VOLATILE_REF_COST);
7518 format %{ "stlrs $src, $mem\t# float" %}
7519
7520 ins_encode( aarch64_enc_fstlrs(src, mem) );
7521
7522 ins_pipe(pipe_class_memory);
7523 %}
7524
7525 // TODO
7526 // implement storeImmF0 and storeFImmPacked
7527
7528 // Store Double
7529 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem (StoreD mem src));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlrd $src, $mem\t# double" %}
7535
7536 ins_encode( aarch64_enc_fstlrd(src, mem) );
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 // ---------------- end of volatile loads and stores ----------------
7542
7543 instruct cacheWB(indirect addr)
7544 %{
7545 predicate(VM_Version::supports_data_cache_line_flush());
7546 match(CacheWB addr);
7547
7548 ins_cost(100);
7549 format %{"cache wb $addr" %}
7550 ins_encode %{
7551 assert($addr->index_position() < 0, "should be");
7552 assert($addr$$disp == 0, "should be");
7553 __ cache_wb(Address($addr$$base$$Register, 0));
7554 %}
7555 ins_pipe(pipe_slow); // XXX
7556 %}
7557
7558 instruct cacheWBPreSync()
7559 %{
7560 predicate(VM_Version::supports_data_cache_line_flush());
7561 match(CacheWBPreSync);
7562
7563 ins_cost(100);
7564 format %{"cache wb presync" %}
7565 ins_encode %{
7566 __ cache_wbsync(true);
7567 %}
7568 ins_pipe(pipe_slow); // XXX
7569 %}
7570
7571 instruct cacheWBPostSync()
7572 %{
7573 predicate(VM_Version::supports_data_cache_line_flush());
7574 match(CacheWBPostSync);
7575
7576 ins_cost(100);
7577 format %{"cache wb postsync" %}
7578 ins_encode %{
7579 __ cache_wbsync(false);
7580 %}
7581 ins_pipe(pipe_slow); // XXX
7582 %}
7583
7584 // ============================================================================
7585 // BSWAP Instructions
7586
7587 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7588 match(Set dst (ReverseBytesI src));
7589
7590 ins_cost(INSN_COST);
7591 format %{ "revw $dst, $src" %}
7592
7593 ins_encode %{
7594 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7595 %}
7596
7597 ins_pipe(ialu_reg);
7598 %}
7599
7600 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7601 match(Set dst (ReverseBytesL src));
7602
7603 ins_cost(INSN_COST);
7604 format %{ "rev $dst, $src" %}
7605
7606 ins_encode %{
7607 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7608 %}
7609
7610 ins_pipe(ialu_reg);
7611 %}
7612
7613 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7614 match(Set dst (ReverseBytesUS src));
7615
7616 ins_cost(INSN_COST);
7617 format %{ "rev16w $dst, $src" %}
7618
7619 ins_encode %{
7620 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7621 %}
7622
7623 ins_pipe(ialu_reg);
7624 %}
7625
7626 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7627 match(Set dst (ReverseBytesS src));
7628
7629 ins_cost(INSN_COST);
7630 format %{ "rev16w $dst, $src\n\t"
7631 "sbfmw $dst, $dst, #0, #15" %}
7632
7633 ins_encode %{
7634 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7635 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7636 %}
7637
7638 ins_pipe(ialu_reg);
7639 %}
7640
7641 // ============================================================================
7642 // Zero Count Instructions
7643
7644 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7645 match(Set dst (CountLeadingZerosI src));
7646
7647 ins_cost(INSN_COST);
7648 format %{ "clzw $dst, $src" %}
7649 ins_encode %{
7650 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7651 %}
7652
7653 ins_pipe(ialu_reg);
7654 %}
7655
7656 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7657 match(Set dst (CountLeadingZerosL src));
7658
7659 ins_cost(INSN_COST);
7660 format %{ "clz $dst, $src" %}
7661 ins_encode %{
7662 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7663 %}
7664
7665 ins_pipe(ialu_reg);
7666 %}
7667
7668 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7669 match(Set dst (CountTrailingZerosI src));
7670
7671 ins_cost(INSN_COST * 2);
7672 format %{ "rbitw $dst, $src\n\t"
7673 "clzw $dst, $dst" %}
7674 ins_encode %{
7675 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7676 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7677 %}
7678
7679 ins_pipe(ialu_reg);
7680 %}
7681
7682 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7683 match(Set dst (CountTrailingZerosL src));
7684
7685 ins_cost(INSN_COST * 2);
7686 format %{ "rbit $dst, $src\n\t"
7687 "clz $dst, $dst" %}
7688 ins_encode %{
7689 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7690 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7691 %}
7692
7693 ins_pipe(ialu_reg);
7694 %}
7695
7696 //---------- Population Count Instructions -------------------------------------
7697 //
7698
7699 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7700 match(Set dst (PopCountI src));
7701 effect(TEMP tmp);
7702 ins_cost(INSN_COST * 13);
7703
7704 format %{ "movw $src, $src\n\t"
7705 "mov $tmp, $src\t# vector (1D)\n\t"
7706 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7707 "addv $tmp, $tmp\t# vector (8B)\n\t"
7708 "mov $dst, $tmp\t# vector (1D)" %}
7709 ins_encode %{
7710 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7711 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7712 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7713 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7714 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7715 %}
7716
7717 ins_pipe(pipe_class_default);
7718 %}
7719
7720 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7721 match(Set dst (PopCountI (LoadI mem)));
7722 effect(TEMP tmp);
7723 ins_cost(INSN_COST * 13);
7724
7725 format %{ "ldrs $tmp, $mem\n\t"
7726 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7727 "addv $tmp, $tmp\t# vector (8B)\n\t"
7728 "mov $dst, $tmp\t# vector (1D)" %}
7729 ins_encode %{
7730 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7731 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7732 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7733 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7734 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7735 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7736 %}
7737
7738 ins_pipe(pipe_class_default);
7739 %}
7740
7741 // Note: Long.bitCount(long) returns an int.
7742 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7743 match(Set dst (PopCountL src));
7744 effect(TEMP tmp);
7745 ins_cost(INSN_COST * 13);
7746
7747 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7748 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7749 "addv $tmp, $tmp\t# vector (8B)\n\t"
7750 "mov $dst, $tmp\t# vector (1D)" %}
7751 ins_encode %{
7752 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7753 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7754 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7755 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7756 %}
7757
7758 ins_pipe(pipe_class_default);
7759 %}
7760
7761 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7762 match(Set dst (PopCountL (LoadL mem)));
7763 effect(TEMP tmp);
7764 ins_cost(INSN_COST * 13);
7765
7766 format %{ "ldrd $tmp, $mem\n\t"
7767 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7768 "addv $tmp, $tmp\t# vector (8B)\n\t"
7769 "mov $dst, $tmp\t# vector (1D)" %}
7770 ins_encode %{
7771 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7772 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7773 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7774 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7775 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7776 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7777 %}
7778
7779 ins_pipe(pipe_class_default);
7780 %}
7781
7782 // ============================================================================
7783 // VerifyVectorAlignment Instruction
7784
7785 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7786 match(Set addr (VerifyVectorAlignment addr mask));
7787 effect(KILL cr);
7788 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7789 ins_encode %{
7790 Label Lskip;
7791 // check if masked bits of addr are zero
7792 __ tst($addr$$Register, $mask$$constant);
7793 __ br(Assembler::EQ, Lskip);
7794 __ stop("verify_vector_alignment found a misaligned vector memory access");
7795 __ bind(Lskip);
7796 %}
7797 ins_pipe(pipe_slow);
7798 %}
7799
7800 // ============================================================================
7801 // MemBar Instruction
7802
7803 instruct load_fence() %{
7804 match(LoadFence);
7805 ins_cost(VOLATILE_REF_COST);
7806
7807 format %{ "load_fence" %}
7808
7809 ins_encode %{
7810 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7811 %}
7812 ins_pipe(pipe_serial);
7813 %}
7814
7815 instruct unnecessary_membar_acquire() %{
7816 predicate(unnecessary_acquire(n));
7817 match(MemBarAcquire);
7818 ins_cost(0);
7819
7820 format %{ "membar_acquire (elided)" %}
7821
7822 ins_encode %{
7823 __ block_comment("membar_acquire (elided)");
7824 %}
7825
7826 ins_pipe(pipe_class_empty);
7827 %}
7828
7829 instruct membar_acquire() %{
7830 match(MemBarAcquire);
7831 ins_cost(VOLATILE_REF_COST);
7832
7833 format %{ "membar_acquire\n\t"
7834 "dmb ishld" %}
7835
7836 ins_encode %{
7837 __ block_comment("membar_acquire");
7838 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7839 %}
7840
7841 ins_pipe(pipe_serial);
7842 %}
7843
7844
7845 instruct membar_acquire_lock() %{
7846 match(MemBarAcquireLock);
7847 ins_cost(VOLATILE_REF_COST);
7848
7849 format %{ "membar_acquire_lock (elided)" %}
7850
7851 ins_encode %{
7852 __ block_comment("membar_acquire_lock (elided)");
7853 %}
7854
7855 ins_pipe(pipe_serial);
7856 %}
7857
7858 instruct store_fence() %{
7859 match(StoreFence);
7860 ins_cost(VOLATILE_REF_COST);
7861
7862 format %{ "store_fence" %}
7863
7864 ins_encode %{
7865 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7866 %}
7867 ins_pipe(pipe_serial);
7868 %}
7869
7870 instruct unnecessary_membar_release() %{
7871 predicate(unnecessary_release(n));
7872 match(MemBarRelease);
7873 ins_cost(0);
7874
7875 format %{ "membar_release (elided)" %}
7876
7877 ins_encode %{
7878 __ block_comment("membar_release (elided)");
7879 %}
7880 ins_pipe(pipe_serial);
7881 %}
7882
7883 instruct membar_release() %{
7884 match(MemBarRelease);
7885 ins_cost(VOLATILE_REF_COST);
7886
7887 format %{ "membar_release\n\t"
7888 "dmb ishst\n\tdmb ishld" %}
7889
7890 ins_encode %{
7891 __ block_comment("membar_release");
7892 // These will be merged if AlwaysMergeDMB is enabled.
7893 __ membar(Assembler::StoreStore);
7894 __ membar(Assembler::LoadStore);
7895 %}
7896 ins_pipe(pipe_serial);
7897 %}
7898
7899 instruct membar_storestore() %{
7900 match(MemBarStoreStore);
7901 match(StoreStoreFence);
7902 ins_cost(VOLATILE_REF_COST);
7903
7904 format %{ "MEMBAR-store-store" %}
7905
7906 ins_encode %{
7907 __ membar(Assembler::StoreStore);
7908 %}
7909 ins_pipe(pipe_serial);
7910 %}
7911
7912 instruct membar_release_lock() %{
7913 match(MemBarReleaseLock);
7914 ins_cost(VOLATILE_REF_COST);
7915
7916 format %{ "membar_release_lock (elided)" %}
7917
7918 ins_encode %{
7919 __ block_comment("membar_release_lock (elided)");
7920 %}
7921
7922 ins_pipe(pipe_serial);
7923 %}
7924
7925 instruct unnecessary_membar_volatile() %{
7926 predicate(unnecessary_volatile(n));
7927 match(MemBarVolatile);
7928 ins_cost(0);
7929
7930 format %{ "membar_volatile (elided)" %}
7931
7932 ins_encode %{
7933 __ block_comment("membar_volatile (elided)");
7934 %}
7935
7936 ins_pipe(pipe_serial);
7937 %}
7938
7939 instruct membar_volatile() %{
7940 match(MemBarVolatile);
7941 ins_cost(VOLATILE_REF_COST*100);
7942
7943 format %{ "membar_volatile\n\t"
7944 "dmb ish"%}
7945
7946 ins_encode %{
7947 __ block_comment("membar_volatile");
7948 __ membar(Assembler::StoreLoad);
7949 %}
7950
7951 ins_pipe(pipe_serial);
7952 %}
7953
7954 // ============================================================================
7955 // Cast/Convert Instructions
7956
7957 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7958 match(Set dst (CastX2P src));
7959
7960 ins_cost(INSN_COST);
7961 format %{ "mov $dst, $src\t# long -> ptr" %}
7962
7963 ins_encode %{
7964 if ($dst$$reg != $src$$reg) {
7965 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7966 }
7967 %}
7968
7969 ins_pipe(ialu_reg);
7970 %}
7971
7972 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7973 match(Set dst (CastP2X src));
7974
7975 ins_cost(INSN_COST);
7976 format %{ "mov $dst, $src\t# ptr -> long" %}
7977
7978 ins_encode %{
7979 if ($dst$$reg != $src$$reg) {
7980 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7981 }
7982 %}
7983
7984 ins_pipe(ialu_reg);
7985 %}
7986
7987 // Convert oop into int for vectors alignment masking
7988 instruct convP2I(iRegINoSp dst, iRegP src) %{
7989 match(Set dst (ConvL2I (CastP2X src)));
7990
7991 ins_cost(INSN_COST);
7992 format %{ "movw $dst, $src\t# ptr -> int" %}
7993 ins_encode %{
7994 __ movw($dst$$Register, $src$$Register);
7995 %}
7996
7997 ins_pipe(ialu_reg);
7998 %}
7999
8000 // Convert compressed oop into int for vectors alignment masking
8001 // in case of 32bit oops (heap < 4Gb).
8002 instruct convN2I(iRegINoSp dst, iRegN src)
8003 %{
8004 predicate(CompressedOops::shift() == 0);
8005 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8006
8007 ins_cost(INSN_COST);
8008 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8009 ins_encode %{
8010 __ movw($dst$$Register, $src$$Register);
8011 %}
8012
8013 ins_pipe(ialu_reg);
8014 %}
8015
8016
8017 // Convert oop pointer into compressed form
8018 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8019 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8020 match(Set dst (EncodeP src));
8021 effect(KILL cr);
8022 ins_cost(INSN_COST * 3);
8023 format %{ "encode_heap_oop $dst, $src" %}
8024 ins_encode %{
8025 Register s = $src$$Register;
8026 Register d = $dst$$Register;
8027 __ encode_heap_oop(d, s);
8028 %}
8029 ins_pipe(ialu_reg);
8030 %}
8031
8032 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8033 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8034 match(Set dst (EncodeP src));
8035 ins_cost(INSN_COST * 3);
8036 format %{ "encode_heap_oop_not_null $dst, $src" %}
8037 ins_encode %{
8038 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8039 %}
8040 ins_pipe(ialu_reg);
8041 %}
8042
8043 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8044 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8045 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8046 match(Set dst (DecodeN src));
8047 ins_cost(INSN_COST * 3);
8048 format %{ "decode_heap_oop $dst, $src" %}
8049 ins_encode %{
8050 Register s = $src$$Register;
8051 Register d = $dst$$Register;
8052 __ decode_heap_oop(d, s);
8053 %}
8054 ins_pipe(ialu_reg);
8055 %}
8056
8057 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8058 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8059 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8060 match(Set dst (DecodeN src));
8061 ins_cost(INSN_COST * 3);
8062 format %{ "decode_heap_oop_not_null $dst, $src" %}
8063 ins_encode %{
8064 Register s = $src$$Register;
8065 Register d = $dst$$Register;
8066 __ decode_heap_oop_not_null(d, s);
8067 %}
8068 ins_pipe(ialu_reg);
8069 %}
8070
8071 // n.b. AArch64 implementations of encode_klass_not_null and
8072 // decode_klass_not_null do not modify the flags register so, unlike
8073 // Intel, we don't kill CR as a side effect here
8074
8075 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8076 match(Set dst (EncodePKlass src));
8077
8078 ins_cost(INSN_COST * 3);
8079 format %{ "encode_klass_not_null $dst,$src" %}
8080
8081 ins_encode %{
8082 Register src_reg = as_Register($src$$reg);
8083 Register dst_reg = as_Register($dst$$reg);
8084 __ encode_klass_not_null(dst_reg, src_reg);
8085 %}
8086
8087 ins_pipe(ialu_reg);
8088 %}
8089
8090 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8091 match(Set dst (DecodeNKlass src));
8092
8093 ins_cost(INSN_COST * 3);
8094 format %{ "decode_klass_not_null $dst,$src" %}
8095
8096 ins_encode %{
8097 Register src_reg = as_Register($src$$reg);
8098 Register dst_reg = as_Register($dst$$reg);
8099 if (dst_reg != src_reg) {
8100 __ decode_klass_not_null(dst_reg, src_reg);
8101 } else {
8102 __ decode_klass_not_null(dst_reg);
8103 }
8104 %}
8105
8106 ins_pipe(ialu_reg);
8107 %}
8108
8109 instruct checkCastPP(iRegPNoSp dst)
8110 %{
8111 match(Set dst (CheckCastPP dst));
8112
8113 size(0);
8114 format %{ "# checkcastPP of $dst" %}
8115 ins_encode(/* empty encoding */);
8116 ins_pipe(pipe_class_empty);
8117 %}
8118
8119 instruct castPP(iRegPNoSp dst)
8120 %{
8121 match(Set dst (CastPP dst));
8122
8123 size(0);
8124 format %{ "# castPP of $dst" %}
8125 ins_encode(/* empty encoding */);
8126 ins_pipe(pipe_class_empty);
8127 %}
8128
8129 instruct castII(iRegI dst)
8130 %{
8131 match(Set dst (CastII dst));
8132
8133 size(0);
8134 format %{ "# castII of $dst" %}
8135 ins_encode(/* empty encoding */);
8136 ins_cost(0);
8137 ins_pipe(pipe_class_empty);
8138 %}
8139
8140 instruct castLL(iRegL dst)
8141 %{
8142 match(Set dst (CastLL dst));
8143
8144 size(0);
8145 format %{ "# castLL of $dst" %}
8146 ins_encode(/* empty encoding */);
8147 ins_cost(0);
8148 ins_pipe(pipe_class_empty);
8149 %}
8150
8151 instruct castFF(vRegF dst)
8152 %{
8153 match(Set dst (CastFF dst));
8154
8155 size(0);
8156 format %{ "# castFF of $dst" %}
8157 ins_encode(/* empty encoding */);
8158 ins_cost(0);
8159 ins_pipe(pipe_class_empty);
8160 %}
8161
8162 instruct castDD(vRegD dst)
8163 %{
8164 match(Set dst (CastDD dst));
8165
8166 size(0);
8167 format %{ "# castDD of $dst" %}
8168 ins_encode(/* empty encoding */);
8169 ins_cost(0);
8170 ins_pipe(pipe_class_empty);
8171 %}
8172
8173 instruct castVV(vReg dst)
8174 %{
8175 match(Set dst (CastVV dst));
8176
8177 size(0);
8178 format %{ "# castVV of $dst" %}
8179 ins_encode(/* empty encoding */);
8180 ins_cost(0);
8181 ins_pipe(pipe_class_empty);
8182 %}
8183
8184 instruct castVVMask(pRegGov dst)
8185 %{
8186 match(Set dst (CastVV dst));
8187
8188 size(0);
8189 format %{ "# castVV of $dst" %}
8190 ins_encode(/* empty encoding */);
8191 ins_cost(0);
8192 ins_pipe(pipe_class_empty);
8193 %}
8194
8195 // ============================================================================
8196 // Atomic operation instructions
8197 //
8198
8199 // standard CompareAndSwapX when we are using barriers
8200 // these have higher priority than the rules selected by a predicate
8201
8202 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8203 // can't match them
8204
8205 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8206
8207 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8208 ins_cost(2 * VOLATILE_REF_COST);
8209
8210 effect(KILL cr);
8211
8212 format %{
8213 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8214 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8215 %}
8216
8217 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8218 aarch64_enc_cset_eq(res));
8219
8220 ins_pipe(pipe_slow);
8221 %}
8222
8223 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8224
8225 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8226 ins_cost(2 * VOLATILE_REF_COST);
8227
8228 effect(KILL cr);
8229
8230 format %{
8231 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8232 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8233 %}
8234
8235 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8236 aarch64_enc_cset_eq(res));
8237
8238 ins_pipe(pipe_slow);
8239 %}
8240
8241 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8242
8243 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8244 ins_cost(2 * VOLATILE_REF_COST);
8245
8246 effect(KILL cr);
8247
8248 format %{
8249 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8250 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8251 %}
8252
8253 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8254 aarch64_enc_cset_eq(res));
8255
8256 ins_pipe(pipe_slow);
8257 %}
8258
8259 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8260
8261 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8262 ins_cost(2 * VOLATILE_REF_COST);
8263
8264 effect(KILL cr);
8265
8266 format %{
8267 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8268 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8269 %}
8270
8271 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8272 aarch64_enc_cset_eq(res));
8273
8274 ins_pipe(pipe_slow);
8275 %}
8276
8277 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8278
8279 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8280 predicate(n->as_LoadStore()->barrier_data() == 0);
8281 ins_cost(2 * VOLATILE_REF_COST);
8282
8283 effect(KILL cr);
8284
8285 format %{
8286 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8287 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8288 %}
8289
8290 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8291 aarch64_enc_cset_eq(res));
8292
8293 ins_pipe(pipe_slow);
8294 %}
8295
8296 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8297
8298 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8299 predicate(n->as_LoadStore()->barrier_data() == 0);
8300 ins_cost(2 * VOLATILE_REF_COST);
8301
8302 effect(KILL cr);
8303
8304 format %{
8305 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8306 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8307 %}
8308
8309 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8310 aarch64_enc_cset_eq(res));
8311
8312 ins_pipe(pipe_slow);
8313 %}
8314
8315 // alternative CompareAndSwapX when we are eliding barriers
8316
8317 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8318
8319 predicate(needs_acquiring_load_exclusive(n));
8320 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8321 ins_cost(VOLATILE_REF_COST);
8322
8323 effect(KILL cr);
8324
8325 format %{
8326 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8327 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8328 %}
8329
8330 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8331 aarch64_enc_cset_eq(res));
8332
8333 ins_pipe(pipe_slow);
8334 %}
8335
8336 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8337
8338 predicate(needs_acquiring_load_exclusive(n));
8339 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8340 ins_cost(VOLATILE_REF_COST);
8341
8342 effect(KILL cr);
8343
8344 format %{
8345 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8346 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8347 %}
8348
8349 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8350 aarch64_enc_cset_eq(res));
8351
8352 ins_pipe(pipe_slow);
8353 %}
8354
8355 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8356
8357 predicate(needs_acquiring_load_exclusive(n));
8358 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8359 ins_cost(VOLATILE_REF_COST);
8360
8361 effect(KILL cr);
8362
8363 format %{
8364 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8365 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8366 %}
8367
8368 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8369 aarch64_enc_cset_eq(res));
8370
8371 ins_pipe(pipe_slow);
8372 %}
8373
8374 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8375
8376 predicate(needs_acquiring_load_exclusive(n));
8377 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8378 ins_cost(VOLATILE_REF_COST);
8379
8380 effect(KILL cr);
8381
8382 format %{
8383 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8384 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8385 %}
8386
8387 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8388 aarch64_enc_cset_eq(res));
8389
8390 ins_pipe(pipe_slow);
8391 %}
8392
8393 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8394
8395 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8396 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8397 ins_cost(VOLATILE_REF_COST);
8398
8399 effect(KILL cr);
8400
8401 format %{
8402 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8403 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8404 %}
8405
8406 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8407 aarch64_enc_cset_eq(res));
8408
8409 ins_pipe(pipe_slow);
8410 %}
8411
8412 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8413
8414 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8415 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8416 ins_cost(VOLATILE_REF_COST);
8417
8418 effect(KILL cr);
8419
8420 format %{
8421 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8422 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8423 %}
8424
8425 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8426 aarch64_enc_cset_eq(res));
8427
8428 ins_pipe(pipe_slow);
8429 %}
8430
8431
8432 // ---------------------------------------------------------------------
8433
8434 // BEGIN This section of the file is automatically generated. Do not edit --------------
8435
8436 // Sundry CAS operations. Note that release is always true,
8437 // regardless of the memory ordering of the CAS. This is because we
8438 // need the volatile case to be sequentially consistent but there is
8439 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8440 // can't check the type of memory ordering here, so we always emit a
8441 // STLXR.
8442
8443 // This section is generated from cas.m4
8444
8445
8446 // This pattern is generated automatically from cas.m4.
8447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8448 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8449 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8450 ins_cost(2 * VOLATILE_REF_COST);
8451 effect(TEMP_DEF res, KILL cr);
8452 format %{
8453 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8454 %}
8455 ins_encode %{
8456 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8457 Assembler::byte, /*acquire*/ false, /*release*/ true,
8458 /*weak*/ false, $res$$Register);
8459 __ sxtbw($res$$Register, $res$$Register);
8460 %}
8461 ins_pipe(pipe_slow);
8462 %}
8463
8464 // This pattern is generated automatically from cas.m4.
8465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8466 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8467 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8468 ins_cost(2 * VOLATILE_REF_COST);
8469 effect(TEMP_DEF res, KILL cr);
8470 format %{
8471 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8472 %}
8473 ins_encode %{
8474 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8475 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8476 /*weak*/ false, $res$$Register);
8477 __ sxthw($res$$Register, $res$$Register);
8478 %}
8479 ins_pipe(pipe_slow);
8480 %}
8481
8482 // This pattern is generated automatically from cas.m4.
8483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8484 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8485 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8486 ins_cost(2 * VOLATILE_REF_COST);
8487 effect(TEMP_DEF res, KILL cr);
8488 format %{
8489 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8490 %}
8491 ins_encode %{
8492 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8493 Assembler::word, /*acquire*/ false, /*release*/ true,
8494 /*weak*/ false, $res$$Register);
8495 %}
8496 ins_pipe(pipe_slow);
8497 %}
8498
8499 // This pattern is generated automatically from cas.m4.
8500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8501 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8502 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8503 ins_cost(2 * VOLATILE_REF_COST);
8504 effect(TEMP_DEF res, KILL cr);
8505 format %{
8506 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8507 %}
8508 ins_encode %{
8509 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8510 Assembler::xword, /*acquire*/ false, /*release*/ true,
8511 /*weak*/ false, $res$$Register);
8512 %}
8513 ins_pipe(pipe_slow);
8514 %}
8515
8516 // This pattern is generated automatically from cas.m4.
8517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8518 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8519 predicate(n->as_LoadStore()->barrier_data() == 0);
8520 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8521 ins_cost(2 * VOLATILE_REF_COST);
8522 effect(TEMP_DEF res, KILL cr);
8523 format %{
8524 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8525 %}
8526 ins_encode %{
8527 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8528 Assembler::word, /*acquire*/ false, /*release*/ true,
8529 /*weak*/ false, $res$$Register);
8530 %}
8531 ins_pipe(pipe_slow);
8532 %}
8533
8534 // This pattern is generated automatically from cas.m4.
8535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8536 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8537 predicate(n->as_LoadStore()->barrier_data() == 0);
8538 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8539 ins_cost(2 * VOLATILE_REF_COST);
8540 effect(TEMP_DEF res, KILL cr);
8541 format %{
8542 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8543 %}
8544 ins_encode %{
8545 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8546 Assembler::xword, /*acquire*/ false, /*release*/ true,
8547 /*weak*/ false, $res$$Register);
8548 %}
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552 // This pattern is generated automatically from cas.m4.
8553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8554 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8555 predicate(needs_acquiring_load_exclusive(n));
8556 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8557 ins_cost(VOLATILE_REF_COST);
8558 effect(TEMP_DEF res, KILL cr);
8559 format %{
8560 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8561 %}
8562 ins_encode %{
8563 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8564 Assembler::byte, /*acquire*/ true, /*release*/ true,
8565 /*weak*/ false, $res$$Register);
8566 __ sxtbw($res$$Register, $res$$Register);
8567 %}
8568 ins_pipe(pipe_slow);
8569 %}
8570
8571 // This pattern is generated automatically from cas.m4.
8572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8573 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8574 predicate(needs_acquiring_load_exclusive(n));
8575 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8576 ins_cost(VOLATILE_REF_COST);
8577 effect(TEMP_DEF res, KILL cr);
8578 format %{
8579 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8580 %}
8581 ins_encode %{
8582 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8583 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8584 /*weak*/ false, $res$$Register);
8585 __ sxthw($res$$Register, $res$$Register);
8586 %}
8587 ins_pipe(pipe_slow);
8588 %}
8589
8590 // This pattern is generated automatically from cas.m4.
8591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8592 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8593 predicate(needs_acquiring_load_exclusive(n));
8594 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8595 ins_cost(VOLATILE_REF_COST);
8596 effect(TEMP_DEF res, KILL cr);
8597 format %{
8598 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8599 %}
8600 ins_encode %{
8601 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8602 Assembler::word, /*acquire*/ true, /*release*/ true,
8603 /*weak*/ false, $res$$Register);
8604 %}
8605 ins_pipe(pipe_slow);
8606 %}
8607
8608 // This pattern is generated automatically from cas.m4.
8609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8610 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8611 predicate(needs_acquiring_load_exclusive(n));
8612 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8613 ins_cost(VOLATILE_REF_COST);
8614 effect(TEMP_DEF res, KILL cr);
8615 format %{
8616 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8617 %}
8618 ins_encode %{
8619 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8620 Assembler::xword, /*acquire*/ true, /*release*/ true,
8621 /*weak*/ false, $res$$Register);
8622 %}
8623 ins_pipe(pipe_slow);
8624 %}
8625
8626 // This pattern is generated automatically from cas.m4.
8627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8628 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8629 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8630 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8631 ins_cost(VOLATILE_REF_COST);
8632 effect(TEMP_DEF res, KILL cr);
8633 format %{
8634 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8635 %}
8636 ins_encode %{
8637 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8638 Assembler::word, /*acquire*/ true, /*release*/ true,
8639 /*weak*/ false, $res$$Register);
8640 %}
8641 ins_pipe(pipe_slow);
8642 %}
8643
8644 // This pattern is generated automatically from cas.m4.
8645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8646 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8647 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8648 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8649 ins_cost(VOLATILE_REF_COST);
8650 effect(TEMP_DEF res, KILL cr);
8651 format %{
8652 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8653 %}
8654 ins_encode %{
8655 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8656 Assembler::xword, /*acquire*/ true, /*release*/ true,
8657 /*weak*/ false, $res$$Register);
8658 %}
8659 ins_pipe(pipe_slow);
8660 %}
8661
8662 // This pattern is generated automatically from cas.m4.
8663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8664 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8665 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8666 ins_cost(2 * VOLATILE_REF_COST);
8667 effect(KILL cr);
8668 format %{
8669 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8670 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8671 %}
8672 ins_encode %{
8673 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8674 Assembler::byte, /*acquire*/ false, /*release*/ true,
8675 /*weak*/ true, noreg);
8676 __ csetw($res$$Register, Assembler::EQ);
8677 %}
8678 ins_pipe(pipe_slow);
8679 %}
8680
8681 // This pattern is generated automatically from cas.m4.
8682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8683 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8684 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8685 ins_cost(2 * VOLATILE_REF_COST);
8686 effect(KILL cr);
8687 format %{
8688 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8689 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8690 %}
8691 ins_encode %{
8692 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8693 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8694 /*weak*/ true, noreg);
8695 __ csetw($res$$Register, Assembler::EQ);
8696 %}
8697 ins_pipe(pipe_slow);
8698 %}
8699
8700 // This pattern is generated automatically from cas.m4.
8701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8702 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8703 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8704 ins_cost(2 * VOLATILE_REF_COST);
8705 effect(KILL cr);
8706 format %{
8707 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8708 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8709 %}
8710 ins_encode %{
8711 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8712 Assembler::word, /*acquire*/ false, /*release*/ true,
8713 /*weak*/ true, noreg);
8714 __ csetw($res$$Register, Assembler::EQ);
8715 %}
8716 ins_pipe(pipe_slow);
8717 %}
8718
8719 // This pattern is generated automatically from cas.m4.
8720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8721 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8722 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8723 ins_cost(2 * VOLATILE_REF_COST);
8724 effect(KILL cr);
8725 format %{
8726 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8727 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8728 %}
8729 ins_encode %{
8730 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731 Assembler::xword, /*acquire*/ false, /*release*/ true,
8732 /*weak*/ true, noreg);
8733 __ csetw($res$$Register, Assembler::EQ);
8734 %}
8735 ins_pipe(pipe_slow);
8736 %}
8737
8738 // This pattern is generated automatically from cas.m4.
8739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8740 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8741 predicate(n->as_LoadStore()->barrier_data() == 0);
8742 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8743 ins_cost(2 * VOLATILE_REF_COST);
8744 effect(KILL cr);
8745 format %{
8746 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8747 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8748 %}
8749 ins_encode %{
8750 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8751 Assembler::word, /*acquire*/ false, /*release*/ true,
8752 /*weak*/ true, noreg);
8753 __ csetw($res$$Register, Assembler::EQ);
8754 %}
8755 ins_pipe(pipe_slow);
8756 %}
8757
8758 // This pattern is generated automatically from cas.m4.
8759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8760 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8761 predicate(n->as_LoadStore()->barrier_data() == 0);
8762 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8763 ins_cost(2 * VOLATILE_REF_COST);
8764 effect(KILL cr);
8765 format %{
8766 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8767 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8768 %}
8769 ins_encode %{
8770 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8771 Assembler::xword, /*acquire*/ false, /*release*/ true,
8772 /*weak*/ true, noreg);
8773 __ csetw($res$$Register, Assembler::EQ);
8774 %}
8775 ins_pipe(pipe_slow);
8776 %}
8777
8778 // This pattern is generated automatically from cas.m4.
8779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8780 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8781 predicate(needs_acquiring_load_exclusive(n));
8782 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8783 ins_cost(VOLATILE_REF_COST);
8784 effect(KILL cr);
8785 format %{
8786 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8787 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::byte, /*acquire*/ true, /*release*/ true,
8792 /*weak*/ true, noreg);
8793 __ csetw($res$$Register, Assembler::EQ);
8794 %}
8795 ins_pipe(pipe_slow);
8796 %}
8797
8798 // This pattern is generated automatically from cas.m4.
8799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8800 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8801 predicate(needs_acquiring_load_exclusive(n));
8802 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8803 ins_cost(VOLATILE_REF_COST);
8804 effect(KILL cr);
8805 format %{
8806 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8807 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8808 %}
8809 ins_encode %{
8810 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8811 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8812 /*weak*/ true, noreg);
8813 __ csetw($res$$Register, Assembler::EQ);
8814 %}
8815 ins_pipe(pipe_slow);
8816 %}
8817
8818 // This pattern is generated automatically from cas.m4.
8819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8820 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8821 predicate(needs_acquiring_load_exclusive(n));
8822 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8823 ins_cost(VOLATILE_REF_COST);
8824 effect(KILL cr);
8825 format %{
8826 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8827 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8828 %}
8829 ins_encode %{
8830 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8831 Assembler::word, /*acquire*/ true, /*release*/ true,
8832 /*weak*/ true, noreg);
8833 __ csetw($res$$Register, Assembler::EQ);
8834 %}
8835 ins_pipe(pipe_slow);
8836 %}
8837
8838 // This pattern is generated automatically from cas.m4.
8839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8840 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8841 predicate(needs_acquiring_load_exclusive(n));
8842 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8843 ins_cost(VOLATILE_REF_COST);
8844 effect(KILL cr);
8845 format %{
8846 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8847 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8848 %}
8849 ins_encode %{
8850 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8851 Assembler::xword, /*acquire*/ true, /*release*/ true,
8852 /*weak*/ true, noreg);
8853 __ csetw($res$$Register, Assembler::EQ);
8854 %}
8855 ins_pipe(pipe_slow);
8856 %}
8857
8858 // This pattern is generated automatically from cas.m4.
8859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8860 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8861 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8862 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8863 ins_cost(VOLATILE_REF_COST);
8864 effect(KILL cr);
8865 format %{
8866 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8867 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8868 %}
8869 ins_encode %{
8870 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8871 Assembler::word, /*acquire*/ true, /*release*/ true,
8872 /*weak*/ true, noreg);
8873 __ csetw($res$$Register, Assembler::EQ);
8874 %}
8875 ins_pipe(pipe_slow);
8876 %}
8877
8878 // This pattern is generated automatically from cas.m4.
8879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8880 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8881 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8882 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8883 ins_cost(VOLATILE_REF_COST);
8884 effect(KILL cr);
8885 format %{
8886 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8887 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8888 %}
8889 ins_encode %{
8890 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8891 Assembler::xword, /*acquire*/ true, /*release*/ true,
8892 /*weak*/ true, noreg);
8893 __ csetw($res$$Register, Assembler::EQ);
8894 %}
8895 ins_pipe(pipe_slow);
8896 %}
8897
8898 // END This section of the file is automatically generated. Do not edit --------------
8899 // ---------------------------------------------------------------------
8900
8901 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8902 match(Set prev (GetAndSetI mem newv));
8903 ins_cost(2 * VOLATILE_REF_COST);
8904 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8905 ins_encode %{
8906 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8907 %}
8908 ins_pipe(pipe_serial);
8909 %}
8910
8911 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8912 match(Set prev (GetAndSetL mem newv));
8913 ins_cost(2 * VOLATILE_REF_COST);
8914 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8915 ins_encode %{
8916 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8917 %}
8918 ins_pipe(pipe_serial);
8919 %}
8920
8921 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8922 predicate(n->as_LoadStore()->barrier_data() == 0);
8923 match(Set prev (GetAndSetN mem newv));
8924 ins_cost(2 * VOLATILE_REF_COST);
8925 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8926 ins_encode %{
8927 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8928 %}
8929 ins_pipe(pipe_serial);
8930 %}
8931
8932 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8933 predicate(n->as_LoadStore()->barrier_data() == 0);
8934 match(Set prev (GetAndSetP mem newv));
8935 ins_cost(2 * VOLATILE_REF_COST);
8936 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8937 ins_encode %{
8938 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8939 %}
8940 ins_pipe(pipe_serial);
8941 %}
8942
8943 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8944 predicate(needs_acquiring_load_exclusive(n));
8945 match(Set prev (GetAndSetI mem newv));
8946 ins_cost(VOLATILE_REF_COST);
8947 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8948 ins_encode %{
8949 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8950 %}
8951 ins_pipe(pipe_serial);
8952 %}
8953
8954 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8955 predicate(needs_acquiring_load_exclusive(n));
8956 match(Set prev (GetAndSetL mem newv));
8957 ins_cost(VOLATILE_REF_COST);
8958 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8959 ins_encode %{
8960 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8961 %}
8962 ins_pipe(pipe_serial);
8963 %}
8964
8965 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8966 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8967 match(Set prev (GetAndSetN mem newv));
8968 ins_cost(VOLATILE_REF_COST);
8969 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8970 ins_encode %{
8971 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8972 %}
8973 ins_pipe(pipe_serial);
8974 %}
8975
8976 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8977 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8978 match(Set prev (GetAndSetP mem newv));
8979 ins_cost(VOLATILE_REF_COST);
8980 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8981 ins_encode %{
8982 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8983 %}
8984 ins_pipe(pipe_serial);
8985 %}
8986
8987
8988 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8989 match(Set newval (GetAndAddL mem incr));
8990 ins_cost(2 * VOLATILE_REF_COST + 1);
8991 format %{ "get_and_addL $newval, [$mem], $incr" %}
8992 ins_encode %{
8993 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8994 %}
8995 ins_pipe(pipe_serial);
8996 %}
8997
8998 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8999 predicate(n->as_LoadStore()->result_not_used());
9000 match(Set dummy (GetAndAddL mem incr));
9001 ins_cost(2 * VOLATILE_REF_COST);
9002 format %{ "get_and_addL [$mem], $incr" %}
9003 ins_encode %{
9004 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9005 %}
9006 ins_pipe(pipe_serial);
9007 %}
9008
9009 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9010 match(Set newval (GetAndAddL mem incr));
9011 ins_cost(2 * VOLATILE_REF_COST + 1);
9012 format %{ "get_and_addL $newval, [$mem], $incr" %}
9013 ins_encode %{
9014 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9015 %}
9016 ins_pipe(pipe_serial);
9017 %}
9018
9019 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9020 predicate(n->as_LoadStore()->result_not_used());
9021 match(Set dummy (GetAndAddL mem incr));
9022 ins_cost(2 * VOLATILE_REF_COST);
9023 format %{ "get_and_addL [$mem], $incr" %}
9024 ins_encode %{
9025 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9026 %}
9027 ins_pipe(pipe_serial);
9028 %}
9029
9030 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9031 match(Set newval (GetAndAddI mem incr));
9032 ins_cost(2 * VOLATILE_REF_COST + 1);
9033 format %{ "get_and_addI $newval, [$mem], $incr" %}
9034 ins_encode %{
9035 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9036 %}
9037 ins_pipe(pipe_serial);
9038 %}
9039
9040 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9041 predicate(n->as_LoadStore()->result_not_used());
9042 match(Set dummy (GetAndAddI mem incr));
9043 ins_cost(2 * VOLATILE_REF_COST);
9044 format %{ "get_and_addI [$mem], $incr" %}
9045 ins_encode %{
9046 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9047 %}
9048 ins_pipe(pipe_serial);
9049 %}
9050
9051 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9052 match(Set newval (GetAndAddI mem incr));
9053 ins_cost(2 * VOLATILE_REF_COST + 1);
9054 format %{ "get_and_addI $newval, [$mem], $incr" %}
9055 ins_encode %{
9056 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9057 %}
9058 ins_pipe(pipe_serial);
9059 %}
9060
9061 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9062 predicate(n->as_LoadStore()->result_not_used());
9063 match(Set dummy (GetAndAddI mem incr));
9064 ins_cost(2 * VOLATILE_REF_COST);
9065 format %{ "get_and_addI [$mem], $incr" %}
9066 ins_encode %{
9067 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9068 %}
9069 ins_pipe(pipe_serial);
9070 %}
9071
9072 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9073 predicate(needs_acquiring_load_exclusive(n));
9074 match(Set newval (GetAndAddL mem incr));
9075 ins_cost(VOLATILE_REF_COST + 1);
9076 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9077 ins_encode %{
9078 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9079 %}
9080 ins_pipe(pipe_serial);
9081 %}
9082
9083 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9084 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9085 match(Set dummy (GetAndAddL mem incr));
9086 ins_cost(VOLATILE_REF_COST);
9087 format %{ "get_and_addL_acq [$mem], $incr" %}
9088 ins_encode %{
9089 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9090 %}
9091 ins_pipe(pipe_serial);
9092 %}
9093
9094 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9095 predicate(needs_acquiring_load_exclusive(n));
9096 match(Set newval (GetAndAddL mem incr));
9097 ins_cost(VOLATILE_REF_COST + 1);
9098 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9099 ins_encode %{
9100 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9101 %}
9102 ins_pipe(pipe_serial);
9103 %}
9104
9105 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9106 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9107 match(Set dummy (GetAndAddL mem incr));
9108 ins_cost(VOLATILE_REF_COST);
9109 format %{ "get_and_addL_acq [$mem], $incr" %}
9110 ins_encode %{
9111 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9112 %}
9113 ins_pipe(pipe_serial);
9114 %}
9115
9116 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9117 predicate(needs_acquiring_load_exclusive(n));
9118 match(Set newval (GetAndAddI mem incr));
9119 ins_cost(VOLATILE_REF_COST + 1);
9120 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9121 ins_encode %{
9122 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9123 %}
9124 ins_pipe(pipe_serial);
9125 %}
9126
9127 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9128 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9129 match(Set dummy (GetAndAddI mem incr));
9130 ins_cost(VOLATILE_REF_COST);
9131 format %{ "get_and_addI_acq [$mem], $incr" %}
9132 ins_encode %{
9133 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9134 %}
9135 ins_pipe(pipe_serial);
9136 %}
9137
9138 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9139 predicate(needs_acquiring_load_exclusive(n));
9140 match(Set newval (GetAndAddI mem incr));
9141 ins_cost(VOLATILE_REF_COST + 1);
9142 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9143 ins_encode %{
9144 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9145 %}
9146 ins_pipe(pipe_serial);
9147 %}
9148
9149 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9150 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9151 match(Set dummy (GetAndAddI mem incr));
9152 ins_cost(VOLATILE_REF_COST);
9153 format %{ "get_and_addI_acq [$mem], $incr" %}
9154 ins_encode %{
9155 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9156 %}
9157 ins_pipe(pipe_serial);
9158 %}
9159
9160 // Manifest a CmpU result in an integer register.
9161 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9162 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9163 %{
9164 match(Set dst (CmpU3 src1 src2));
9165 effect(KILL flags);
9166
9167 ins_cost(INSN_COST * 3);
9168 format %{
9169 "cmpw $src1, $src2\n\t"
9170 "csetw $dst, ne\n\t"
9171 "cnegw $dst, lo\t# CmpU3(reg)"
9172 %}
9173 ins_encode %{
9174 __ cmpw($src1$$Register, $src2$$Register);
9175 __ csetw($dst$$Register, Assembler::NE);
9176 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9177 %}
9178
9179 ins_pipe(pipe_class_default);
9180 %}
9181
9182 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9183 %{
9184 match(Set dst (CmpU3 src1 src2));
9185 effect(KILL flags);
9186
9187 ins_cost(INSN_COST * 3);
9188 format %{
9189 "subsw zr, $src1, $src2\n\t"
9190 "csetw $dst, ne\n\t"
9191 "cnegw $dst, lo\t# CmpU3(imm)"
9192 %}
9193 ins_encode %{
9194 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9195 __ csetw($dst$$Register, Assembler::NE);
9196 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9197 %}
9198
9199 ins_pipe(pipe_class_default);
9200 %}
9201
9202 // Manifest a CmpUL result in an integer register.
9203 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9204 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9205 %{
9206 match(Set dst (CmpUL3 src1 src2));
9207 effect(KILL flags);
9208
9209 ins_cost(INSN_COST * 3);
9210 format %{
9211 "cmp $src1, $src2\n\t"
9212 "csetw $dst, ne\n\t"
9213 "cnegw $dst, lo\t# CmpUL3(reg)"
9214 %}
9215 ins_encode %{
9216 __ cmp($src1$$Register, $src2$$Register);
9217 __ csetw($dst$$Register, Assembler::NE);
9218 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9219 %}
9220
9221 ins_pipe(pipe_class_default);
9222 %}
9223
9224 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9225 %{
9226 match(Set dst (CmpUL3 src1 src2));
9227 effect(KILL flags);
9228
9229 ins_cost(INSN_COST * 3);
9230 format %{
9231 "subs zr, $src1, $src2\n\t"
9232 "csetw $dst, ne\n\t"
9233 "cnegw $dst, lo\t# CmpUL3(imm)"
9234 %}
9235 ins_encode %{
9236 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9237 __ csetw($dst$$Register, Assembler::NE);
9238 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9239 %}
9240
9241 ins_pipe(pipe_class_default);
9242 %}
9243
9244 // Manifest a CmpL result in an integer register.
9245 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9246 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9247 %{
9248 match(Set dst (CmpL3 src1 src2));
9249 effect(KILL flags);
9250
9251 ins_cost(INSN_COST * 3);
9252 format %{
9253 "cmp $src1, $src2\n\t"
9254 "csetw $dst, ne\n\t"
9255 "cnegw $dst, lt\t# CmpL3(reg)"
9256 %}
9257 ins_encode %{
9258 __ cmp($src1$$Register, $src2$$Register);
9259 __ csetw($dst$$Register, Assembler::NE);
9260 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9261 %}
9262
9263 ins_pipe(pipe_class_default);
9264 %}
9265
9266 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9267 %{
9268 match(Set dst (CmpL3 src1 src2));
9269 effect(KILL flags);
9270
9271 ins_cost(INSN_COST * 3);
9272 format %{
9273 "subs zr, $src1, $src2\n\t"
9274 "csetw $dst, ne\n\t"
9275 "cnegw $dst, lt\t# CmpL3(imm)"
9276 %}
9277 ins_encode %{
9278 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9279 __ csetw($dst$$Register, Assembler::NE);
9280 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9281 %}
9282
9283 ins_pipe(pipe_class_default);
9284 %}
9285
9286 // ============================================================================
9287 // Conditional Move Instructions
9288
9289 // n.b. we have identical rules for both a signed compare op (cmpOp)
9290 // and an unsigned compare op (cmpOpU). it would be nice if we could
9291 // define an op class which merged both inputs and use it to type the
9292 // argument to a single rule. unfortunatelyt his fails because the
9293 // opclass does not live up to the COND_INTER interface of its
9294 // component operands. When the generic code tries to negate the
9295 // operand it ends up running the generci Machoper::negate method
9296 // which throws a ShouldNotHappen. So, we have to provide two flavours
9297 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9298
9299 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9300 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9301
9302 ins_cost(INSN_COST * 2);
9303 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9304
9305 ins_encode %{
9306 __ cselw(as_Register($dst$$reg),
9307 as_Register($src2$$reg),
9308 as_Register($src1$$reg),
9309 (Assembler::Condition)$cmp$$cmpcode);
9310 %}
9311
9312 ins_pipe(icond_reg_reg);
9313 %}
9314
9315 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9316 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9317
9318 ins_cost(INSN_COST * 2);
9319 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9320
9321 ins_encode %{
9322 __ cselw(as_Register($dst$$reg),
9323 as_Register($src2$$reg),
9324 as_Register($src1$$reg),
9325 (Assembler::Condition)$cmp$$cmpcode);
9326 %}
9327
9328 ins_pipe(icond_reg_reg);
9329 %}
9330
9331 // special cases where one arg is zero
9332
9333 // n.b. this is selected in preference to the rule above because it
9334 // avoids loading constant 0 into a source register
9335
9336 // TODO
9337 // we ought only to be able to cull one of these variants as the ideal
9338 // transforms ought always to order the zero consistently (to left/right?)
9339
9340 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9341 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9342
9343 ins_cost(INSN_COST * 2);
9344 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9345
9346 ins_encode %{
9347 __ cselw(as_Register($dst$$reg),
9348 as_Register($src$$reg),
9349 zr,
9350 (Assembler::Condition)$cmp$$cmpcode);
9351 %}
9352
9353 ins_pipe(icond_reg);
9354 %}
9355
9356 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9357 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9358
9359 ins_cost(INSN_COST * 2);
9360 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9361
9362 ins_encode %{
9363 __ cselw(as_Register($dst$$reg),
9364 as_Register($src$$reg),
9365 zr,
9366 (Assembler::Condition)$cmp$$cmpcode);
9367 %}
9368
9369 ins_pipe(icond_reg);
9370 %}
9371
9372 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9373 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9374
9375 ins_cost(INSN_COST * 2);
9376 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9377
9378 ins_encode %{
9379 __ cselw(as_Register($dst$$reg),
9380 zr,
9381 as_Register($src$$reg),
9382 (Assembler::Condition)$cmp$$cmpcode);
9383 %}
9384
9385 ins_pipe(icond_reg);
9386 %}
9387
9388 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9389 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9390
9391 ins_cost(INSN_COST * 2);
9392 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9393
9394 ins_encode %{
9395 __ cselw(as_Register($dst$$reg),
9396 zr,
9397 as_Register($src$$reg),
9398 (Assembler::Condition)$cmp$$cmpcode);
9399 %}
9400
9401 ins_pipe(icond_reg);
9402 %}
9403
9404 // special case for creating a boolean 0 or 1
9405
9406 // n.b. this is selected in preference to the rule above because it
9407 // avoids loading constants 0 and 1 into a source register
9408
9409 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9410 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9411
9412 ins_cost(INSN_COST * 2);
9413 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9414
9415 ins_encode %{
9416 // equivalently
9417 // cset(as_Register($dst$$reg),
9418 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9419 __ csincw(as_Register($dst$$reg),
9420 zr,
9421 zr,
9422 (Assembler::Condition)$cmp$$cmpcode);
9423 %}
9424
9425 ins_pipe(icond_none);
9426 %}
9427
9428 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9429 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9430
9431 ins_cost(INSN_COST * 2);
9432 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9433
9434 ins_encode %{
9435 // equivalently
9436 // cset(as_Register($dst$$reg),
9437 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9438 __ csincw(as_Register($dst$$reg),
9439 zr,
9440 zr,
9441 (Assembler::Condition)$cmp$$cmpcode);
9442 %}
9443
9444 ins_pipe(icond_none);
9445 %}
9446
9447 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9448 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9449
9450 ins_cost(INSN_COST * 2);
9451 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9452
9453 ins_encode %{
9454 __ csel(as_Register($dst$$reg),
9455 as_Register($src2$$reg),
9456 as_Register($src1$$reg),
9457 (Assembler::Condition)$cmp$$cmpcode);
9458 %}
9459
9460 ins_pipe(icond_reg_reg);
9461 %}
9462
9463 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9464 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9465
9466 ins_cost(INSN_COST * 2);
9467 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9468
9469 ins_encode %{
9470 __ csel(as_Register($dst$$reg),
9471 as_Register($src2$$reg),
9472 as_Register($src1$$reg),
9473 (Assembler::Condition)$cmp$$cmpcode);
9474 %}
9475
9476 ins_pipe(icond_reg_reg);
9477 %}
9478
9479 // special cases where one arg is zero
9480
9481 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9482 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9483
9484 ins_cost(INSN_COST * 2);
9485 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9486
9487 ins_encode %{
9488 __ csel(as_Register($dst$$reg),
9489 zr,
9490 as_Register($src$$reg),
9491 (Assembler::Condition)$cmp$$cmpcode);
9492 %}
9493
9494 ins_pipe(icond_reg);
9495 %}
9496
9497 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9498 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9499
9500 ins_cost(INSN_COST * 2);
9501 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9502
9503 ins_encode %{
9504 __ csel(as_Register($dst$$reg),
9505 zr,
9506 as_Register($src$$reg),
9507 (Assembler::Condition)$cmp$$cmpcode);
9508 %}
9509
9510 ins_pipe(icond_reg);
9511 %}
9512
9513 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9514 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9515
9516 ins_cost(INSN_COST * 2);
9517 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9518
9519 ins_encode %{
9520 __ csel(as_Register($dst$$reg),
9521 as_Register($src$$reg),
9522 zr,
9523 (Assembler::Condition)$cmp$$cmpcode);
9524 %}
9525
9526 ins_pipe(icond_reg);
9527 %}
9528
9529 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9530 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9531
9532 ins_cost(INSN_COST * 2);
9533 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9534
9535 ins_encode %{
9536 __ csel(as_Register($dst$$reg),
9537 as_Register($src$$reg),
9538 zr,
9539 (Assembler::Condition)$cmp$$cmpcode);
9540 %}
9541
9542 ins_pipe(icond_reg);
9543 %}
9544
9545 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9546 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9547
9548 ins_cost(INSN_COST * 2);
9549 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9550
9551 ins_encode %{
9552 __ csel(as_Register($dst$$reg),
9553 as_Register($src2$$reg),
9554 as_Register($src1$$reg),
9555 (Assembler::Condition)$cmp$$cmpcode);
9556 %}
9557
9558 ins_pipe(icond_reg_reg);
9559 %}
9560
9561 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9562 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9563
9564 ins_cost(INSN_COST * 2);
9565 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9566
9567 ins_encode %{
9568 __ csel(as_Register($dst$$reg),
9569 as_Register($src2$$reg),
9570 as_Register($src1$$reg),
9571 (Assembler::Condition)$cmp$$cmpcode);
9572 %}
9573
9574 ins_pipe(icond_reg_reg);
9575 %}
9576
9577 // special cases where one arg is zero
9578
9579 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9580 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9581
9582 ins_cost(INSN_COST * 2);
9583 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9584
9585 ins_encode %{
9586 __ csel(as_Register($dst$$reg),
9587 zr,
9588 as_Register($src$$reg),
9589 (Assembler::Condition)$cmp$$cmpcode);
9590 %}
9591
9592 ins_pipe(icond_reg);
9593 %}
9594
9595 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9596 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9597
9598 ins_cost(INSN_COST * 2);
9599 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9600
9601 ins_encode %{
9602 __ csel(as_Register($dst$$reg),
9603 zr,
9604 as_Register($src$$reg),
9605 (Assembler::Condition)$cmp$$cmpcode);
9606 %}
9607
9608 ins_pipe(icond_reg);
9609 %}
9610
9611 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9612 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9613
9614 ins_cost(INSN_COST * 2);
9615 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9616
9617 ins_encode %{
9618 __ csel(as_Register($dst$$reg),
9619 as_Register($src$$reg),
9620 zr,
9621 (Assembler::Condition)$cmp$$cmpcode);
9622 %}
9623
9624 ins_pipe(icond_reg);
9625 %}
9626
9627 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9628 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9629
9630 ins_cost(INSN_COST * 2);
9631 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9632
9633 ins_encode %{
9634 __ csel(as_Register($dst$$reg),
9635 as_Register($src$$reg),
9636 zr,
9637 (Assembler::Condition)$cmp$$cmpcode);
9638 %}
9639
9640 ins_pipe(icond_reg);
9641 %}
9642
9643 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9644 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9645
9646 ins_cost(INSN_COST * 2);
9647 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9648
9649 ins_encode %{
9650 __ cselw(as_Register($dst$$reg),
9651 as_Register($src2$$reg),
9652 as_Register($src1$$reg),
9653 (Assembler::Condition)$cmp$$cmpcode);
9654 %}
9655
9656 ins_pipe(icond_reg_reg);
9657 %}
9658
9659 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9660 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9661
9662 ins_cost(INSN_COST * 2);
9663 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9664
9665 ins_encode %{
9666 __ cselw(as_Register($dst$$reg),
9667 as_Register($src2$$reg),
9668 as_Register($src1$$reg),
9669 (Assembler::Condition)$cmp$$cmpcode);
9670 %}
9671
9672 ins_pipe(icond_reg_reg);
9673 %}
9674
9675 // special cases where one arg is zero
9676
9677 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9678 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9679
9680 ins_cost(INSN_COST * 2);
9681 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9682
9683 ins_encode %{
9684 __ cselw(as_Register($dst$$reg),
9685 zr,
9686 as_Register($src$$reg),
9687 (Assembler::Condition)$cmp$$cmpcode);
9688 %}
9689
9690 ins_pipe(icond_reg);
9691 %}
9692
9693 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9694 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9695
9696 ins_cost(INSN_COST * 2);
9697 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9698
9699 ins_encode %{
9700 __ cselw(as_Register($dst$$reg),
9701 zr,
9702 as_Register($src$$reg),
9703 (Assembler::Condition)$cmp$$cmpcode);
9704 %}
9705
9706 ins_pipe(icond_reg);
9707 %}
9708
9709 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9710 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9711
9712 ins_cost(INSN_COST * 2);
9713 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9714
9715 ins_encode %{
9716 __ cselw(as_Register($dst$$reg),
9717 as_Register($src$$reg),
9718 zr,
9719 (Assembler::Condition)$cmp$$cmpcode);
9720 %}
9721
9722 ins_pipe(icond_reg);
9723 %}
9724
9725 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9726 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9727
9728 ins_cost(INSN_COST * 2);
9729 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9730
9731 ins_encode %{
9732 __ cselw(as_Register($dst$$reg),
9733 as_Register($src$$reg),
9734 zr,
9735 (Assembler::Condition)$cmp$$cmpcode);
9736 %}
9737
9738 ins_pipe(icond_reg);
9739 %}
9740
9741 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9742 %{
9743 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9744
9745 ins_cost(INSN_COST * 3);
9746
9747 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9748 ins_encode %{
9749 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9750 __ fcsels(as_FloatRegister($dst$$reg),
9751 as_FloatRegister($src2$$reg),
9752 as_FloatRegister($src1$$reg),
9753 cond);
9754 %}
9755
9756 ins_pipe(fp_cond_reg_reg_s);
9757 %}
9758
9759 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9760 %{
9761 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9762
9763 ins_cost(INSN_COST * 3);
9764
9765 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9766 ins_encode %{
9767 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9768 __ fcsels(as_FloatRegister($dst$$reg),
9769 as_FloatRegister($src2$$reg),
9770 as_FloatRegister($src1$$reg),
9771 cond);
9772 %}
9773
9774 ins_pipe(fp_cond_reg_reg_s);
9775 %}
9776
9777 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9778 %{
9779 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9780
9781 ins_cost(INSN_COST * 3);
9782
9783 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9784 ins_encode %{
9785 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9786 __ fcseld(as_FloatRegister($dst$$reg),
9787 as_FloatRegister($src2$$reg),
9788 as_FloatRegister($src1$$reg),
9789 cond);
9790 %}
9791
9792 ins_pipe(fp_cond_reg_reg_d);
9793 %}
9794
9795 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9796 %{
9797 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9798
9799 ins_cost(INSN_COST * 3);
9800
9801 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9802 ins_encode %{
9803 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9804 __ fcseld(as_FloatRegister($dst$$reg),
9805 as_FloatRegister($src2$$reg),
9806 as_FloatRegister($src1$$reg),
9807 cond);
9808 %}
9809
9810 ins_pipe(fp_cond_reg_reg_d);
9811 %}
9812
9813 // ============================================================================
9814 // Arithmetic Instructions
9815 //
9816
9817 // Integer Addition
9818
9819 // TODO
9820 // these currently employ operations which do not set CR and hence are
9821 // not flagged as killing CR but we would like to isolate the cases
9822 // where we want to set flags from those where we don't. need to work
9823 // out how to do that.
9824
9825 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9826 match(Set dst (AddI src1 src2));
9827
9828 ins_cost(INSN_COST);
9829 format %{ "addw $dst, $src1, $src2" %}
9830
9831 ins_encode %{
9832 __ addw(as_Register($dst$$reg),
9833 as_Register($src1$$reg),
9834 as_Register($src2$$reg));
9835 %}
9836
9837 ins_pipe(ialu_reg_reg);
9838 %}
9839
9840 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9841 match(Set dst (AddI src1 src2));
9842
9843 ins_cost(INSN_COST);
9844 format %{ "addw $dst, $src1, $src2" %}
9845
9846 // use opcode to indicate that this is an add not a sub
9847 opcode(0x0);
9848
9849 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9850
9851 ins_pipe(ialu_reg_imm);
9852 %}
9853
9854 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9855 match(Set dst (AddI (ConvL2I src1) src2));
9856
9857 ins_cost(INSN_COST);
9858 format %{ "addw $dst, $src1, $src2" %}
9859
9860 // use opcode to indicate that this is an add not a sub
9861 opcode(0x0);
9862
9863 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9864
9865 ins_pipe(ialu_reg_imm);
9866 %}
9867
9868 // Pointer Addition
9869 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9870 match(Set dst (AddP src1 src2));
9871
9872 ins_cost(INSN_COST);
9873 format %{ "add $dst, $src1, $src2\t# ptr" %}
9874
9875 ins_encode %{
9876 __ add(as_Register($dst$$reg),
9877 as_Register($src1$$reg),
9878 as_Register($src2$$reg));
9879 %}
9880
9881 ins_pipe(ialu_reg_reg);
9882 %}
9883
9884 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9885 match(Set dst (AddP src1 (ConvI2L src2)));
9886
9887 ins_cost(1.9 * INSN_COST);
9888 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9889
9890 ins_encode %{
9891 __ add(as_Register($dst$$reg),
9892 as_Register($src1$$reg),
9893 as_Register($src2$$reg), ext::sxtw);
9894 %}
9895
9896 ins_pipe(ialu_reg_reg);
9897 %}
9898
9899 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9900 match(Set dst (AddP src1 (LShiftL src2 scale)));
9901
9902 ins_cost(1.9 * INSN_COST);
9903 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9904
9905 ins_encode %{
9906 __ lea(as_Register($dst$$reg),
9907 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9908 Address::lsl($scale$$constant)));
9909 %}
9910
9911 ins_pipe(ialu_reg_reg_shift);
9912 %}
9913
9914 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9915 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9916
9917 ins_cost(1.9 * INSN_COST);
9918 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9919
9920 ins_encode %{
9921 __ lea(as_Register($dst$$reg),
9922 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9923 Address::sxtw($scale$$constant)));
9924 %}
9925
9926 ins_pipe(ialu_reg_reg_shift);
9927 %}
9928
9929 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9930 match(Set dst (LShiftL (ConvI2L src) scale));
9931
9932 ins_cost(INSN_COST);
9933 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9934
9935 ins_encode %{
9936 __ sbfiz(as_Register($dst$$reg),
9937 as_Register($src$$reg),
9938 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9939 %}
9940
9941 ins_pipe(ialu_reg_shift);
9942 %}
9943
9944 // Pointer Immediate Addition
9945 // n.b. this needs to be more expensive than using an indirect memory
9946 // operand
9947 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9948 match(Set dst (AddP src1 src2));
9949
9950 ins_cost(INSN_COST);
9951 format %{ "add $dst, $src1, $src2\t# ptr" %}
9952
9953 // use opcode to indicate that this is an add not a sub
9954 opcode(0x0);
9955
9956 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9957
9958 ins_pipe(ialu_reg_imm);
9959 %}
9960
9961 // Long Addition
9962 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9963
9964 match(Set dst (AddL src1 src2));
9965
9966 ins_cost(INSN_COST);
9967 format %{ "add $dst, $src1, $src2" %}
9968
9969 ins_encode %{
9970 __ add(as_Register($dst$$reg),
9971 as_Register($src1$$reg),
9972 as_Register($src2$$reg));
9973 %}
9974
9975 ins_pipe(ialu_reg_reg);
9976 %}
9977
9978 // No constant pool entries requiredLong Immediate Addition.
9979 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9980 match(Set dst (AddL src1 src2));
9981
9982 ins_cost(INSN_COST);
9983 format %{ "add $dst, $src1, $src2" %}
9984
9985 // use opcode to indicate that this is an add not a sub
9986 opcode(0x0);
9987
9988 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9989
9990 ins_pipe(ialu_reg_imm);
9991 %}
9992
9993 // Integer Subtraction
9994 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9995 match(Set dst (SubI src1 src2));
9996
9997 ins_cost(INSN_COST);
9998 format %{ "subw $dst, $src1, $src2" %}
9999
10000 ins_encode %{
10001 __ subw(as_Register($dst$$reg),
10002 as_Register($src1$$reg),
10003 as_Register($src2$$reg));
10004 %}
10005
10006 ins_pipe(ialu_reg_reg);
10007 %}
10008
10009 // Immediate Subtraction
10010 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10011 match(Set dst (SubI src1 src2));
10012
10013 ins_cost(INSN_COST);
10014 format %{ "subw $dst, $src1, $src2" %}
10015
10016 // use opcode to indicate that this is a sub not an add
10017 opcode(0x1);
10018
10019 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10020
10021 ins_pipe(ialu_reg_imm);
10022 %}
10023
10024 // Long Subtraction
10025 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10026
10027 match(Set dst (SubL src1 src2));
10028
10029 ins_cost(INSN_COST);
10030 format %{ "sub $dst, $src1, $src2" %}
10031
10032 ins_encode %{
10033 __ sub(as_Register($dst$$reg),
10034 as_Register($src1$$reg),
10035 as_Register($src2$$reg));
10036 %}
10037
10038 ins_pipe(ialu_reg_reg);
10039 %}
10040
10041 // No constant pool entries requiredLong Immediate Subtraction.
10042 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10043 match(Set dst (SubL src1 src2));
10044
10045 ins_cost(INSN_COST);
10046 format %{ "sub$dst, $src1, $src2" %}
10047
10048 // use opcode to indicate that this is a sub not an add
10049 opcode(0x1);
10050
10051 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10052
10053 ins_pipe(ialu_reg_imm);
10054 %}
10055
10056 // Integer Negation (special case for sub)
10057
10058 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10059 match(Set dst (SubI zero src));
10060
10061 ins_cost(INSN_COST);
10062 format %{ "negw $dst, $src\t# int" %}
10063
10064 ins_encode %{
10065 __ negw(as_Register($dst$$reg),
10066 as_Register($src$$reg));
10067 %}
10068
10069 ins_pipe(ialu_reg);
10070 %}
10071
10072 // Long Negation
10073
10074 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10075 match(Set dst (SubL zero src));
10076
10077 ins_cost(INSN_COST);
10078 format %{ "neg $dst, $src\t# long" %}
10079
10080 ins_encode %{
10081 __ neg(as_Register($dst$$reg),
10082 as_Register($src$$reg));
10083 %}
10084
10085 ins_pipe(ialu_reg);
10086 %}
10087
10088 // Integer Multiply
10089
10090 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10091 match(Set dst (MulI src1 src2));
10092
10093 ins_cost(INSN_COST * 3);
10094 format %{ "mulw $dst, $src1, $src2" %}
10095
10096 ins_encode %{
10097 __ mulw(as_Register($dst$$reg),
10098 as_Register($src1$$reg),
10099 as_Register($src2$$reg));
10100 %}
10101
10102 ins_pipe(imul_reg_reg);
10103 %}
10104
10105 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10106 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10107
10108 ins_cost(INSN_COST * 3);
10109 format %{ "smull $dst, $src1, $src2" %}
10110
10111 ins_encode %{
10112 __ smull(as_Register($dst$$reg),
10113 as_Register($src1$$reg),
10114 as_Register($src2$$reg));
10115 %}
10116
10117 ins_pipe(imul_reg_reg);
10118 %}
10119
10120 // Long Multiply
10121
10122 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10123 match(Set dst (MulL src1 src2));
10124
10125 ins_cost(INSN_COST * 5);
10126 format %{ "mul $dst, $src1, $src2" %}
10127
10128 ins_encode %{
10129 __ mul(as_Register($dst$$reg),
10130 as_Register($src1$$reg),
10131 as_Register($src2$$reg));
10132 %}
10133
10134 ins_pipe(lmul_reg_reg);
10135 %}
10136
10137 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10138 %{
10139 match(Set dst (MulHiL src1 src2));
10140
10141 ins_cost(INSN_COST * 7);
10142 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10143
10144 ins_encode %{
10145 __ smulh(as_Register($dst$$reg),
10146 as_Register($src1$$reg),
10147 as_Register($src2$$reg));
10148 %}
10149
10150 ins_pipe(lmul_reg_reg);
10151 %}
10152
10153 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10154 %{
10155 match(Set dst (UMulHiL src1 src2));
10156
10157 ins_cost(INSN_COST * 7);
10158 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10159
10160 ins_encode %{
10161 __ umulh(as_Register($dst$$reg),
10162 as_Register($src1$$reg),
10163 as_Register($src2$$reg));
10164 %}
10165
10166 ins_pipe(lmul_reg_reg);
10167 %}
10168
10169 // Combined Integer Multiply & Add/Sub
10170
10171 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10172 match(Set dst (AddI src3 (MulI src1 src2)));
10173
10174 ins_cost(INSN_COST * 3);
10175 format %{ "madd $dst, $src1, $src2, $src3" %}
10176
10177 ins_encode %{
10178 __ maddw(as_Register($dst$$reg),
10179 as_Register($src1$$reg),
10180 as_Register($src2$$reg),
10181 as_Register($src3$$reg));
10182 %}
10183
10184 ins_pipe(imac_reg_reg);
10185 %}
10186
10187 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10188 match(Set dst (SubI src3 (MulI src1 src2)));
10189
10190 ins_cost(INSN_COST * 3);
10191 format %{ "msub $dst, $src1, $src2, $src3" %}
10192
10193 ins_encode %{
10194 __ msubw(as_Register($dst$$reg),
10195 as_Register($src1$$reg),
10196 as_Register($src2$$reg),
10197 as_Register($src3$$reg));
10198 %}
10199
10200 ins_pipe(imac_reg_reg);
10201 %}
10202
10203 // Combined Integer Multiply & Neg
10204
10205 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10206 match(Set dst (MulI (SubI zero src1) src2));
10207
10208 ins_cost(INSN_COST * 3);
10209 format %{ "mneg $dst, $src1, $src2" %}
10210
10211 ins_encode %{
10212 __ mnegw(as_Register($dst$$reg),
10213 as_Register($src1$$reg),
10214 as_Register($src2$$reg));
10215 %}
10216
10217 ins_pipe(imac_reg_reg);
10218 %}
10219
10220 // Combined Long Multiply & Add/Sub
10221
10222 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10223 match(Set dst (AddL src3 (MulL src1 src2)));
10224
10225 ins_cost(INSN_COST * 5);
10226 format %{ "madd $dst, $src1, $src2, $src3" %}
10227
10228 ins_encode %{
10229 __ madd(as_Register($dst$$reg),
10230 as_Register($src1$$reg),
10231 as_Register($src2$$reg),
10232 as_Register($src3$$reg));
10233 %}
10234
10235 ins_pipe(lmac_reg_reg);
10236 %}
10237
10238 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10239 match(Set dst (SubL src3 (MulL src1 src2)));
10240
10241 ins_cost(INSN_COST * 5);
10242 format %{ "msub $dst, $src1, $src2, $src3" %}
10243
10244 ins_encode %{
10245 __ msub(as_Register($dst$$reg),
10246 as_Register($src1$$reg),
10247 as_Register($src2$$reg),
10248 as_Register($src3$$reg));
10249 %}
10250
10251 ins_pipe(lmac_reg_reg);
10252 %}
10253
10254 // Combined Long Multiply & Neg
10255
10256 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10257 match(Set dst (MulL (SubL zero src1) src2));
10258
10259 ins_cost(INSN_COST * 5);
10260 format %{ "mneg $dst, $src1, $src2" %}
10261
10262 ins_encode %{
10263 __ mneg(as_Register($dst$$reg),
10264 as_Register($src1$$reg),
10265 as_Register($src2$$reg));
10266 %}
10267
10268 ins_pipe(lmac_reg_reg);
10269 %}
10270
10271 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10272
10273 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10274 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10275
10276 ins_cost(INSN_COST * 3);
10277 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10278
10279 ins_encode %{
10280 __ smaddl(as_Register($dst$$reg),
10281 as_Register($src1$$reg),
10282 as_Register($src2$$reg),
10283 as_Register($src3$$reg));
10284 %}
10285
10286 ins_pipe(imac_reg_reg);
10287 %}
10288
10289 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10290 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10291
10292 ins_cost(INSN_COST * 3);
10293 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10294
10295 ins_encode %{
10296 __ smsubl(as_Register($dst$$reg),
10297 as_Register($src1$$reg),
10298 as_Register($src2$$reg),
10299 as_Register($src3$$reg));
10300 %}
10301
10302 ins_pipe(imac_reg_reg);
10303 %}
10304
10305 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10306 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10307
10308 ins_cost(INSN_COST * 3);
10309 format %{ "smnegl $dst, $src1, $src2" %}
10310
10311 ins_encode %{
10312 __ smnegl(as_Register($dst$$reg),
10313 as_Register($src1$$reg),
10314 as_Register($src2$$reg));
10315 %}
10316
10317 ins_pipe(imac_reg_reg);
10318 %}
10319
10320 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10321
10322 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10323 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10324
10325 ins_cost(INSN_COST * 5);
10326 format %{ "mulw rscratch1, $src1, $src2\n\t"
10327 "maddw $dst, $src3, $src4, rscratch1" %}
10328
10329 ins_encode %{
10330 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10331 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10332
10333 ins_pipe(imac_reg_reg);
10334 %}
10335
10336 // Integer Divide
10337
10338 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10339 match(Set dst (DivI src1 src2));
10340
10341 ins_cost(INSN_COST * 19);
10342 format %{ "sdivw $dst, $src1, $src2" %}
10343
10344 ins_encode(aarch64_enc_divw(dst, src1, src2));
10345 ins_pipe(idiv_reg_reg);
10346 %}
10347
10348 // Long Divide
10349
10350 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10351 match(Set dst (DivL src1 src2));
10352
10353 ins_cost(INSN_COST * 35);
10354 format %{ "sdiv $dst, $src1, $src2" %}
10355
10356 ins_encode(aarch64_enc_div(dst, src1, src2));
10357 ins_pipe(ldiv_reg_reg);
10358 %}
10359
10360 // Integer Remainder
10361
10362 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10363 match(Set dst (ModI src1 src2));
10364
10365 ins_cost(INSN_COST * 22);
10366 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10367 "msubw $dst, rscratch1, $src2, $src1" %}
10368
10369 ins_encode(aarch64_enc_modw(dst, src1, src2));
10370 ins_pipe(idiv_reg_reg);
10371 %}
10372
10373 // Long Remainder
10374
10375 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10376 match(Set dst (ModL src1 src2));
10377
10378 ins_cost(INSN_COST * 38);
10379 format %{ "sdiv rscratch1, $src1, $src2\n"
10380 "msub $dst, rscratch1, $src2, $src1" %}
10381
10382 ins_encode(aarch64_enc_mod(dst, src1, src2));
10383 ins_pipe(ldiv_reg_reg);
10384 %}
10385
10386 // Unsigned Integer Divide
10387
10388 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10389 match(Set dst (UDivI src1 src2));
10390
10391 ins_cost(INSN_COST * 19);
10392 format %{ "udivw $dst, $src1, $src2" %}
10393
10394 ins_encode %{
10395 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10396 %}
10397
10398 ins_pipe(idiv_reg_reg);
10399 %}
10400
10401 // Unsigned Long Divide
10402
10403 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10404 match(Set dst (UDivL src1 src2));
10405
10406 ins_cost(INSN_COST * 35);
10407 format %{ "udiv $dst, $src1, $src2" %}
10408
10409 ins_encode %{
10410 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10411 %}
10412
10413 ins_pipe(ldiv_reg_reg);
10414 %}
10415
10416 // Unsigned Integer Remainder
10417
10418 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10419 match(Set dst (UModI src1 src2));
10420
10421 ins_cost(INSN_COST * 22);
10422 format %{ "udivw rscratch1, $src1, $src2\n\t"
10423 "msubw $dst, rscratch1, $src2, $src1" %}
10424
10425 ins_encode %{
10426 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10427 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10428 %}
10429
10430 ins_pipe(idiv_reg_reg);
10431 %}
10432
10433 // Unsigned Long Remainder
10434
10435 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10436 match(Set dst (UModL src1 src2));
10437
10438 ins_cost(INSN_COST * 38);
10439 format %{ "udiv rscratch1, $src1, $src2\n"
10440 "msub $dst, rscratch1, $src2, $src1" %}
10441
10442 ins_encode %{
10443 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10444 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10445 %}
10446
10447 ins_pipe(ldiv_reg_reg);
10448 %}
10449
10450 // Integer Shifts
10451
10452 // Shift Left Register
10453 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10454 match(Set dst (LShiftI src1 src2));
10455
10456 ins_cost(INSN_COST * 2);
10457 format %{ "lslvw $dst, $src1, $src2" %}
10458
10459 ins_encode %{
10460 __ lslvw(as_Register($dst$$reg),
10461 as_Register($src1$$reg),
10462 as_Register($src2$$reg));
10463 %}
10464
10465 ins_pipe(ialu_reg_reg_vshift);
10466 %}
10467
10468 // Shift Left Immediate
10469 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10470 match(Set dst (LShiftI src1 src2));
10471
10472 ins_cost(INSN_COST);
10473 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10474
10475 ins_encode %{
10476 __ lslw(as_Register($dst$$reg),
10477 as_Register($src1$$reg),
10478 $src2$$constant & 0x1f);
10479 %}
10480
10481 ins_pipe(ialu_reg_shift);
10482 %}
10483
10484 // Shift Right Logical Register
10485 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10486 match(Set dst (URShiftI src1 src2));
10487
10488 ins_cost(INSN_COST * 2);
10489 format %{ "lsrvw $dst, $src1, $src2" %}
10490
10491 ins_encode %{
10492 __ lsrvw(as_Register($dst$$reg),
10493 as_Register($src1$$reg),
10494 as_Register($src2$$reg));
10495 %}
10496
10497 ins_pipe(ialu_reg_reg_vshift);
10498 %}
10499
10500 // Shift Right Logical Immediate
10501 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10502 match(Set dst (URShiftI src1 src2));
10503
10504 ins_cost(INSN_COST);
10505 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10506
10507 ins_encode %{
10508 __ lsrw(as_Register($dst$$reg),
10509 as_Register($src1$$reg),
10510 $src2$$constant & 0x1f);
10511 %}
10512
10513 ins_pipe(ialu_reg_shift);
10514 %}
10515
10516 // Shift Right Arithmetic Register
10517 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10518 match(Set dst (RShiftI src1 src2));
10519
10520 ins_cost(INSN_COST * 2);
10521 format %{ "asrvw $dst, $src1, $src2" %}
10522
10523 ins_encode %{
10524 __ asrvw(as_Register($dst$$reg),
10525 as_Register($src1$$reg),
10526 as_Register($src2$$reg));
10527 %}
10528
10529 ins_pipe(ialu_reg_reg_vshift);
10530 %}
10531
10532 // Shift Right Arithmetic Immediate
10533 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10534 match(Set dst (RShiftI src1 src2));
10535
10536 ins_cost(INSN_COST);
10537 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10538
10539 ins_encode %{
10540 __ asrw(as_Register($dst$$reg),
10541 as_Register($src1$$reg),
10542 $src2$$constant & 0x1f);
10543 %}
10544
10545 ins_pipe(ialu_reg_shift);
10546 %}
10547
10548 // Combined Int Mask and Right Shift (using UBFM)
10549 // TODO
10550
10551 // Long Shifts
10552
10553 // Shift Left Register
10554 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10555 match(Set dst (LShiftL src1 src2));
10556
10557 ins_cost(INSN_COST * 2);
10558 format %{ "lslv $dst, $src1, $src2" %}
10559
10560 ins_encode %{
10561 __ lslv(as_Register($dst$$reg),
10562 as_Register($src1$$reg),
10563 as_Register($src2$$reg));
10564 %}
10565
10566 ins_pipe(ialu_reg_reg_vshift);
10567 %}
10568
10569 // Shift Left Immediate
10570 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10571 match(Set dst (LShiftL src1 src2));
10572
10573 ins_cost(INSN_COST);
10574 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10575
10576 ins_encode %{
10577 __ lsl(as_Register($dst$$reg),
10578 as_Register($src1$$reg),
10579 $src2$$constant & 0x3f);
10580 %}
10581
10582 ins_pipe(ialu_reg_shift);
10583 %}
10584
10585 // Shift Right Logical Register
10586 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10587 match(Set dst (URShiftL src1 src2));
10588
10589 ins_cost(INSN_COST * 2);
10590 format %{ "lsrv $dst, $src1, $src2" %}
10591
10592 ins_encode %{
10593 __ lsrv(as_Register($dst$$reg),
10594 as_Register($src1$$reg),
10595 as_Register($src2$$reg));
10596 %}
10597
10598 ins_pipe(ialu_reg_reg_vshift);
10599 %}
10600
10601 // Shift Right Logical Immediate
10602 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10603 match(Set dst (URShiftL src1 src2));
10604
10605 ins_cost(INSN_COST);
10606 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10607
10608 ins_encode %{
10609 __ lsr(as_Register($dst$$reg),
10610 as_Register($src1$$reg),
10611 $src2$$constant & 0x3f);
10612 %}
10613
10614 ins_pipe(ialu_reg_shift);
10615 %}
10616
10617 // A special-case pattern for card table stores.
10618 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10619 match(Set dst (URShiftL (CastP2X src1) src2));
10620
10621 ins_cost(INSN_COST);
10622 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10623
10624 ins_encode %{
10625 __ lsr(as_Register($dst$$reg),
10626 as_Register($src1$$reg),
10627 $src2$$constant & 0x3f);
10628 %}
10629
10630 ins_pipe(ialu_reg_shift);
10631 %}
10632
10633 // Shift Right Arithmetic Register
10634 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10635 match(Set dst (RShiftL src1 src2));
10636
10637 ins_cost(INSN_COST * 2);
10638 format %{ "asrv $dst, $src1, $src2" %}
10639
10640 ins_encode %{
10641 __ asrv(as_Register($dst$$reg),
10642 as_Register($src1$$reg),
10643 as_Register($src2$$reg));
10644 %}
10645
10646 ins_pipe(ialu_reg_reg_vshift);
10647 %}
10648
10649 // Shift Right Arithmetic Immediate
10650 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10651 match(Set dst (RShiftL src1 src2));
10652
10653 ins_cost(INSN_COST);
10654 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10655
10656 ins_encode %{
10657 __ asr(as_Register($dst$$reg),
10658 as_Register($src1$$reg),
10659 $src2$$constant & 0x3f);
10660 %}
10661
10662 ins_pipe(ialu_reg_shift);
10663 %}
10664
10665 // BEGIN This section of the file is automatically generated. Do not edit --------------
10666 // This section is generated from aarch64_ad.m4
10667
10668 // This pattern is automatically generated from aarch64_ad.m4
10669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10670 instruct regL_not_reg(iRegLNoSp dst,
10671 iRegL src1, immL_M1 m1,
10672 rFlagsReg cr) %{
10673 match(Set dst (XorL src1 m1));
10674 ins_cost(INSN_COST);
10675 format %{ "eon $dst, $src1, zr" %}
10676
10677 ins_encode %{
10678 __ eon(as_Register($dst$$reg),
10679 as_Register($src1$$reg),
10680 zr,
10681 Assembler::LSL, 0);
10682 %}
10683
10684 ins_pipe(ialu_reg);
10685 %}
10686
10687 // This pattern is automatically generated from aarch64_ad.m4
10688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10689 instruct regI_not_reg(iRegINoSp dst,
10690 iRegIorL2I src1, immI_M1 m1,
10691 rFlagsReg cr) %{
10692 match(Set dst (XorI src1 m1));
10693 ins_cost(INSN_COST);
10694 format %{ "eonw $dst, $src1, zr" %}
10695
10696 ins_encode %{
10697 __ eonw(as_Register($dst$$reg),
10698 as_Register($src1$$reg),
10699 zr,
10700 Assembler::LSL, 0);
10701 %}
10702
10703 ins_pipe(ialu_reg);
10704 %}
10705
10706 // This pattern is automatically generated from aarch64_ad.m4
10707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10708 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10709 immI0 zero, iRegIorL2I src1, immI src2) %{
10710 match(Set dst (SubI zero (URShiftI src1 src2)));
10711
10712 ins_cost(1.9 * INSN_COST);
10713 format %{ "negw $dst, $src1, LSR $src2" %}
10714
10715 ins_encode %{
10716 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10717 Assembler::LSR, $src2$$constant & 0x1f);
10718 %}
10719
10720 ins_pipe(ialu_reg_shift);
10721 %}
10722
10723 // This pattern is automatically generated from aarch64_ad.m4
10724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10725 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10726 immI0 zero, iRegIorL2I src1, immI src2) %{
10727 match(Set dst (SubI zero (RShiftI src1 src2)));
10728
10729 ins_cost(1.9 * INSN_COST);
10730 format %{ "negw $dst, $src1, ASR $src2" %}
10731
10732 ins_encode %{
10733 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10734 Assembler::ASR, $src2$$constant & 0x1f);
10735 %}
10736
10737 ins_pipe(ialu_reg_shift);
10738 %}
10739
10740 // This pattern is automatically generated from aarch64_ad.m4
10741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10742 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10743 immI0 zero, iRegIorL2I src1, immI src2) %{
10744 match(Set dst (SubI zero (LShiftI src1 src2)));
10745
10746 ins_cost(1.9 * INSN_COST);
10747 format %{ "negw $dst, $src1, LSL $src2" %}
10748
10749 ins_encode %{
10750 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10751 Assembler::LSL, $src2$$constant & 0x1f);
10752 %}
10753
10754 ins_pipe(ialu_reg_shift);
10755 %}
10756
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10760 immL0 zero, iRegL src1, immI src2) %{
10761 match(Set dst (SubL zero (URShiftL src1 src2)));
10762
10763 ins_cost(1.9 * INSN_COST);
10764 format %{ "neg $dst, $src1, LSR $src2" %}
10765
10766 ins_encode %{
10767 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10768 Assembler::LSR, $src2$$constant & 0x3f);
10769 %}
10770
10771 ins_pipe(ialu_reg_shift);
10772 %}
10773
10774 // This pattern is automatically generated from aarch64_ad.m4
10775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10776 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10777 immL0 zero, iRegL src1, immI src2) %{
10778 match(Set dst (SubL zero (RShiftL src1 src2)));
10779
10780 ins_cost(1.9 * INSN_COST);
10781 format %{ "neg $dst, $src1, ASR $src2" %}
10782
10783 ins_encode %{
10784 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10785 Assembler::ASR, $src2$$constant & 0x3f);
10786 %}
10787
10788 ins_pipe(ialu_reg_shift);
10789 %}
10790
10791 // This pattern is automatically generated from aarch64_ad.m4
10792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10793 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10794 immL0 zero, iRegL src1, immI src2) %{
10795 match(Set dst (SubL zero (LShiftL src1 src2)));
10796
10797 ins_cost(1.9 * INSN_COST);
10798 format %{ "neg $dst, $src1, LSL $src2" %}
10799
10800 ins_encode %{
10801 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10802 Assembler::LSL, $src2$$constant & 0x3f);
10803 %}
10804
10805 ins_pipe(ialu_reg_shift);
10806 %}
10807
10808 // This pattern is automatically generated from aarch64_ad.m4
10809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10810 instruct AndI_reg_not_reg(iRegINoSp dst,
10811 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10812 match(Set dst (AndI src1 (XorI src2 m1)));
10813 ins_cost(INSN_COST);
10814 format %{ "bicw $dst, $src1, $src2" %}
10815
10816 ins_encode %{
10817 __ bicw(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg),
10820 Assembler::LSL, 0);
10821 %}
10822
10823 ins_pipe(ialu_reg_reg);
10824 %}
10825
10826 // This pattern is automatically generated from aarch64_ad.m4
10827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10828 instruct AndL_reg_not_reg(iRegLNoSp dst,
10829 iRegL src1, iRegL src2, immL_M1 m1) %{
10830 match(Set dst (AndL src1 (XorL src2 m1)));
10831 ins_cost(INSN_COST);
10832 format %{ "bic $dst, $src1, $src2" %}
10833
10834 ins_encode %{
10835 __ bic(as_Register($dst$$reg),
10836 as_Register($src1$$reg),
10837 as_Register($src2$$reg),
10838 Assembler::LSL, 0);
10839 %}
10840
10841 ins_pipe(ialu_reg_reg);
10842 %}
10843
10844 // This pattern is automatically generated from aarch64_ad.m4
10845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10846 instruct OrI_reg_not_reg(iRegINoSp dst,
10847 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10848 match(Set dst (OrI src1 (XorI src2 m1)));
10849 ins_cost(INSN_COST);
10850 format %{ "ornw $dst, $src1, $src2" %}
10851
10852 ins_encode %{
10853 __ ornw(as_Register($dst$$reg),
10854 as_Register($src1$$reg),
10855 as_Register($src2$$reg),
10856 Assembler::LSL, 0);
10857 %}
10858
10859 ins_pipe(ialu_reg_reg);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct OrL_reg_not_reg(iRegLNoSp dst,
10865 iRegL src1, iRegL src2, immL_M1 m1) %{
10866 match(Set dst (OrL src1 (XorL src2 m1)));
10867 ins_cost(INSN_COST);
10868 format %{ "orn $dst, $src1, $src2" %}
10869
10870 ins_encode %{
10871 __ orn(as_Register($dst$$reg),
10872 as_Register($src1$$reg),
10873 as_Register($src2$$reg),
10874 Assembler::LSL, 0);
10875 %}
10876
10877 ins_pipe(ialu_reg_reg);
10878 %}
10879
10880 // This pattern is automatically generated from aarch64_ad.m4
10881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10882 instruct XorI_reg_not_reg(iRegINoSp dst,
10883 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10884 match(Set dst (XorI m1 (XorI src2 src1)));
10885 ins_cost(INSN_COST);
10886 format %{ "eonw $dst, $src1, $src2" %}
10887
10888 ins_encode %{
10889 __ eonw(as_Register($dst$$reg),
10890 as_Register($src1$$reg),
10891 as_Register($src2$$reg),
10892 Assembler::LSL, 0);
10893 %}
10894
10895 ins_pipe(ialu_reg_reg);
10896 %}
10897
10898 // This pattern is automatically generated from aarch64_ad.m4
10899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10900 instruct XorL_reg_not_reg(iRegLNoSp dst,
10901 iRegL src1, iRegL src2, immL_M1 m1) %{
10902 match(Set dst (XorL m1 (XorL src2 src1)));
10903 ins_cost(INSN_COST);
10904 format %{ "eon $dst, $src1, $src2" %}
10905
10906 ins_encode %{
10907 __ eon(as_Register($dst$$reg),
10908 as_Register($src1$$reg),
10909 as_Register($src2$$reg),
10910 Assembler::LSL, 0);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg);
10914 %}
10915
10916 // This pattern is automatically generated from aarch64_ad.m4
10917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10918 // val & (-1 ^ (val >>> shift)) ==> bicw
10919 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10920 iRegIorL2I src1, iRegIorL2I src2,
10921 immI src3, immI_M1 src4) %{
10922 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10923 ins_cost(1.9 * INSN_COST);
10924 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10925
10926 ins_encode %{
10927 __ bicw(as_Register($dst$$reg),
10928 as_Register($src1$$reg),
10929 as_Register($src2$$reg),
10930 Assembler::LSR,
10931 $src3$$constant & 0x1f);
10932 %}
10933
10934 ins_pipe(ialu_reg_reg_shift);
10935 %}
10936
10937 // This pattern is automatically generated from aarch64_ad.m4
10938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10939 // val & (-1 ^ (val >>> shift)) ==> bic
10940 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10941 iRegL src1, iRegL src2,
10942 immI src3, immL_M1 src4) %{
10943 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10944 ins_cost(1.9 * INSN_COST);
10945 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10946
10947 ins_encode %{
10948 __ bic(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::LSR,
10952 $src3$$constant & 0x3f);
10953 %}
10954
10955 ins_pipe(ialu_reg_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 // val & (-1 ^ (val >> shift)) ==> bicw
10961 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10962 iRegIorL2I src1, iRegIorL2I src2,
10963 immI src3, immI_M1 src4) %{
10964 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10965 ins_cost(1.9 * INSN_COST);
10966 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10967
10968 ins_encode %{
10969 __ bicw(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 as_Register($src2$$reg),
10972 Assembler::ASR,
10973 $src3$$constant & 0x1f);
10974 %}
10975
10976 ins_pipe(ialu_reg_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 // val & (-1 ^ (val >> shift)) ==> bic
10982 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10983 iRegL src1, iRegL src2,
10984 immI src3, immL_M1 src4) %{
10985 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10986 ins_cost(1.9 * INSN_COST);
10987 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10988
10989 ins_encode %{
10990 __ bic(as_Register($dst$$reg),
10991 as_Register($src1$$reg),
10992 as_Register($src2$$reg),
10993 Assembler::ASR,
10994 $src3$$constant & 0x3f);
10995 %}
10996
10997 ins_pipe(ialu_reg_reg_shift);
10998 %}
10999
11000 // This pattern is automatically generated from aarch64_ad.m4
11001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11002 // val & (-1 ^ (val ror shift)) ==> bicw
11003 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11004 iRegIorL2I src1, iRegIorL2I src2,
11005 immI src3, immI_M1 src4) %{
11006 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11009
11010 ins_encode %{
11011 __ bicw(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg),
11014 Assembler::ROR,
11015 $src3$$constant & 0x1f);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg_shift);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 // val & (-1 ^ (val ror shift)) ==> bic
11024 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11025 iRegL src1, iRegL src2,
11026 immI src3, immL_M1 src4) %{
11027 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11028 ins_cost(1.9 * INSN_COST);
11029 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11030
11031 ins_encode %{
11032 __ bic(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 as_Register($src2$$reg),
11035 Assembler::ROR,
11036 $src3$$constant & 0x3f);
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 // val & (-1 ^ (val << shift)) ==> bicw
11045 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11046 iRegIorL2I src1, iRegIorL2I src2,
11047 immI src3, immI_M1 src4) %{
11048 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11049 ins_cost(1.9 * INSN_COST);
11050 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11051
11052 ins_encode %{
11053 __ bicw(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::LSL,
11057 $src3$$constant & 0x1f);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg_shift);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 // val & (-1 ^ (val << shift)) ==> bic
11066 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11067 iRegL src1, iRegL src2,
11068 immI src3, immL_M1 src4) %{
11069 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11070 ins_cost(1.9 * INSN_COST);
11071 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11072
11073 ins_encode %{
11074 __ bic(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg),
11077 Assembler::LSL,
11078 $src3$$constant & 0x3f);
11079 %}
11080
11081 ins_pipe(ialu_reg_reg_shift);
11082 %}
11083
11084 // This pattern is automatically generated from aarch64_ad.m4
11085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11086 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11087 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11088 iRegIorL2I src1, iRegIorL2I src2,
11089 immI src3, immI_M1 src4) %{
11090 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11091 ins_cost(1.9 * INSN_COST);
11092 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11093
11094 ins_encode %{
11095 __ eonw(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSR,
11099 $src3$$constant & 0x1f);
11100 %}
11101
11102 ins_pipe(ialu_reg_reg_shift);
11103 %}
11104
11105 // This pattern is automatically generated from aarch64_ad.m4
11106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11107 // val ^ (-1 ^ (val >>> shift)) ==> eon
11108 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11109 iRegL src1, iRegL src2,
11110 immI src3, immL_M1 src4) %{
11111 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11112 ins_cost(1.9 * INSN_COST);
11113 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11114
11115 ins_encode %{
11116 __ eon(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::LSR,
11120 $src3$$constant & 0x3f);
11121 %}
11122
11123 ins_pipe(ialu_reg_reg_shift);
11124 %}
11125
11126 // This pattern is automatically generated from aarch64_ad.m4
11127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11128 // val ^ (-1 ^ (val >> shift)) ==> eonw
11129 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11130 iRegIorL2I src1, iRegIorL2I src2,
11131 immI src3, immI_M1 src4) %{
11132 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11135
11136 ins_encode %{
11137 __ eonw(as_Register($dst$$reg),
11138 as_Register($src1$$reg),
11139 as_Register($src2$$reg),
11140 Assembler::ASR,
11141 $src3$$constant & 0x1f);
11142 %}
11143
11144 ins_pipe(ialu_reg_reg_shift);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 // val ^ (-1 ^ (val >> shift)) ==> eon
11150 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11151 iRegL src1, iRegL src2,
11152 immI src3, immL_M1 src4) %{
11153 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11154 ins_cost(1.9 * INSN_COST);
11155 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11156
11157 ins_encode %{
11158 __ eon(as_Register($dst$$reg),
11159 as_Register($src1$$reg),
11160 as_Register($src2$$reg),
11161 Assembler::ASR,
11162 $src3$$constant & 0x3f);
11163 %}
11164
11165 ins_pipe(ialu_reg_reg_shift);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 // val ^ (-1 ^ (val ror shift)) ==> eonw
11171 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11172 iRegIorL2I src1, iRegIorL2I src2,
11173 immI src3, immI_M1 src4) %{
11174 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11175 ins_cost(1.9 * INSN_COST);
11176 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11177
11178 ins_encode %{
11179 __ eonw(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 as_Register($src2$$reg),
11182 Assembler::ROR,
11183 $src3$$constant & 0x1f);
11184 %}
11185
11186 ins_pipe(ialu_reg_reg_shift);
11187 %}
11188
11189 // This pattern is automatically generated from aarch64_ad.m4
11190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11191 // val ^ (-1 ^ (val ror shift)) ==> eon
11192 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11193 iRegL src1, iRegL src2,
11194 immI src3, immL_M1 src4) %{
11195 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11196 ins_cost(1.9 * INSN_COST);
11197 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11198
11199 ins_encode %{
11200 __ eon(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg),
11203 Assembler::ROR,
11204 $src3$$constant & 0x3f);
11205 %}
11206
11207 ins_pipe(ialu_reg_reg_shift);
11208 %}
11209
11210 // This pattern is automatically generated from aarch64_ad.m4
11211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11212 // val ^ (-1 ^ (val << shift)) ==> eonw
11213 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11214 iRegIorL2I src1, iRegIorL2I src2,
11215 immI src3, immI_M1 src4) %{
11216 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11217 ins_cost(1.9 * INSN_COST);
11218 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11219
11220 ins_encode %{
11221 __ eonw(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg),
11224 Assembler::LSL,
11225 $src3$$constant & 0x1f);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg_shift);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 // val ^ (-1 ^ (val << shift)) ==> eon
11234 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11235 iRegL src1, iRegL src2,
11236 immI src3, immL_M1 src4) %{
11237 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11240
11241 ins_encode %{
11242 __ eon(as_Register($dst$$reg),
11243 as_Register($src1$$reg),
11244 as_Register($src2$$reg),
11245 Assembler::LSL,
11246 $src3$$constant & 0x3f);
11247 %}
11248
11249 ins_pipe(ialu_reg_reg_shift);
11250 %}
11251
11252 // This pattern is automatically generated from aarch64_ad.m4
11253 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11254 // val | (-1 ^ (val >>> shift)) ==> ornw
11255 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11256 iRegIorL2I src1, iRegIorL2I src2,
11257 immI src3, immI_M1 src4) %{
11258 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11261
11262 ins_encode %{
11263 __ ornw(as_Register($dst$$reg),
11264 as_Register($src1$$reg),
11265 as_Register($src2$$reg),
11266 Assembler::LSR,
11267 $src3$$constant & 0x1f);
11268 %}
11269
11270 ins_pipe(ialu_reg_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 // val | (-1 ^ (val >>> shift)) ==> orn
11276 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11277 iRegL src1, iRegL src2,
11278 immI src3, immL_M1 src4) %{
11279 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11280 ins_cost(1.9 * INSN_COST);
11281 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11282
11283 ins_encode %{
11284 __ orn(as_Register($dst$$reg),
11285 as_Register($src1$$reg),
11286 as_Register($src2$$reg),
11287 Assembler::LSR,
11288 $src3$$constant & 0x3f);
11289 %}
11290
11291 ins_pipe(ialu_reg_reg_shift);
11292 %}
11293
11294 // This pattern is automatically generated from aarch64_ad.m4
11295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11296 // val | (-1 ^ (val >> shift)) ==> ornw
11297 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11298 iRegIorL2I src1, iRegIorL2I src2,
11299 immI src3, immI_M1 src4) %{
11300 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11301 ins_cost(1.9 * INSN_COST);
11302 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11303
11304 ins_encode %{
11305 __ ornw(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 Assembler::ASR,
11309 $src3$$constant & 0x1f);
11310 %}
11311
11312 ins_pipe(ialu_reg_reg_shift);
11313 %}
11314
11315 // This pattern is automatically generated from aarch64_ad.m4
11316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11317 // val | (-1 ^ (val >> shift)) ==> orn
11318 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11319 iRegL src1, iRegL src2,
11320 immI src3, immL_M1 src4) %{
11321 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11322 ins_cost(1.9 * INSN_COST);
11323 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11324
11325 ins_encode %{
11326 __ orn(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 as_Register($src2$$reg),
11329 Assembler::ASR,
11330 $src3$$constant & 0x3f);
11331 %}
11332
11333 ins_pipe(ialu_reg_reg_shift);
11334 %}
11335
11336 // This pattern is automatically generated from aarch64_ad.m4
11337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11338 // val | (-1 ^ (val ror shift)) ==> ornw
11339 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11340 iRegIorL2I src1, iRegIorL2I src2,
11341 immI src3, immI_M1 src4) %{
11342 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11343 ins_cost(1.9 * INSN_COST);
11344 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11345
11346 ins_encode %{
11347 __ ornw(as_Register($dst$$reg),
11348 as_Register($src1$$reg),
11349 as_Register($src2$$reg),
11350 Assembler::ROR,
11351 $src3$$constant & 0x1f);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg_shift);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 // val | (-1 ^ (val ror shift)) ==> orn
11360 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11361 iRegL src1, iRegL src2,
11362 immI src3, immL_M1 src4) %{
11363 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11366
11367 ins_encode %{
11368 __ orn(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::ROR,
11372 $src3$$constant & 0x3f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 // val | (-1 ^ (val << shift)) ==> ornw
11381 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11382 iRegIorL2I src1, iRegIorL2I src2,
11383 immI src3, immI_M1 src4) %{
11384 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11387
11388 ins_encode %{
11389 __ ornw(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSL,
11393 $src3$$constant & 0x1f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 // val | (-1 ^ (val << shift)) ==> orn
11402 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11403 iRegL src1, iRegL src2,
11404 immI src3, immL_M1 src4) %{
11405 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11408
11409 ins_encode %{
11410 __ orn(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::LSL,
11414 $src3$$constant & 0x3f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11423 iRegIorL2I src1, iRegIorL2I src2,
11424 immI src3) %{
11425 match(Set dst (AndI src1 (URShiftI src2 src3)));
11426
11427 ins_cost(1.9 * INSN_COST);
11428 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11429
11430 ins_encode %{
11431 __ andw(as_Register($dst$$reg),
11432 as_Register($src1$$reg),
11433 as_Register($src2$$reg),
11434 Assembler::LSR,
11435 $src3$$constant & 0x1f);
11436 %}
11437
11438 ins_pipe(ialu_reg_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11444 iRegL src1, iRegL src2,
11445 immI src3) %{
11446 match(Set dst (AndL src1 (URShiftL src2 src3)));
11447
11448 ins_cost(1.9 * INSN_COST);
11449 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11450
11451 ins_encode %{
11452 __ andr(as_Register($dst$$reg),
11453 as_Register($src1$$reg),
11454 as_Register($src2$$reg),
11455 Assembler::LSR,
11456 $src3$$constant & 0x3f);
11457 %}
11458
11459 ins_pipe(ialu_reg_reg_shift);
11460 %}
11461
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11465 iRegIorL2I src1, iRegIorL2I src2,
11466 immI src3) %{
11467 match(Set dst (AndI src1 (RShiftI src2 src3)));
11468
11469 ins_cost(1.9 * INSN_COST);
11470 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11471
11472 ins_encode %{
11473 __ andw(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::ASR,
11477 $src3$$constant & 0x1f);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg_shift);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11486 iRegL src1, iRegL src2,
11487 immI src3) %{
11488 match(Set dst (AndL src1 (RShiftL src2 src3)));
11489
11490 ins_cost(1.9 * INSN_COST);
11491 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11492
11493 ins_encode %{
11494 __ andr(as_Register($dst$$reg),
11495 as_Register($src1$$reg),
11496 as_Register($src2$$reg),
11497 Assembler::ASR,
11498 $src3$$constant & 0x3f);
11499 %}
11500
11501 ins_pipe(ialu_reg_reg_shift);
11502 %}
11503
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11507 iRegIorL2I src1, iRegIorL2I src2,
11508 immI src3) %{
11509 match(Set dst (AndI src1 (LShiftI src2 src3)));
11510
11511 ins_cost(1.9 * INSN_COST);
11512 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11513
11514 ins_encode %{
11515 __ andw(as_Register($dst$$reg),
11516 as_Register($src1$$reg),
11517 as_Register($src2$$reg),
11518 Assembler::LSL,
11519 $src3$$constant & 0x1f);
11520 %}
11521
11522 ins_pipe(ialu_reg_reg_shift);
11523 %}
11524
11525 // This pattern is automatically generated from aarch64_ad.m4
11526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11527 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11528 iRegL src1, iRegL src2,
11529 immI src3) %{
11530 match(Set dst (AndL src1 (LShiftL src2 src3)));
11531
11532 ins_cost(1.9 * INSN_COST);
11533 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11534
11535 ins_encode %{
11536 __ andr(as_Register($dst$$reg),
11537 as_Register($src1$$reg),
11538 as_Register($src2$$reg),
11539 Assembler::LSL,
11540 $src3$$constant & 0x3f);
11541 %}
11542
11543 ins_pipe(ialu_reg_reg_shift);
11544 %}
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11549 iRegIorL2I src1, iRegIorL2I src2,
11550 immI src3) %{
11551 match(Set dst (AndI src1 (RotateRight src2 src3)));
11552
11553 ins_cost(1.9 * INSN_COST);
11554 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11555
11556 ins_encode %{
11557 __ andw(as_Register($dst$$reg),
11558 as_Register($src1$$reg),
11559 as_Register($src2$$reg),
11560 Assembler::ROR,
11561 $src3$$constant & 0x1f);
11562 %}
11563
11564 ins_pipe(ialu_reg_reg_shift);
11565 %}
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11570 iRegL src1, iRegL src2,
11571 immI src3) %{
11572 match(Set dst (AndL src1 (RotateRight src2 src3)));
11573
11574 ins_cost(1.9 * INSN_COST);
11575 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11576
11577 ins_encode %{
11578 __ andr(as_Register($dst$$reg),
11579 as_Register($src1$$reg),
11580 as_Register($src2$$reg),
11581 Assembler::ROR,
11582 $src3$$constant & 0x3f);
11583 %}
11584
11585 ins_pipe(ialu_reg_reg_shift);
11586 %}
11587
11588 // This pattern is automatically generated from aarch64_ad.m4
11589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11590 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11591 iRegIorL2I src1, iRegIorL2I src2,
11592 immI src3) %{
11593 match(Set dst (XorI src1 (URShiftI src2 src3)));
11594
11595 ins_cost(1.9 * INSN_COST);
11596 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11597
11598 ins_encode %{
11599 __ eorw(as_Register($dst$$reg),
11600 as_Register($src1$$reg),
11601 as_Register($src2$$reg),
11602 Assembler::LSR,
11603 $src3$$constant & 0x1f);
11604 %}
11605
11606 ins_pipe(ialu_reg_reg_shift);
11607 %}
11608
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11612 iRegL src1, iRegL src2,
11613 immI src3) %{
11614 match(Set dst (XorL src1 (URShiftL src2 src3)));
11615
11616 ins_cost(1.9 * INSN_COST);
11617 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11618
11619 ins_encode %{
11620 __ eor(as_Register($dst$$reg),
11621 as_Register($src1$$reg),
11622 as_Register($src2$$reg),
11623 Assembler::LSR,
11624 $src3$$constant & 0x3f);
11625 %}
11626
11627 ins_pipe(ialu_reg_reg_shift);
11628 %}
11629
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11633 iRegIorL2I src1, iRegIorL2I src2,
11634 immI src3) %{
11635 match(Set dst (XorI src1 (RShiftI src2 src3)));
11636
11637 ins_cost(1.9 * INSN_COST);
11638 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11639
11640 ins_encode %{
11641 __ eorw(as_Register($dst$$reg),
11642 as_Register($src1$$reg),
11643 as_Register($src2$$reg),
11644 Assembler::ASR,
11645 $src3$$constant & 0x1f);
11646 %}
11647
11648 ins_pipe(ialu_reg_reg_shift);
11649 %}
11650
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11654 iRegL src1, iRegL src2,
11655 immI src3) %{
11656 match(Set dst (XorL src1 (RShiftL src2 src3)));
11657
11658 ins_cost(1.9 * INSN_COST);
11659 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11660
11661 ins_encode %{
11662 __ eor(as_Register($dst$$reg),
11663 as_Register($src1$$reg),
11664 as_Register($src2$$reg),
11665 Assembler::ASR,
11666 $src3$$constant & 0x3f);
11667 %}
11668
11669 ins_pipe(ialu_reg_reg_shift);
11670 %}
11671
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11675 iRegIorL2I src1, iRegIorL2I src2,
11676 immI src3) %{
11677 match(Set dst (XorI src1 (LShiftI src2 src3)));
11678
11679 ins_cost(1.9 * INSN_COST);
11680 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11681
11682 ins_encode %{
11683 __ eorw(as_Register($dst$$reg),
11684 as_Register($src1$$reg),
11685 as_Register($src2$$reg),
11686 Assembler::LSL,
11687 $src3$$constant & 0x1f);
11688 %}
11689
11690 ins_pipe(ialu_reg_reg_shift);
11691 %}
11692
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11696 iRegL src1, iRegL src2,
11697 immI src3) %{
11698 match(Set dst (XorL src1 (LShiftL src2 src3)));
11699
11700 ins_cost(1.9 * INSN_COST);
11701 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11702
11703 ins_encode %{
11704 __ eor(as_Register($dst$$reg),
11705 as_Register($src1$$reg),
11706 as_Register($src2$$reg),
11707 Assembler::LSL,
11708 $src3$$constant & 0x3f);
11709 %}
11710
11711 ins_pipe(ialu_reg_reg_shift);
11712 %}
11713
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11717 iRegIorL2I src1, iRegIorL2I src2,
11718 immI src3) %{
11719 match(Set dst (XorI src1 (RotateRight src2 src3)));
11720
11721 ins_cost(1.9 * INSN_COST);
11722 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11723
11724 ins_encode %{
11725 __ eorw(as_Register($dst$$reg),
11726 as_Register($src1$$reg),
11727 as_Register($src2$$reg),
11728 Assembler::ROR,
11729 $src3$$constant & 0x1f);
11730 %}
11731
11732 ins_pipe(ialu_reg_reg_shift);
11733 %}
11734
11735 // This pattern is automatically generated from aarch64_ad.m4
11736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11737 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11738 iRegL src1, iRegL src2,
11739 immI src3) %{
11740 match(Set dst (XorL src1 (RotateRight src2 src3)));
11741
11742 ins_cost(1.9 * INSN_COST);
11743 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11744
11745 ins_encode %{
11746 __ eor(as_Register($dst$$reg),
11747 as_Register($src1$$reg),
11748 as_Register($src2$$reg),
11749 Assembler::ROR,
11750 $src3$$constant & 0x3f);
11751 %}
11752
11753 ins_pipe(ialu_reg_reg_shift);
11754 %}
11755
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11759 iRegIorL2I src1, iRegIorL2I src2,
11760 immI src3) %{
11761 match(Set dst (OrI src1 (URShiftI src2 src3)));
11762
11763 ins_cost(1.9 * INSN_COST);
11764 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11765
11766 ins_encode %{
11767 __ orrw(as_Register($dst$$reg),
11768 as_Register($src1$$reg),
11769 as_Register($src2$$reg),
11770 Assembler::LSR,
11771 $src3$$constant & 0x1f);
11772 %}
11773
11774 ins_pipe(ialu_reg_reg_shift);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11780 iRegL src1, iRegL src2,
11781 immI src3) %{
11782 match(Set dst (OrL src1 (URShiftL src2 src3)));
11783
11784 ins_cost(1.9 * INSN_COST);
11785 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11786
11787 ins_encode %{
11788 __ orr(as_Register($dst$$reg),
11789 as_Register($src1$$reg),
11790 as_Register($src2$$reg),
11791 Assembler::LSR,
11792 $src3$$constant & 0x3f);
11793 %}
11794
11795 ins_pipe(ialu_reg_reg_shift);
11796 %}
11797
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11801 iRegIorL2I src1, iRegIorL2I src2,
11802 immI src3) %{
11803 match(Set dst (OrI src1 (RShiftI src2 src3)));
11804
11805 ins_cost(1.9 * INSN_COST);
11806 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11807
11808 ins_encode %{
11809 __ orrw(as_Register($dst$$reg),
11810 as_Register($src1$$reg),
11811 as_Register($src2$$reg),
11812 Assembler::ASR,
11813 $src3$$constant & 0x1f);
11814 %}
11815
11816 ins_pipe(ialu_reg_reg_shift);
11817 %}
11818
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11822 iRegL src1, iRegL src2,
11823 immI src3) %{
11824 match(Set dst (OrL src1 (RShiftL src2 src3)));
11825
11826 ins_cost(1.9 * INSN_COST);
11827 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11828
11829 ins_encode %{
11830 __ orr(as_Register($dst$$reg),
11831 as_Register($src1$$reg),
11832 as_Register($src2$$reg),
11833 Assembler::ASR,
11834 $src3$$constant & 0x3f);
11835 %}
11836
11837 ins_pipe(ialu_reg_reg_shift);
11838 %}
11839
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11843 iRegIorL2I src1, iRegIorL2I src2,
11844 immI src3) %{
11845 match(Set dst (OrI src1 (LShiftI src2 src3)));
11846
11847 ins_cost(1.9 * INSN_COST);
11848 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11849
11850 ins_encode %{
11851 __ orrw(as_Register($dst$$reg),
11852 as_Register($src1$$reg),
11853 as_Register($src2$$reg),
11854 Assembler::LSL,
11855 $src3$$constant & 0x1f);
11856 %}
11857
11858 ins_pipe(ialu_reg_reg_shift);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11864 iRegL src1, iRegL src2,
11865 immI src3) %{
11866 match(Set dst (OrL src1 (LShiftL src2 src3)));
11867
11868 ins_cost(1.9 * INSN_COST);
11869 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11870
11871 ins_encode %{
11872 __ orr(as_Register($dst$$reg),
11873 as_Register($src1$$reg),
11874 as_Register($src2$$reg),
11875 Assembler::LSL,
11876 $src3$$constant & 0x3f);
11877 %}
11878
11879 ins_pipe(ialu_reg_reg_shift);
11880 %}
11881
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11885 iRegIorL2I src1, iRegIorL2I src2,
11886 immI src3) %{
11887 match(Set dst (OrI src1 (RotateRight src2 src3)));
11888
11889 ins_cost(1.9 * INSN_COST);
11890 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11891
11892 ins_encode %{
11893 __ orrw(as_Register($dst$$reg),
11894 as_Register($src1$$reg),
11895 as_Register($src2$$reg),
11896 Assembler::ROR,
11897 $src3$$constant & 0x1f);
11898 %}
11899
11900 ins_pipe(ialu_reg_reg_shift);
11901 %}
11902
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11906 iRegL src1, iRegL src2,
11907 immI src3) %{
11908 match(Set dst (OrL src1 (RotateRight src2 src3)));
11909
11910 ins_cost(1.9 * INSN_COST);
11911 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11912
11913 ins_encode %{
11914 __ orr(as_Register($dst$$reg),
11915 as_Register($src1$$reg),
11916 as_Register($src2$$reg),
11917 Assembler::ROR,
11918 $src3$$constant & 0x3f);
11919 %}
11920
11921 ins_pipe(ialu_reg_reg_shift);
11922 %}
11923
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11927 iRegIorL2I src1, iRegIorL2I src2,
11928 immI src3) %{
11929 match(Set dst (AddI src1 (URShiftI src2 src3)));
11930
11931 ins_cost(1.9 * INSN_COST);
11932 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11933
11934 ins_encode %{
11935 __ addw(as_Register($dst$$reg),
11936 as_Register($src1$$reg),
11937 as_Register($src2$$reg),
11938 Assembler::LSR,
11939 $src3$$constant & 0x1f);
11940 %}
11941
11942 ins_pipe(ialu_reg_reg_shift);
11943 %}
11944
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11948 iRegL src1, iRegL src2,
11949 immI src3) %{
11950 match(Set dst (AddL src1 (URShiftL src2 src3)));
11951
11952 ins_cost(1.9 * INSN_COST);
11953 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11954
11955 ins_encode %{
11956 __ add(as_Register($dst$$reg),
11957 as_Register($src1$$reg),
11958 as_Register($src2$$reg),
11959 Assembler::LSR,
11960 $src3$$constant & 0x3f);
11961 %}
11962
11963 ins_pipe(ialu_reg_reg_shift);
11964 %}
11965
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11969 iRegIorL2I src1, iRegIorL2I src2,
11970 immI src3) %{
11971 match(Set dst (AddI src1 (RShiftI src2 src3)));
11972
11973 ins_cost(1.9 * INSN_COST);
11974 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11975
11976 ins_encode %{
11977 __ addw(as_Register($dst$$reg),
11978 as_Register($src1$$reg),
11979 as_Register($src2$$reg),
11980 Assembler::ASR,
11981 $src3$$constant & 0x1f);
11982 %}
11983
11984 ins_pipe(ialu_reg_reg_shift);
11985 %}
11986
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11990 iRegL src1, iRegL src2,
11991 immI src3) %{
11992 match(Set dst (AddL src1 (RShiftL src2 src3)));
11993
11994 ins_cost(1.9 * INSN_COST);
11995 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11996
11997 ins_encode %{
11998 __ add(as_Register($dst$$reg),
11999 as_Register($src1$$reg),
12000 as_Register($src2$$reg),
12001 Assembler::ASR,
12002 $src3$$constant & 0x3f);
12003 %}
12004
12005 ins_pipe(ialu_reg_reg_shift);
12006 %}
12007
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12011 iRegIorL2I src1, iRegIorL2I src2,
12012 immI src3) %{
12013 match(Set dst (AddI src1 (LShiftI src2 src3)));
12014
12015 ins_cost(1.9 * INSN_COST);
12016 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12017
12018 ins_encode %{
12019 __ addw(as_Register($dst$$reg),
12020 as_Register($src1$$reg),
12021 as_Register($src2$$reg),
12022 Assembler::LSL,
12023 $src3$$constant & 0x1f);
12024 %}
12025
12026 ins_pipe(ialu_reg_reg_shift);
12027 %}
12028
12029 // This pattern is automatically generated from aarch64_ad.m4
12030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12031 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12032 iRegL src1, iRegL src2,
12033 immI src3) %{
12034 match(Set dst (AddL src1 (LShiftL src2 src3)));
12035
12036 ins_cost(1.9 * INSN_COST);
12037 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12038
12039 ins_encode %{
12040 __ add(as_Register($dst$$reg),
12041 as_Register($src1$$reg),
12042 as_Register($src2$$reg),
12043 Assembler::LSL,
12044 $src3$$constant & 0x3f);
12045 %}
12046
12047 ins_pipe(ialu_reg_reg_shift);
12048 %}
12049
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12053 iRegIorL2I src1, iRegIorL2I src2,
12054 immI src3) %{
12055 match(Set dst (SubI src1 (URShiftI src2 src3)));
12056
12057 ins_cost(1.9 * INSN_COST);
12058 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12059
12060 ins_encode %{
12061 __ subw(as_Register($dst$$reg),
12062 as_Register($src1$$reg),
12063 as_Register($src2$$reg),
12064 Assembler::LSR,
12065 $src3$$constant & 0x1f);
12066 %}
12067
12068 ins_pipe(ialu_reg_reg_shift);
12069 %}
12070
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12074 iRegL src1, iRegL src2,
12075 immI src3) %{
12076 match(Set dst (SubL src1 (URShiftL src2 src3)));
12077
12078 ins_cost(1.9 * INSN_COST);
12079 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12080
12081 ins_encode %{
12082 __ sub(as_Register($dst$$reg),
12083 as_Register($src1$$reg),
12084 as_Register($src2$$reg),
12085 Assembler::LSR,
12086 $src3$$constant & 0x3f);
12087 %}
12088
12089 ins_pipe(ialu_reg_reg_shift);
12090 %}
12091
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12095 iRegIorL2I src1, iRegIorL2I src2,
12096 immI src3) %{
12097 match(Set dst (SubI src1 (RShiftI src2 src3)));
12098
12099 ins_cost(1.9 * INSN_COST);
12100 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12101
12102 ins_encode %{
12103 __ subw(as_Register($dst$$reg),
12104 as_Register($src1$$reg),
12105 as_Register($src2$$reg),
12106 Assembler::ASR,
12107 $src3$$constant & 0x1f);
12108 %}
12109
12110 ins_pipe(ialu_reg_reg_shift);
12111 %}
12112
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12116 iRegL src1, iRegL src2,
12117 immI src3) %{
12118 match(Set dst (SubL src1 (RShiftL src2 src3)));
12119
12120 ins_cost(1.9 * INSN_COST);
12121 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12122
12123 ins_encode %{
12124 __ sub(as_Register($dst$$reg),
12125 as_Register($src1$$reg),
12126 as_Register($src2$$reg),
12127 Assembler::ASR,
12128 $src3$$constant & 0x3f);
12129 %}
12130
12131 ins_pipe(ialu_reg_reg_shift);
12132 %}
12133
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12137 iRegIorL2I src1, iRegIorL2I src2,
12138 immI src3) %{
12139 match(Set dst (SubI src1 (LShiftI src2 src3)));
12140
12141 ins_cost(1.9 * INSN_COST);
12142 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12143
12144 ins_encode %{
12145 __ subw(as_Register($dst$$reg),
12146 as_Register($src1$$reg),
12147 as_Register($src2$$reg),
12148 Assembler::LSL,
12149 $src3$$constant & 0x1f);
12150 %}
12151
12152 ins_pipe(ialu_reg_reg_shift);
12153 %}
12154
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12158 iRegL src1, iRegL src2,
12159 immI src3) %{
12160 match(Set dst (SubL src1 (LShiftL src2 src3)));
12161
12162 ins_cost(1.9 * INSN_COST);
12163 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12164
12165 ins_encode %{
12166 __ sub(as_Register($dst$$reg),
12167 as_Register($src1$$reg),
12168 as_Register($src2$$reg),
12169 Assembler::LSL,
12170 $src3$$constant & 0x3f);
12171 %}
12172
12173 ins_pipe(ialu_reg_reg_shift);
12174 %}
12175
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178
12179 // Shift Left followed by Shift Right.
12180 // This idiom is used by the compiler for the i2b bytecode etc.
12181 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12182 %{
12183 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12184 ins_cost(INSN_COST * 2);
12185 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12186 ins_encode %{
12187 int lshift = $lshift_count$$constant & 63;
12188 int rshift = $rshift_count$$constant & 63;
12189 int s = 63 - lshift;
12190 int r = (rshift - lshift) & 63;
12191 __ sbfm(as_Register($dst$$reg),
12192 as_Register($src$$reg),
12193 r, s);
12194 %}
12195
12196 ins_pipe(ialu_reg_shift);
12197 %}
12198
12199 // This pattern is automatically generated from aarch64_ad.m4
12200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12201
12202 // Shift Left followed by Shift Right.
12203 // This idiom is used by the compiler for the i2b bytecode etc.
12204 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12205 %{
12206 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12207 ins_cost(INSN_COST * 2);
12208 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12209 ins_encode %{
12210 int lshift = $lshift_count$$constant & 31;
12211 int rshift = $rshift_count$$constant & 31;
12212 int s = 31 - lshift;
12213 int r = (rshift - lshift) & 31;
12214 __ sbfmw(as_Register($dst$$reg),
12215 as_Register($src$$reg),
12216 r, s);
12217 %}
12218
12219 ins_pipe(ialu_reg_shift);
12220 %}
12221
12222 // This pattern is automatically generated from aarch64_ad.m4
12223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12224
12225 // Shift Left followed by Shift Right.
12226 // This idiom is used by the compiler for the i2b bytecode etc.
12227 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12228 %{
12229 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12230 ins_cost(INSN_COST * 2);
12231 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12232 ins_encode %{
12233 int lshift = $lshift_count$$constant & 63;
12234 int rshift = $rshift_count$$constant & 63;
12235 int s = 63 - lshift;
12236 int r = (rshift - lshift) & 63;
12237 __ ubfm(as_Register($dst$$reg),
12238 as_Register($src$$reg),
12239 r, s);
12240 %}
12241
12242 ins_pipe(ialu_reg_shift);
12243 %}
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247
12248 // Shift Left followed by Shift Right.
12249 // This idiom is used by the compiler for the i2b bytecode etc.
12250 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12251 %{
12252 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12253 ins_cost(INSN_COST * 2);
12254 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12255 ins_encode %{
12256 int lshift = $lshift_count$$constant & 31;
12257 int rshift = $rshift_count$$constant & 31;
12258 int s = 31 - lshift;
12259 int r = (rshift - lshift) & 31;
12260 __ ubfmw(as_Register($dst$$reg),
12261 as_Register($src$$reg),
12262 r, s);
12263 %}
12264
12265 ins_pipe(ialu_reg_shift);
12266 %}
12267
12268 // Bitfield extract with shift & mask
12269
12270 // This pattern is automatically generated from aarch64_ad.m4
12271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12272 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12273 %{
12274 match(Set dst (AndI (URShiftI src rshift) mask));
12275 // Make sure we are not going to exceed what ubfxw can do.
12276 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12277
12278 ins_cost(INSN_COST);
12279 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12280 ins_encode %{
12281 int rshift = $rshift$$constant & 31;
12282 intptr_t mask = $mask$$constant;
12283 int width = exact_log2(mask+1);
12284 __ ubfxw(as_Register($dst$$reg),
12285 as_Register($src$$reg), rshift, width);
12286 %}
12287 ins_pipe(ialu_reg_shift);
12288 %}
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12293 %{
12294 match(Set dst (AndL (URShiftL src rshift) mask));
12295 // Make sure we are not going to exceed what ubfx can do.
12296 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12297
12298 ins_cost(INSN_COST);
12299 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12300 ins_encode %{
12301 int rshift = $rshift$$constant & 63;
12302 intptr_t mask = $mask$$constant;
12303 int width = exact_log2_long(mask+1);
12304 __ ubfx(as_Register($dst$$reg),
12305 as_Register($src$$reg), rshift, width);
12306 %}
12307 ins_pipe(ialu_reg_shift);
12308 %}
12309
12310
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313
12314 // We can use ubfx when extending an And with a mask when we know mask
12315 // is positive. We know that because immI_bitmask guarantees it.
12316 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12317 %{
12318 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12319 // Make sure we are not going to exceed what ubfxw can do.
12320 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12321
12322 ins_cost(INSN_COST * 2);
12323 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12324 ins_encode %{
12325 int rshift = $rshift$$constant & 31;
12326 intptr_t mask = $mask$$constant;
12327 int width = exact_log2(mask+1);
12328 __ ubfx(as_Register($dst$$reg),
12329 as_Register($src$$reg), rshift, width);
12330 %}
12331 ins_pipe(ialu_reg_shift);
12332 %}
12333
12334
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337
12338 // We can use ubfiz when masking by a positive number and then left shifting the result.
12339 // We know that the mask is positive because immI_bitmask guarantees it.
12340 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12341 %{
12342 match(Set dst (LShiftI (AndI src mask) lshift));
12343 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12344
12345 ins_cost(INSN_COST);
12346 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12347 ins_encode %{
12348 int lshift = $lshift$$constant & 31;
12349 intptr_t mask = $mask$$constant;
12350 int width = exact_log2(mask+1);
12351 __ ubfizw(as_Register($dst$$reg),
12352 as_Register($src$$reg), lshift, width);
12353 %}
12354 ins_pipe(ialu_reg_shift);
12355 %}
12356
12357 // This pattern is automatically generated from aarch64_ad.m4
12358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12359
12360 // We can use ubfiz when masking by a positive number and then left shifting the result.
12361 // We know that the mask is positive because immL_bitmask guarantees it.
12362 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12363 %{
12364 match(Set dst (LShiftL (AndL src mask) lshift));
12365 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12366
12367 ins_cost(INSN_COST);
12368 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12369 ins_encode %{
12370 int lshift = $lshift$$constant & 63;
12371 intptr_t mask = $mask$$constant;
12372 int width = exact_log2_long(mask+1);
12373 __ ubfiz(as_Register($dst$$reg),
12374 as_Register($src$$reg), lshift, width);
12375 %}
12376 ins_pipe(ialu_reg_shift);
12377 %}
12378
12379 // This pattern is automatically generated from aarch64_ad.m4
12380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12381
12382 // We can use ubfiz when masking by a positive number and then left shifting the result.
12383 // We know that the mask is positive because immI_bitmask guarantees it.
12384 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12385 %{
12386 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12387 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12388
12389 ins_cost(INSN_COST);
12390 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12391 ins_encode %{
12392 int lshift = $lshift$$constant & 31;
12393 intptr_t mask = $mask$$constant;
12394 int width = exact_log2(mask+1);
12395 __ ubfizw(as_Register($dst$$reg),
12396 as_Register($src$$reg), lshift, width);
12397 %}
12398 ins_pipe(ialu_reg_shift);
12399 %}
12400
12401 // This pattern is automatically generated from aarch64_ad.m4
12402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12403
12404 // We can use ubfiz when masking by a positive number and then left shifting the result.
12405 // We know that the mask is positive because immL_bitmask guarantees it.
12406 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12407 %{
12408 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12409 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12410
12411 ins_cost(INSN_COST);
12412 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12413 ins_encode %{
12414 int lshift = $lshift$$constant & 63;
12415 intptr_t mask = $mask$$constant;
12416 int width = exact_log2_long(mask+1);
12417 __ ubfiz(as_Register($dst$$reg),
12418 as_Register($src$$reg), lshift, width);
12419 %}
12420 ins_pipe(ialu_reg_shift);
12421 %}
12422
12423
12424 // This pattern is automatically generated from aarch64_ad.m4
12425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12426
12427 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12428 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12429 %{
12430 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12431 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12432
12433 ins_cost(INSN_COST);
12434 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12435 ins_encode %{
12436 int lshift = $lshift$$constant & 63;
12437 intptr_t mask = $mask$$constant;
12438 int width = exact_log2(mask+1);
12439 __ ubfiz(as_Register($dst$$reg),
12440 as_Register($src$$reg), lshift, width);
12441 %}
12442 ins_pipe(ialu_reg_shift);
12443 %}
12444
12445 // This pattern is automatically generated from aarch64_ad.m4
12446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12447
12448 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12449 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12450 %{
12451 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12452 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12453
12454 ins_cost(INSN_COST);
12455 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12456 ins_encode %{
12457 int lshift = $lshift$$constant & 31;
12458 intptr_t mask = $mask$$constant;
12459 int width = exact_log2(mask+1);
12460 __ ubfiz(as_Register($dst$$reg),
12461 as_Register($src$$reg), lshift, width);
12462 %}
12463 ins_pipe(ialu_reg_shift);
12464 %}
12465
12466 // This pattern is automatically generated from aarch64_ad.m4
12467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12468
12469 // Can skip int2long conversions after AND with small bitmask
12470 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12471 %{
12472 match(Set dst (ConvI2L (AndI src msk)));
12473 ins_cost(INSN_COST);
12474 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12475 ins_encode %{
12476 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12477 %}
12478 ins_pipe(ialu_reg_shift);
12479 %}
12480
12481
12482 // Rotations
12483
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12487 %{
12488 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12489 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12490
12491 ins_cost(INSN_COST);
12492 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12493
12494 ins_encode %{
12495 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12496 $rshift$$constant & 63);
12497 %}
12498 ins_pipe(ialu_reg_reg_extr);
12499 %}
12500
12501
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12505 %{
12506 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12507 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12508
12509 ins_cost(INSN_COST);
12510 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12511
12512 ins_encode %{
12513 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12514 $rshift$$constant & 31);
12515 %}
12516 ins_pipe(ialu_reg_reg_extr);
12517 %}
12518
12519
12520 // This pattern is automatically generated from aarch64_ad.m4
12521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12522 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12523 %{
12524 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12525 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12526
12527 ins_cost(INSN_COST);
12528 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12529
12530 ins_encode %{
12531 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12532 $rshift$$constant & 63);
12533 %}
12534 ins_pipe(ialu_reg_reg_extr);
12535 %}
12536
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12541 %{
12542 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12543 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12544
12545 ins_cost(INSN_COST);
12546 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12547
12548 ins_encode %{
12549 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12550 $rshift$$constant & 31);
12551 %}
12552 ins_pipe(ialu_reg_reg_extr);
12553 %}
12554
12555 // This pattern is automatically generated from aarch64_ad.m4
12556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12557 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12558 %{
12559 match(Set dst (RotateRight src shift));
12560
12561 ins_cost(INSN_COST);
12562 format %{ "ror $dst, $src, $shift" %}
12563
12564 ins_encode %{
12565 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12566 $shift$$constant & 0x1f);
12567 %}
12568 ins_pipe(ialu_reg_reg_vshift);
12569 %}
12570
12571 // This pattern is automatically generated from aarch64_ad.m4
12572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12573 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12574 %{
12575 match(Set dst (RotateRight src shift));
12576
12577 ins_cost(INSN_COST);
12578 format %{ "ror $dst, $src, $shift" %}
12579
12580 ins_encode %{
12581 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12582 $shift$$constant & 0x3f);
12583 %}
12584 ins_pipe(ialu_reg_reg_vshift);
12585 %}
12586
12587 // This pattern is automatically generated from aarch64_ad.m4
12588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12589 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12590 %{
12591 match(Set dst (RotateRight src shift));
12592
12593 ins_cost(INSN_COST);
12594 format %{ "ror $dst, $src, $shift" %}
12595
12596 ins_encode %{
12597 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12598 %}
12599 ins_pipe(ialu_reg_reg_vshift);
12600 %}
12601
12602 // This pattern is automatically generated from aarch64_ad.m4
12603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12604 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12605 %{
12606 match(Set dst (RotateRight src shift));
12607
12608 ins_cost(INSN_COST);
12609 format %{ "ror $dst, $src, $shift" %}
12610
12611 ins_encode %{
12612 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12613 %}
12614 ins_pipe(ialu_reg_reg_vshift);
12615 %}
12616
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12620 %{
12621 match(Set dst (RotateLeft src shift));
12622
12623 ins_cost(INSN_COST);
12624 format %{ "rol $dst, $src, $shift" %}
12625
12626 ins_encode %{
12627 __ subw(rscratch1, zr, as_Register($shift$$reg));
12628 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12629 %}
12630 ins_pipe(ialu_reg_reg_vshift);
12631 %}
12632
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12636 %{
12637 match(Set dst (RotateLeft src shift));
12638
12639 ins_cost(INSN_COST);
12640 format %{ "rol $dst, $src, $shift" %}
12641
12642 ins_encode %{
12643 __ subw(rscratch1, zr, as_Register($shift$$reg));
12644 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12645 %}
12646 ins_pipe(ialu_reg_reg_vshift);
12647 %}
12648
12649
12650 // Add/subtract (extended)
12651
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12655 %{
12656 match(Set dst (AddL src1 (ConvI2L src2)));
12657 ins_cost(INSN_COST);
12658 format %{ "add $dst, $src1, $src2, sxtw" %}
12659
12660 ins_encode %{
12661 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12662 as_Register($src2$$reg), ext::sxtw);
12663 %}
12664 ins_pipe(ialu_reg_reg);
12665 %}
12666
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12670 %{
12671 match(Set dst (SubL src1 (ConvI2L src2)));
12672 ins_cost(INSN_COST);
12673 format %{ "sub $dst, $src1, $src2, sxtw" %}
12674
12675 ins_encode %{
12676 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12677 as_Register($src2$$reg), ext::sxtw);
12678 %}
12679 ins_pipe(ialu_reg_reg);
12680 %}
12681
12682 // This pattern is automatically generated from aarch64_ad.m4
12683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12684 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12685 %{
12686 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12687 ins_cost(INSN_COST);
12688 format %{ "add $dst, $src1, $src2, sxth" %}
12689
12690 ins_encode %{
12691 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12692 as_Register($src2$$reg), ext::sxth);
12693 %}
12694 ins_pipe(ialu_reg_reg);
12695 %}
12696
12697 // This pattern is automatically generated from aarch64_ad.m4
12698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12699 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12700 %{
12701 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12702 ins_cost(INSN_COST);
12703 format %{ "add $dst, $src1, $src2, sxtb" %}
12704
12705 ins_encode %{
12706 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12707 as_Register($src2$$reg), ext::sxtb);
12708 %}
12709 ins_pipe(ialu_reg_reg);
12710 %}
12711
12712 // This pattern is automatically generated from aarch64_ad.m4
12713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12714 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12715 %{
12716 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12717 ins_cost(INSN_COST);
12718 format %{ "add $dst, $src1, $src2, uxtb" %}
12719
12720 ins_encode %{
12721 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12722 as_Register($src2$$reg), ext::uxtb);
12723 %}
12724 ins_pipe(ialu_reg_reg);
12725 %}
12726
12727 // This pattern is automatically generated from aarch64_ad.m4
12728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12729 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12730 %{
12731 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12732 ins_cost(INSN_COST);
12733 format %{ "add $dst, $src1, $src2, sxth" %}
12734
12735 ins_encode %{
12736 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12737 as_Register($src2$$reg), ext::sxth);
12738 %}
12739 ins_pipe(ialu_reg_reg);
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12745 %{
12746 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12747 ins_cost(INSN_COST);
12748 format %{ "add $dst, $src1, $src2, sxtw" %}
12749
12750 ins_encode %{
12751 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12752 as_Register($src2$$reg), ext::sxtw);
12753 %}
12754 ins_pipe(ialu_reg_reg);
12755 %}
12756
12757 // This pattern is automatically generated from aarch64_ad.m4
12758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12759 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12760 %{
12761 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12762 ins_cost(INSN_COST);
12763 format %{ "add $dst, $src1, $src2, sxtb" %}
12764
12765 ins_encode %{
12766 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12767 as_Register($src2$$reg), ext::sxtb);
12768 %}
12769 ins_pipe(ialu_reg_reg);
12770 %}
12771
12772 // This pattern is automatically generated from aarch64_ad.m4
12773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12774 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12775 %{
12776 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12777 ins_cost(INSN_COST);
12778 format %{ "add $dst, $src1, $src2, uxtb" %}
12779
12780 ins_encode %{
12781 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12782 as_Register($src2$$reg), ext::uxtb);
12783 %}
12784 ins_pipe(ialu_reg_reg);
12785 %}
12786
12787 // This pattern is automatically generated from aarch64_ad.m4
12788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12789 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12790 %{
12791 match(Set dst (AddI src1 (AndI src2 mask)));
12792 ins_cost(INSN_COST);
12793 format %{ "addw $dst, $src1, $src2, uxtb" %}
12794
12795 ins_encode %{
12796 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12797 as_Register($src2$$reg), ext::uxtb);
12798 %}
12799 ins_pipe(ialu_reg_reg);
12800 %}
12801
12802 // This pattern is automatically generated from aarch64_ad.m4
12803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12804 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12805 %{
12806 match(Set dst (AddI src1 (AndI src2 mask)));
12807 ins_cost(INSN_COST);
12808 format %{ "addw $dst, $src1, $src2, uxth" %}
12809
12810 ins_encode %{
12811 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12812 as_Register($src2$$reg), ext::uxth);
12813 %}
12814 ins_pipe(ialu_reg_reg);
12815 %}
12816
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12820 %{
12821 match(Set dst (AddL src1 (AndL src2 mask)));
12822 ins_cost(INSN_COST);
12823 format %{ "add $dst, $src1, $src2, uxtb" %}
12824
12825 ins_encode %{
12826 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12827 as_Register($src2$$reg), ext::uxtb);
12828 %}
12829 ins_pipe(ialu_reg_reg);
12830 %}
12831
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12835 %{
12836 match(Set dst (AddL src1 (AndL src2 mask)));
12837 ins_cost(INSN_COST);
12838 format %{ "add $dst, $src1, $src2, uxth" %}
12839
12840 ins_encode %{
12841 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12842 as_Register($src2$$reg), ext::uxth);
12843 %}
12844 ins_pipe(ialu_reg_reg);
12845 %}
12846
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12850 %{
12851 match(Set dst (AddL src1 (AndL src2 mask)));
12852 ins_cost(INSN_COST);
12853 format %{ "add $dst, $src1, $src2, uxtw" %}
12854
12855 ins_encode %{
12856 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12857 as_Register($src2$$reg), ext::uxtw);
12858 %}
12859 ins_pipe(ialu_reg_reg);
12860 %}
12861
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12865 %{
12866 match(Set dst (SubI src1 (AndI src2 mask)));
12867 ins_cost(INSN_COST);
12868 format %{ "subw $dst, $src1, $src2, uxtb" %}
12869
12870 ins_encode %{
12871 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12872 as_Register($src2$$reg), ext::uxtb);
12873 %}
12874 ins_pipe(ialu_reg_reg);
12875 %}
12876
12877 // This pattern is automatically generated from aarch64_ad.m4
12878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12879 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12880 %{
12881 match(Set dst (SubI src1 (AndI src2 mask)));
12882 ins_cost(INSN_COST);
12883 format %{ "subw $dst, $src1, $src2, uxth" %}
12884
12885 ins_encode %{
12886 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12887 as_Register($src2$$reg), ext::uxth);
12888 %}
12889 ins_pipe(ialu_reg_reg);
12890 %}
12891
12892 // This pattern is automatically generated from aarch64_ad.m4
12893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12894 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12895 %{
12896 match(Set dst (SubL src1 (AndL src2 mask)));
12897 ins_cost(INSN_COST);
12898 format %{ "sub $dst, $src1, $src2, uxtb" %}
12899
12900 ins_encode %{
12901 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12902 as_Register($src2$$reg), ext::uxtb);
12903 %}
12904 ins_pipe(ialu_reg_reg);
12905 %}
12906
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12910 %{
12911 match(Set dst (SubL src1 (AndL src2 mask)));
12912 ins_cost(INSN_COST);
12913 format %{ "sub $dst, $src1, $src2, uxth" %}
12914
12915 ins_encode %{
12916 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12917 as_Register($src2$$reg), ext::uxth);
12918 %}
12919 ins_pipe(ialu_reg_reg);
12920 %}
12921
12922 // This pattern is automatically generated from aarch64_ad.m4
12923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12924 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12925 %{
12926 match(Set dst (SubL src1 (AndL src2 mask)));
12927 ins_cost(INSN_COST);
12928 format %{ "sub $dst, $src1, $src2, uxtw" %}
12929
12930 ins_encode %{
12931 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12932 as_Register($src2$$reg), ext::uxtw);
12933 %}
12934 ins_pipe(ialu_reg_reg);
12935 %}
12936
12937
12938 // This pattern is automatically generated from aarch64_ad.m4
12939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12940 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12941 %{
12942 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12943 ins_cost(1.9 * INSN_COST);
12944 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12945
12946 ins_encode %{
12947 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12948 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12949 %}
12950 ins_pipe(ialu_reg_reg_shift);
12951 %}
12952
12953 // This pattern is automatically generated from aarch64_ad.m4
12954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12955 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12956 %{
12957 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12958 ins_cost(1.9 * INSN_COST);
12959 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12960
12961 ins_encode %{
12962 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12963 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12964 %}
12965 ins_pipe(ialu_reg_reg_shift);
12966 %}
12967
12968 // This pattern is automatically generated from aarch64_ad.m4
12969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12970 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12971 %{
12972 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12973 ins_cost(1.9 * INSN_COST);
12974 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12975
12976 ins_encode %{
12977 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12978 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12979 %}
12980 ins_pipe(ialu_reg_reg_shift);
12981 %}
12982
12983 // This pattern is automatically generated from aarch64_ad.m4
12984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12985 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12986 %{
12987 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12988 ins_cost(1.9 * INSN_COST);
12989 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12990
12991 ins_encode %{
12992 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12993 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12994 %}
12995 ins_pipe(ialu_reg_reg_shift);
12996 %}
12997
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13001 %{
13002 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13003 ins_cost(1.9 * INSN_COST);
13004 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13005
13006 ins_encode %{
13007 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13008 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13009 %}
13010 ins_pipe(ialu_reg_reg_shift);
13011 %}
13012
13013 // This pattern is automatically generated from aarch64_ad.m4
13014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13015 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13016 %{
13017 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13018 ins_cost(1.9 * INSN_COST);
13019 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13020
13021 ins_encode %{
13022 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13023 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13024 %}
13025 ins_pipe(ialu_reg_reg_shift);
13026 %}
13027
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13031 %{
13032 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13033 ins_cost(1.9 * INSN_COST);
13034 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13035
13036 ins_encode %{
13037 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13038 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13039 %}
13040 ins_pipe(ialu_reg_reg_shift);
13041 %}
13042
13043 // This pattern is automatically generated from aarch64_ad.m4
13044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13045 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13046 %{
13047 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13048 ins_cost(1.9 * INSN_COST);
13049 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13050
13051 ins_encode %{
13052 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13053 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13054 %}
13055 ins_pipe(ialu_reg_reg_shift);
13056 %}
13057
13058 // This pattern is automatically generated from aarch64_ad.m4
13059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13060 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13061 %{
13062 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13063 ins_cost(1.9 * INSN_COST);
13064 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13065
13066 ins_encode %{
13067 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13068 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13069 %}
13070 ins_pipe(ialu_reg_reg_shift);
13071 %}
13072
13073 // This pattern is automatically generated from aarch64_ad.m4
13074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13075 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13076 %{
13077 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13078 ins_cost(1.9 * INSN_COST);
13079 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13080
13081 ins_encode %{
13082 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13083 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13084 %}
13085 ins_pipe(ialu_reg_reg_shift);
13086 %}
13087
13088 // This pattern is automatically generated from aarch64_ad.m4
13089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13090 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13091 %{
13092 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13093 ins_cost(1.9 * INSN_COST);
13094 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13095
13096 ins_encode %{
13097 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13098 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13099 %}
13100 ins_pipe(ialu_reg_reg_shift);
13101 %}
13102
13103 // This pattern is automatically generated from aarch64_ad.m4
13104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13105 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13106 %{
13107 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13108 ins_cost(1.9 * INSN_COST);
13109 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13110
13111 ins_encode %{
13112 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13113 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13114 %}
13115 ins_pipe(ialu_reg_reg_shift);
13116 %}
13117
13118 // This pattern is automatically generated from aarch64_ad.m4
13119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13120 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13121 %{
13122 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13123 ins_cost(1.9 * INSN_COST);
13124 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13125
13126 ins_encode %{
13127 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13128 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13129 %}
13130 ins_pipe(ialu_reg_reg_shift);
13131 %}
13132
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13136 %{
13137 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13138 ins_cost(1.9 * INSN_COST);
13139 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13140
13141 ins_encode %{
13142 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13143 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13144 %}
13145 ins_pipe(ialu_reg_reg_shift);
13146 %}
13147
13148 // This pattern is automatically generated from aarch64_ad.m4
13149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13150 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13151 %{
13152 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13153 ins_cost(1.9 * INSN_COST);
13154 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13155
13156 ins_encode %{
13157 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13158 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13159 %}
13160 ins_pipe(ialu_reg_reg_shift);
13161 %}
13162
13163 // This pattern is automatically generated from aarch64_ad.m4
13164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13165 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13166 %{
13167 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13168 ins_cost(1.9 * INSN_COST);
13169 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13170
13171 ins_encode %{
13172 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13173 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13174 %}
13175 ins_pipe(ialu_reg_reg_shift);
13176 %}
13177
13178 // This pattern is automatically generated from aarch64_ad.m4
13179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13180 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13181 %{
13182 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13183 ins_cost(1.9 * INSN_COST);
13184 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13185
13186 ins_encode %{
13187 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13188 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13189 %}
13190 ins_pipe(ialu_reg_reg_shift);
13191 %}
13192
13193 // This pattern is automatically generated from aarch64_ad.m4
13194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13195 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13196 %{
13197 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13198 ins_cost(1.9 * INSN_COST);
13199 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13200
13201 ins_encode %{
13202 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13203 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13204 %}
13205 ins_pipe(ialu_reg_reg_shift);
13206 %}
13207
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13211 %{
13212 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13213 ins_cost(1.9 * INSN_COST);
13214 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13215
13216 ins_encode %{
13217 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13218 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13219 %}
13220 ins_pipe(ialu_reg_reg_shift);
13221 %}
13222
13223 // This pattern is automatically generated from aarch64_ad.m4
13224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13225 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13226 %{
13227 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13228 ins_cost(1.9 * INSN_COST);
13229 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13230
13231 ins_encode %{
13232 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13233 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13234 %}
13235 ins_pipe(ialu_reg_reg_shift);
13236 %}
13237
13238 // This pattern is automatically generated from aarch64_ad.m4
13239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13240 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13241 %{
13242 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13243 ins_cost(1.9 * INSN_COST);
13244 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13245
13246 ins_encode %{
13247 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13248 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13249 %}
13250 ins_pipe(ialu_reg_reg_shift);
13251 %}
13252
13253 // This pattern is automatically generated from aarch64_ad.m4
13254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13255 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13256 %{
13257 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13258 ins_cost(1.9 * INSN_COST);
13259 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13260
13261 ins_encode %{
13262 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13263 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13264 %}
13265 ins_pipe(ialu_reg_reg_shift);
13266 %}
13267
13268 // This pattern is automatically generated from aarch64_ad.m4
13269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13270 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13271 %{
13272 effect(DEF dst, USE src1, USE src2, USE cr);
13273 ins_cost(INSN_COST * 2);
13274 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13275
13276 ins_encode %{
13277 __ cselw($dst$$Register,
13278 $src1$$Register,
13279 $src2$$Register,
13280 Assembler::LT);
13281 %}
13282 ins_pipe(icond_reg_reg);
13283 %}
13284
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13288 %{
13289 effect(DEF dst, USE src1, USE src2, USE cr);
13290 ins_cost(INSN_COST * 2);
13291 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13292
13293 ins_encode %{
13294 __ cselw($dst$$Register,
13295 $src1$$Register,
13296 $src2$$Register,
13297 Assembler::GT);
13298 %}
13299 ins_pipe(icond_reg_reg);
13300 %}
13301
13302 // This pattern is automatically generated from aarch64_ad.m4
13303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13304 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13305 %{
13306 effect(DEF dst, USE src1, USE cr);
13307 ins_cost(INSN_COST * 2);
13308 format %{ "cselw $dst, $src1, zr lt\t" %}
13309
13310 ins_encode %{
13311 __ cselw($dst$$Register,
13312 $src1$$Register,
13313 zr,
13314 Assembler::LT);
13315 %}
13316 ins_pipe(icond_reg);
13317 %}
13318
13319 // This pattern is automatically generated from aarch64_ad.m4
13320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13321 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13322 %{
13323 effect(DEF dst, USE src1, USE cr);
13324 ins_cost(INSN_COST * 2);
13325 format %{ "cselw $dst, $src1, zr gt\t" %}
13326
13327 ins_encode %{
13328 __ cselw($dst$$Register,
13329 $src1$$Register,
13330 zr,
13331 Assembler::GT);
13332 %}
13333 ins_pipe(icond_reg);
13334 %}
13335
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13339 %{
13340 effect(DEF dst, USE src1, USE cr);
13341 ins_cost(INSN_COST * 2);
13342 format %{ "csincw $dst, $src1, zr le\t" %}
13343
13344 ins_encode %{
13345 __ csincw($dst$$Register,
13346 $src1$$Register,
13347 zr,
13348 Assembler::LE);
13349 %}
13350 ins_pipe(icond_reg);
13351 %}
13352
13353 // This pattern is automatically generated from aarch64_ad.m4
13354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13355 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13356 %{
13357 effect(DEF dst, USE src1, USE cr);
13358 ins_cost(INSN_COST * 2);
13359 format %{ "csincw $dst, $src1, zr gt\t" %}
13360
13361 ins_encode %{
13362 __ csincw($dst$$Register,
13363 $src1$$Register,
13364 zr,
13365 Assembler::GT);
13366 %}
13367 ins_pipe(icond_reg);
13368 %}
13369
13370 // This pattern is automatically generated from aarch64_ad.m4
13371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13372 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13373 %{
13374 effect(DEF dst, USE src1, USE cr);
13375 ins_cost(INSN_COST * 2);
13376 format %{ "csinvw $dst, $src1, zr lt\t" %}
13377
13378 ins_encode %{
13379 __ csinvw($dst$$Register,
13380 $src1$$Register,
13381 zr,
13382 Assembler::LT);
13383 %}
13384 ins_pipe(icond_reg);
13385 %}
13386
13387 // This pattern is automatically generated from aarch64_ad.m4
13388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13389 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13390 %{
13391 effect(DEF dst, USE src1, USE cr);
13392 ins_cost(INSN_COST * 2);
13393 format %{ "csinvw $dst, $src1, zr ge\t" %}
13394
13395 ins_encode %{
13396 __ csinvw($dst$$Register,
13397 $src1$$Register,
13398 zr,
13399 Assembler::GE);
13400 %}
13401 ins_pipe(icond_reg);
13402 %}
13403
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13407 %{
13408 match(Set dst (MinI src imm));
13409 ins_cost(INSN_COST * 3);
13410 expand %{
13411 rFlagsReg cr;
13412 compI_reg_imm0(cr, src);
13413 cmovI_reg_imm0_lt(dst, src, cr);
13414 %}
13415 %}
13416
13417 // This pattern is automatically generated from aarch64_ad.m4
13418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13419 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13420 %{
13421 match(Set dst (MinI imm src));
13422 ins_cost(INSN_COST * 3);
13423 expand %{
13424 rFlagsReg cr;
13425 compI_reg_imm0(cr, src);
13426 cmovI_reg_imm0_lt(dst, src, cr);
13427 %}
13428 %}
13429
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13433 %{
13434 match(Set dst (MinI src imm));
13435 ins_cost(INSN_COST * 3);
13436 expand %{
13437 rFlagsReg cr;
13438 compI_reg_imm0(cr, src);
13439 cmovI_reg_imm1_le(dst, src, cr);
13440 %}
13441 %}
13442
13443 // This pattern is automatically generated from aarch64_ad.m4
13444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13445 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13446 %{
13447 match(Set dst (MinI imm src));
13448 ins_cost(INSN_COST * 3);
13449 expand %{
13450 rFlagsReg cr;
13451 compI_reg_imm0(cr, src);
13452 cmovI_reg_imm1_le(dst, src, cr);
13453 %}
13454 %}
13455
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13459 %{
13460 match(Set dst (MinI src imm));
13461 ins_cost(INSN_COST * 3);
13462 expand %{
13463 rFlagsReg cr;
13464 compI_reg_imm0(cr, src);
13465 cmovI_reg_immM1_lt(dst, src, cr);
13466 %}
13467 %}
13468
13469 // This pattern is automatically generated from aarch64_ad.m4
13470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13471 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13472 %{
13473 match(Set dst (MinI imm src));
13474 ins_cost(INSN_COST * 3);
13475 expand %{
13476 rFlagsReg cr;
13477 compI_reg_imm0(cr, src);
13478 cmovI_reg_immM1_lt(dst, src, cr);
13479 %}
13480 %}
13481
13482 // This pattern is automatically generated from aarch64_ad.m4
13483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13484 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13485 %{
13486 match(Set dst (MaxI src imm));
13487 ins_cost(INSN_COST * 3);
13488 expand %{
13489 rFlagsReg cr;
13490 compI_reg_imm0(cr, src);
13491 cmovI_reg_imm0_gt(dst, src, cr);
13492 %}
13493 %}
13494
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13498 %{
13499 match(Set dst (MaxI imm src));
13500 ins_cost(INSN_COST * 3);
13501 expand %{
13502 rFlagsReg cr;
13503 compI_reg_imm0(cr, src);
13504 cmovI_reg_imm0_gt(dst, src, cr);
13505 %}
13506 %}
13507
13508 // This pattern is automatically generated from aarch64_ad.m4
13509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13510 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13511 %{
13512 match(Set dst (MaxI src imm));
13513 ins_cost(INSN_COST * 3);
13514 expand %{
13515 rFlagsReg cr;
13516 compI_reg_imm0(cr, src);
13517 cmovI_reg_imm1_gt(dst, src, cr);
13518 %}
13519 %}
13520
13521 // This pattern is automatically generated from aarch64_ad.m4
13522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13523 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13524 %{
13525 match(Set dst (MaxI imm src));
13526 ins_cost(INSN_COST * 3);
13527 expand %{
13528 rFlagsReg cr;
13529 compI_reg_imm0(cr, src);
13530 cmovI_reg_imm1_gt(dst, src, cr);
13531 %}
13532 %}
13533
13534 // This pattern is automatically generated from aarch64_ad.m4
13535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13536 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13537 %{
13538 match(Set dst (MaxI src imm));
13539 ins_cost(INSN_COST * 3);
13540 expand %{
13541 rFlagsReg cr;
13542 compI_reg_imm0(cr, src);
13543 cmovI_reg_immM1_ge(dst, src, cr);
13544 %}
13545 %}
13546
13547 // This pattern is automatically generated from aarch64_ad.m4
13548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13549 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13550 %{
13551 match(Set dst (MaxI imm src));
13552 ins_cost(INSN_COST * 3);
13553 expand %{
13554 rFlagsReg cr;
13555 compI_reg_imm0(cr, src);
13556 cmovI_reg_immM1_ge(dst, src, cr);
13557 %}
13558 %}
13559
13560 // This pattern is automatically generated from aarch64_ad.m4
13561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13562 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13563 %{
13564 match(Set dst (ReverseI src));
13565 ins_cost(INSN_COST);
13566 format %{ "rbitw $dst, $src" %}
13567 ins_encode %{
13568 __ rbitw($dst$$Register, $src$$Register);
13569 %}
13570 ins_pipe(ialu_reg);
13571 %}
13572
13573 // This pattern is automatically generated from aarch64_ad.m4
13574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13575 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13576 %{
13577 match(Set dst (ReverseL src));
13578 ins_cost(INSN_COST);
13579 format %{ "rbit $dst, $src" %}
13580 ins_encode %{
13581 __ rbit($dst$$Register, $src$$Register);
13582 %}
13583 ins_pipe(ialu_reg);
13584 %}
13585
13586
13587 // END This section of the file is automatically generated. Do not edit --------------
13588
13589
13590 // ============================================================================
13591 // Floating Point Arithmetic Instructions
13592
13593 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13594 match(Set dst (AddF src1 src2));
13595
13596 ins_cost(INSN_COST * 5);
13597 format %{ "fadds $dst, $src1, $src2" %}
13598
13599 ins_encode %{
13600 __ fadds(as_FloatRegister($dst$$reg),
13601 as_FloatRegister($src1$$reg),
13602 as_FloatRegister($src2$$reg));
13603 %}
13604
13605 ins_pipe(fp_dop_reg_reg_s);
13606 %}
13607
13608 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13609 match(Set dst (AddD src1 src2));
13610
13611 ins_cost(INSN_COST * 5);
13612 format %{ "faddd $dst, $src1, $src2" %}
13613
13614 ins_encode %{
13615 __ faddd(as_FloatRegister($dst$$reg),
13616 as_FloatRegister($src1$$reg),
13617 as_FloatRegister($src2$$reg));
13618 %}
13619
13620 ins_pipe(fp_dop_reg_reg_d);
13621 %}
13622
13623 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13624 match(Set dst (SubF src1 src2));
13625
13626 ins_cost(INSN_COST * 5);
13627 format %{ "fsubs $dst, $src1, $src2" %}
13628
13629 ins_encode %{
13630 __ fsubs(as_FloatRegister($dst$$reg),
13631 as_FloatRegister($src1$$reg),
13632 as_FloatRegister($src2$$reg));
13633 %}
13634
13635 ins_pipe(fp_dop_reg_reg_s);
13636 %}
13637
13638 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13639 match(Set dst (SubD src1 src2));
13640
13641 ins_cost(INSN_COST * 5);
13642 format %{ "fsubd $dst, $src1, $src2" %}
13643
13644 ins_encode %{
13645 __ fsubd(as_FloatRegister($dst$$reg),
13646 as_FloatRegister($src1$$reg),
13647 as_FloatRegister($src2$$reg));
13648 %}
13649
13650 ins_pipe(fp_dop_reg_reg_d);
13651 %}
13652
13653 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13654 match(Set dst (MulF src1 src2));
13655
13656 ins_cost(INSN_COST * 6);
13657 format %{ "fmuls $dst, $src1, $src2" %}
13658
13659 ins_encode %{
13660 __ fmuls(as_FloatRegister($dst$$reg),
13661 as_FloatRegister($src1$$reg),
13662 as_FloatRegister($src2$$reg));
13663 %}
13664
13665 ins_pipe(fp_dop_reg_reg_s);
13666 %}
13667
13668 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13669 match(Set dst (MulD src1 src2));
13670
13671 ins_cost(INSN_COST * 6);
13672 format %{ "fmuld $dst, $src1, $src2" %}
13673
13674 ins_encode %{
13675 __ fmuld(as_FloatRegister($dst$$reg),
13676 as_FloatRegister($src1$$reg),
13677 as_FloatRegister($src2$$reg));
13678 %}
13679
13680 ins_pipe(fp_dop_reg_reg_d);
13681 %}
13682
13683 // src1 * src2 + src3
13684 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13685 match(Set dst (FmaF src3 (Binary src1 src2)));
13686
13687 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13688
13689 ins_encode %{
13690 assert(UseFMA, "Needs FMA instructions support.");
13691 __ fmadds(as_FloatRegister($dst$$reg),
13692 as_FloatRegister($src1$$reg),
13693 as_FloatRegister($src2$$reg),
13694 as_FloatRegister($src3$$reg));
13695 %}
13696
13697 ins_pipe(pipe_class_default);
13698 %}
13699
13700 // src1 * src2 + src3
13701 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13702 match(Set dst (FmaD src3 (Binary src1 src2)));
13703
13704 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13705
13706 ins_encode %{
13707 assert(UseFMA, "Needs FMA instructions support.");
13708 __ fmaddd(as_FloatRegister($dst$$reg),
13709 as_FloatRegister($src1$$reg),
13710 as_FloatRegister($src2$$reg),
13711 as_FloatRegister($src3$$reg));
13712 %}
13713
13714 ins_pipe(pipe_class_default);
13715 %}
13716
13717 // src1 * (-src2) + src3
13718 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13719 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13720 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13721
13722 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13723
13724 ins_encode %{
13725 assert(UseFMA, "Needs FMA instructions support.");
13726 __ fmsubs(as_FloatRegister($dst$$reg),
13727 as_FloatRegister($src1$$reg),
13728 as_FloatRegister($src2$$reg),
13729 as_FloatRegister($src3$$reg));
13730 %}
13731
13732 ins_pipe(pipe_class_default);
13733 %}
13734
13735 // src1 * (-src2) + src3
13736 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13737 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13738 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13739
13740 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13741
13742 ins_encode %{
13743 assert(UseFMA, "Needs FMA instructions support.");
13744 __ fmsubd(as_FloatRegister($dst$$reg),
13745 as_FloatRegister($src1$$reg),
13746 as_FloatRegister($src2$$reg),
13747 as_FloatRegister($src3$$reg));
13748 %}
13749
13750 ins_pipe(pipe_class_default);
13751 %}
13752
13753 // src1 * (-src2) - src3
13754 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13755 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13756 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13757
13758 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13759
13760 ins_encode %{
13761 assert(UseFMA, "Needs FMA instructions support.");
13762 __ fnmadds(as_FloatRegister($dst$$reg),
13763 as_FloatRegister($src1$$reg),
13764 as_FloatRegister($src2$$reg),
13765 as_FloatRegister($src3$$reg));
13766 %}
13767
13768 ins_pipe(pipe_class_default);
13769 %}
13770
13771 // src1 * (-src2) - src3
13772 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13773 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13774 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13775
13776 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13777
13778 ins_encode %{
13779 assert(UseFMA, "Needs FMA instructions support.");
13780 __ fnmaddd(as_FloatRegister($dst$$reg),
13781 as_FloatRegister($src1$$reg),
13782 as_FloatRegister($src2$$reg),
13783 as_FloatRegister($src3$$reg));
13784 %}
13785
13786 ins_pipe(pipe_class_default);
13787 %}
13788
13789 // src1 * src2 - src3
13790 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13791 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13792
13793 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13794
13795 ins_encode %{
13796 assert(UseFMA, "Needs FMA instructions support.");
13797 __ fnmsubs(as_FloatRegister($dst$$reg),
13798 as_FloatRegister($src1$$reg),
13799 as_FloatRegister($src2$$reg),
13800 as_FloatRegister($src3$$reg));
13801 %}
13802
13803 ins_pipe(pipe_class_default);
13804 %}
13805
13806 // src1 * src2 - src3
13807 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13808 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13809
13810 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13811
13812 ins_encode %{
13813 assert(UseFMA, "Needs FMA instructions support.");
13814 // n.b. insn name should be fnmsubd
13815 __ fnmsub(as_FloatRegister($dst$$reg),
13816 as_FloatRegister($src1$$reg),
13817 as_FloatRegister($src2$$reg),
13818 as_FloatRegister($src3$$reg));
13819 %}
13820
13821 ins_pipe(pipe_class_default);
13822 %}
13823
13824
13825 // Math.max(FF)F
13826 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13827 match(Set dst (MaxF src1 src2));
13828
13829 format %{ "fmaxs $dst, $src1, $src2" %}
13830 ins_encode %{
13831 __ fmaxs(as_FloatRegister($dst$$reg),
13832 as_FloatRegister($src1$$reg),
13833 as_FloatRegister($src2$$reg));
13834 %}
13835
13836 ins_pipe(fp_dop_reg_reg_s);
13837 %}
13838
13839 // Math.min(FF)F
13840 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13841 match(Set dst (MinF src1 src2));
13842
13843 format %{ "fmins $dst, $src1, $src2" %}
13844 ins_encode %{
13845 __ fmins(as_FloatRegister($dst$$reg),
13846 as_FloatRegister($src1$$reg),
13847 as_FloatRegister($src2$$reg));
13848 %}
13849
13850 ins_pipe(fp_dop_reg_reg_s);
13851 %}
13852
13853 // Math.max(DD)D
13854 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13855 match(Set dst (MaxD src1 src2));
13856
13857 format %{ "fmaxd $dst, $src1, $src2" %}
13858 ins_encode %{
13859 __ fmaxd(as_FloatRegister($dst$$reg),
13860 as_FloatRegister($src1$$reg),
13861 as_FloatRegister($src2$$reg));
13862 %}
13863
13864 ins_pipe(fp_dop_reg_reg_d);
13865 %}
13866
13867 // Math.min(DD)D
13868 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13869 match(Set dst (MinD src1 src2));
13870
13871 format %{ "fmind $dst, $src1, $src2" %}
13872 ins_encode %{
13873 __ fmind(as_FloatRegister($dst$$reg),
13874 as_FloatRegister($src1$$reg),
13875 as_FloatRegister($src2$$reg));
13876 %}
13877
13878 ins_pipe(fp_dop_reg_reg_d);
13879 %}
13880
13881
13882 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13883 match(Set dst (DivF src1 src2));
13884
13885 ins_cost(INSN_COST * 18);
13886 format %{ "fdivs $dst, $src1, $src2" %}
13887
13888 ins_encode %{
13889 __ fdivs(as_FloatRegister($dst$$reg),
13890 as_FloatRegister($src1$$reg),
13891 as_FloatRegister($src2$$reg));
13892 %}
13893
13894 ins_pipe(fp_div_s);
13895 %}
13896
13897 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13898 match(Set dst (DivD src1 src2));
13899
13900 ins_cost(INSN_COST * 32);
13901 format %{ "fdivd $dst, $src1, $src2" %}
13902
13903 ins_encode %{
13904 __ fdivd(as_FloatRegister($dst$$reg),
13905 as_FloatRegister($src1$$reg),
13906 as_FloatRegister($src2$$reg));
13907 %}
13908
13909 ins_pipe(fp_div_d);
13910 %}
13911
13912 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13913 match(Set dst (NegF src));
13914
13915 ins_cost(INSN_COST * 3);
13916 format %{ "fneg $dst, $src" %}
13917
13918 ins_encode %{
13919 __ fnegs(as_FloatRegister($dst$$reg),
13920 as_FloatRegister($src$$reg));
13921 %}
13922
13923 ins_pipe(fp_uop_s);
13924 %}
13925
13926 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13927 match(Set dst (NegD src));
13928
13929 ins_cost(INSN_COST * 3);
13930 format %{ "fnegd $dst, $src" %}
13931
13932 ins_encode %{
13933 __ fnegd(as_FloatRegister($dst$$reg),
13934 as_FloatRegister($src$$reg));
13935 %}
13936
13937 ins_pipe(fp_uop_d);
13938 %}
13939
13940 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13941 %{
13942 match(Set dst (AbsI src));
13943
13944 effect(KILL cr);
13945 ins_cost(INSN_COST * 2);
13946 format %{ "cmpw $src, zr\n\t"
13947 "cnegw $dst, $src, Assembler::LT\t# int abs"
13948 %}
13949
13950 ins_encode %{
13951 __ cmpw(as_Register($src$$reg), zr);
13952 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13953 %}
13954 ins_pipe(pipe_class_default);
13955 %}
13956
13957 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13958 %{
13959 match(Set dst (AbsL src));
13960
13961 effect(KILL cr);
13962 ins_cost(INSN_COST * 2);
13963 format %{ "cmp $src, zr\n\t"
13964 "cneg $dst, $src, Assembler::LT\t# long abs"
13965 %}
13966
13967 ins_encode %{
13968 __ cmp(as_Register($src$$reg), zr);
13969 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13970 %}
13971 ins_pipe(pipe_class_default);
13972 %}
13973
13974 instruct absF_reg(vRegF dst, vRegF src) %{
13975 match(Set dst (AbsF src));
13976
13977 ins_cost(INSN_COST * 3);
13978 format %{ "fabss $dst, $src" %}
13979 ins_encode %{
13980 __ fabss(as_FloatRegister($dst$$reg),
13981 as_FloatRegister($src$$reg));
13982 %}
13983
13984 ins_pipe(fp_uop_s);
13985 %}
13986
13987 instruct absD_reg(vRegD dst, vRegD src) %{
13988 match(Set dst (AbsD src));
13989
13990 ins_cost(INSN_COST * 3);
13991 format %{ "fabsd $dst, $src" %}
13992 ins_encode %{
13993 __ fabsd(as_FloatRegister($dst$$reg),
13994 as_FloatRegister($src$$reg));
13995 %}
13996
13997 ins_pipe(fp_uop_d);
13998 %}
13999
14000 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14001 match(Set dst (AbsF (SubF src1 src2)));
14002
14003 ins_cost(INSN_COST * 3);
14004 format %{ "fabds $dst, $src1, $src2" %}
14005 ins_encode %{
14006 __ fabds(as_FloatRegister($dst$$reg),
14007 as_FloatRegister($src1$$reg),
14008 as_FloatRegister($src2$$reg));
14009 %}
14010
14011 ins_pipe(fp_uop_s);
14012 %}
14013
14014 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14015 match(Set dst (AbsD (SubD src1 src2)));
14016
14017 ins_cost(INSN_COST * 3);
14018 format %{ "fabdd $dst, $src1, $src2" %}
14019 ins_encode %{
14020 __ fabdd(as_FloatRegister($dst$$reg),
14021 as_FloatRegister($src1$$reg),
14022 as_FloatRegister($src2$$reg));
14023 %}
14024
14025 ins_pipe(fp_uop_d);
14026 %}
14027
14028 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14029 match(Set dst (SqrtD src));
14030
14031 ins_cost(INSN_COST * 50);
14032 format %{ "fsqrtd $dst, $src" %}
14033 ins_encode %{
14034 __ fsqrtd(as_FloatRegister($dst$$reg),
14035 as_FloatRegister($src$$reg));
14036 %}
14037
14038 ins_pipe(fp_div_s);
14039 %}
14040
14041 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14042 match(Set dst (SqrtF src));
14043
14044 ins_cost(INSN_COST * 50);
14045 format %{ "fsqrts $dst, $src" %}
14046 ins_encode %{
14047 __ fsqrts(as_FloatRegister($dst$$reg),
14048 as_FloatRegister($src$$reg));
14049 %}
14050
14051 ins_pipe(fp_div_d);
14052 %}
14053
14054 // Math.rint, floor, ceil
14055 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14056 match(Set dst (RoundDoubleMode src rmode));
14057 format %{ "frint $dst, $src, $rmode" %}
14058 ins_encode %{
14059 switch ($rmode$$constant) {
14060 case RoundDoubleModeNode::rmode_rint:
14061 __ frintnd(as_FloatRegister($dst$$reg),
14062 as_FloatRegister($src$$reg));
14063 break;
14064 case RoundDoubleModeNode::rmode_floor:
14065 __ frintmd(as_FloatRegister($dst$$reg),
14066 as_FloatRegister($src$$reg));
14067 break;
14068 case RoundDoubleModeNode::rmode_ceil:
14069 __ frintpd(as_FloatRegister($dst$$reg),
14070 as_FloatRegister($src$$reg));
14071 break;
14072 }
14073 %}
14074 ins_pipe(fp_uop_d);
14075 %}
14076
14077 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14078 match(Set dst (CopySignD src1 (Binary src2 zero)));
14079 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14080 format %{ "CopySignD $dst $src1 $src2" %}
14081 ins_encode %{
14082 FloatRegister dst = as_FloatRegister($dst$$reg),
14083 src1 = as_FloatRegister($src1$$reg),
14084 src2 = as_FloatRegister($src2$$reg),
14085 zero = as_FloatRegister($zero$$reg);
14086 __ fnegd(dst, zero);
14087 __ bsl(dst, __ T8B, src2, src1);
14088 %}
14089 ins_pipe(fp_uop_d);
14090 %}
14091
14092 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14093 match(Set dst (CopySignF src1 src2));
14094 effect(TEMP_DEF dst, USE src1, USE src2);
14095 format %{ "CopySignF $dst $src1 $src2" %}
14096 ins_encode %{
14097 FloatRegister dst = as_FloatRegister($dst$$reg),
14098 src1 = as_FloatRegister($src1$$reg),
14099 src2 = as_FloatRegister($src2$$reg);
14100 __ movi(dst, __ T2S, 0x80, 24);
14101 __ bsl(dst, __ T8B, src2, src1);
14102 %}
14103 ins_pipe(fp_uop_d);
14104 %}
14105
14106 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14107 match(Set dst (SignumD src (Binary zero one)));
14108 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14109 format %{ "signumD $dst, $src" %}
14110 ins_encode %{
14111 FloatRegister src = as_FloatRegister($src$$reg),
14112 dst = as_FloatRegister($dst$$reg),
14113 zero = as_FloatRegister($zero$$reg),
14114 one = as_FloatRegister($one$$reg);
14115 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14116 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14117 // Bit selection instruction gets bit from "one" for each enabled bit in
14118 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14119 // NaN the whole "src" will be copied because "dst" is zero. For all other
14120 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14121 // from "src", and all other bits are copied from 1.0.
14122 __ bsl(dst, __ T8B, one, src);
14123 %}
14124 ins_pipe(fp_uop_d);
14125 %}
14126
14127 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14128 match(Set dst (SignumF src (Binary zero one)));
14129 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14130 format %{ "signumF $dst, $src" %}
14131 ins_encode %{
14132 FloatRegister src = as_FloatRegister($src$$reg),
14133 dst = as_FloatRegister($dst$$reg),
14134 zero = as_FloatRegister($zero$$reg),
14135 one = as_FloatRegister($one$$reg);
14136 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14137 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14138 // Bit selection instruction gets bit from "one" for each enabled bit in
14139 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14140 // NaN the whole "src" will be copied because "dst" is zero. For all other
14141 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14142 // from "src", and all other bits are copied from 1.0.
14143 __ bsl(dst, __ T8B, one, src);
14144 %}
14145 ins_pipe(fp_uop_d);
14146 %}
14147
14148 instruct onspinwait() %{
14149 match(OnSpinWait);
14150 ins_cost(INSN_COST);
14151
14152 format %{ "onspinwait" %}
14153
14154 ins_encode %{
14155 __ spin_wait();
14156 %}
14157 ins_pipe(pipe_class_empty);
14158 %}
14159
14160 // ============================================================================
14161 // Logical Instructions
14162
14163 // Integer Logical Instructions
14164
14165 // And Instructions
14166
14167
14168 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14169 match(Set dst (AndI src1 src2));
14170
14171 format %{ "andw $dst, $src1, $src2\t# int" %}
14172
14173 ins_cost(INSN_COST);
14174 ins_encode %{
14175 __ andw(as_Register($dst$$reg),
14176 as_Register($src1$$reg),
14177 as_Register($src2$$reg));
14178 %}
14179
14180 ins_pipe(ialu_reg_reg);
14181 %}
14182
14183 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14184 match(Set dst (AndI src1 src2));
14185
14186 format %{ "andsw $dst, $src1, $src2\t# int" %}
14187
14188 ins_cost(INSN_COST);
14189 ins_encode %{
14190 __ andw(as_Register($dst$$reg),
14191 as_Register($src1$$reg),
14192 (uint64_t)($src2$$constant));
14193 %}
14194
14195 ins_pipe(ialu_reg_imm);
14196 %}
14197
14198 // Or Instructions
14199
14200 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14201 match(Set dst (OrI src1 src2));
14202
14203 format %{ "orrw $dst, $src1, $src2\t# int" %}
14204
14205 ins_cost(INSN_COST);
14206 ins_encode %{
14207 __ orrw(as_Register($dst$$reg),
14208 as_Register($src1$$reg),
14209 as_Register($src2$$reg));
14210 %}
14211
14212 ins_pipe(ialu_reg_reg);
14213 %}
14214
14215 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14216 match(Set dst (OrI src1 src2));
14217
14218 format %{ "orrw $dst, $src1, $src2\t# int" %}
14219
14220 ins_cost(INSN_COST);
14221 ins_encode %{
14222 __ orrw(as_Register($dst$$reg),
14223 as_Register($src1$$reg),
14224 (uint64_t)($src2$$constant));
14225 %}
14226
14227 ins_pipe(ialu_reg_imm);
14228 %}
14229
14230 // Xor Instructions
14231
14232 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14233 match(Set dst (XorI src1 src2));
14234
14235 format %{ "eorw $dst, $src1, $src2\t# int" %}
14236
14237 ins_cost(INSN_COST);
14238 ins_encode %{
14239 __ eorw(as_Register($dst$$reg),
14240 as_Register($src1$$reg),
14241 as_Register($src2$$reg));
14242 %}
14243
14244 ins_pipe(ialu_reg_reg);
14245 %}
14246
14247 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14248 match(Set dst (XorI src1 src2));
14249
14250 format %{ "eorw $dst, $src1, $src2\t# int" %}
14251
14252 ins_cost(INSN_COST);
14253 ins_encode %{
14254 __ eorw(as_Register($dst$$reg),
14255 as_Register($src1$$reg),
14256 (uint64_t)($src2$$constant));
14257 %}
14258
14259 ins_pipe(ialu_reg_imm);
14260 %}
14261
14262 // Long Logical Instructions
14263 // TODO
14264
14265 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14266 match(Set dst (AndL src1 src2));
14267
14268 format %{ "and $dst, $src1, $src2\t# int" %}
14269
14270 ins_cost(INSN_COST);
14271 ins_encode %{
14272 __ andr(as_Register($dst$$reg),
14273 as_Register($src1$$reg),
14274 as_Register($src2$$reg));
14275 %}
14276
14277 ins_pipe(ialu_reg_reg);
14278 %}
14279
14280 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14281 match(Set dst (AndL src1 src2));
14282
14283 format %{ "and $dst, $src1, $src2\t# int" %}
14284
14285 ins_cost(INSN_COST);
14286 ins_encode %{
14287 __ andr(as_Register($dst$$reg),
14288 as_Register($src1$$reg),
14289 (uint64_t)($src2$$constant));
14290 %}
14291
14292 ins_pipe(ialu_reg_imm);
14293 %}
14294
14295 // Or Instructions
14296
14297 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14298 match(Set dst (OrL src1 src2));
14299
14300 format %{ "orr $dst, $src1, $src2\t# int" %}
14301
14302 ins_cost(INSN_COST);
14303 ins_encode %{
14304 __ orr(as_Register($dst$$reg),
14305 as_Register($src1$$reg),
14306 as_Register($src2$$reg));
14307 %}
14308
14309 ins_pipe(ialu_reg_reg);
14310 %}
14311
14312 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14313 match(Set dst (OrL src1 src2));
14314
14315 format %{ "orr $dst, $src1, $src2\t# int" %}
14316
14317 ins_cost(INSN_COST);
14318 ins_encode %{
14319 __ orr(as_Register($dst$$reg),
14320 as_Register($src1$$reg),
14321 (uint64_t)($src2$$constant));
14322 %}
14323
14324 ins_pipe(ialu_reg_imm);
14325 %}
14326
14327 // Xor Instructions
14328
14329 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14330 match(Set dst (XorL src1 src2));
14331
14332 format %{ "eor $dst, $src1, $src2\t# int" %}
14333
14334 ins_cost(INSN_COST);
14335 ins_encode %{
14336 __ eor(as_Register($dst$$reg),
14337 as_Register($src1$$reg),
14338 as_Register($src2$$reg));
14339 %}
14340
14341 ins_pipe(ialu_reg_reg);
14342 %}
14343
14344 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14345 match(Set dst (XorL src1 src2));
14346
14347 ins_cost(INSN_COST);
14348 format %{ "eor $dst, $src1, $src2\t# int" %}
14349
14350 ins_encode %{
14351 __ eor(as_Register($dst$$reg),
14352 as_Register($src1$$reg),
14353 (uint64_t)($src2$$constant));
14354 %}
14355
14356 ins_pipe(ialu_reg_imm);
14357 %}
14358
14359 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14360 %{
14361 match(Set dst (ConvI2L src));
14362
14363 ins_cost(INSN_COST);
14364 format %{ "sxtw $dst, $src\t# i2l" %}
14365 ins_encode %{
14366 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14367 %}
14368 ins_pipe(ialu_reg_shift);
14369 %}
14370
14371 // this pattern occurs in bigmath arithmetic
14372 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14373 %{
14374 match(Set dst (AndL (ConvI2L src) mask));
14375
14376 ins_cost(INSN_COST);
14377 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14378 ins_encode %{
14379 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14380 %}
14381
14382 ins_pipe(ialu_reg_shift);
14383 %}
14384
14385 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14386 match(Set dst (ConvL2I src));
14387
14388 ins_cost(INSN_COST);
14389 format %{ "movw $dst, $src \t// l2i" %}
14390
14391 ins_encode %{
14392 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14393 %}
14394
14395 ins_pipe(ialu_reg);
14396 %}
14397
14398 instruct convD2F_reg(vRegF dst, vRegD src) %{
14399 match(Set dst (ConvD2F src));
14400
14401 ins_cost(INSN_COST * 5);
14402 format %{ "fcvtd $dst, $src \t// d2f" %}
14403
14404 ins_encode %{
14405 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14406 %}
14407
14408 ins_pipe(fp_d2f);
14409 %}
14410
14411 instruct convF2D_reg(vRegD dst, vRegF src) %{
14412 match(Set dst (ConvF2D src));
14413
14414 ins_cost(INSN_COST * 5);
14415 format %{ "fcvts $dst, $src \t// f2d" %}
14416
14417 ins_encode %{
14418 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14419 %}
14420
14421 ins_pipe(fp_f2d);
14422 %}
14423
14424 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14425 match(Set dst (ConvF2I src));
14426
14427 ins_cost(INSN_COST * 5);
14428 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14429
14430 ins_encode %{
14431 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14432 %}
14433
14434 ins_pipe(fp_f2i);
14435 %}
14436
14437 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14438 match(Set dst (ConvF2L src));
14439
14440 ins_cost(INSN_COST * 5);
14441 format %{ "fcvtzs $dst, $src \t// f2l" %}
14442
14443 ins_encode %{
14444 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14445 %}
14446
14447 ins_pipe(fp_f2l);
14448 %}
14449
14450 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14451 match(Set dst (ConvF2HF src));
14452 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14453 "smov $dst, $tmp\t# move result from $tmp to $dst"
14454 %}
14455 effect(TEMP tmp);
14456 ins_encode %{
14457 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14458 %}
14459 ins_pipe(pipe_slow);
14460 %}
14461
14462 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14463 match(Set dst (ConvHF2F src));
14464 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14465 "fcvt $dst, $tmp\t# convert half to single precision"
14466 %}
14467 effect(TEMP tmp);
14468 ins_encode %{
14469 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14470 %}
14471 ins_pipe(pipe_slow);
14472 %}
14473
14474 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14475 match(Set dst (ConvI2F src));
14476
14477 ins_cost(INSN_COST * 5);
14478 format %{ "scvtfws $dst, $src \t// i2f" %}
14479
14480 ins_encode %{
14481 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14482 %}
14483
14484 ins_pipe(fp_i2f);
14485 %}
14486
14487 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14488 match(Set dst (ConvL2F src));
14489
14490 ins_cost(INSN_COST * 5);
14491 format %{ "scvtfs $dst, $src \t// l2f" %}
14492
14493 ins_encode %{
14494 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14495 %}
14496
14497 ins_pipe(fp_l2f);
14498 %}
14499
14500 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14501 match(Set dst (ConvD2I src));
14502
14503 ins_cost(INSN_COST * 5);
14504 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14505
14506 ins_encode %{
14507 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14508 %}
14509
14510 ins_pipe(fp_d2i);
14511 %}
14512
14513 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14514 match(Set dst (ConvD2L src));
14515
14516 ins_cost(INSN_COST * 5);
14517 format %{ "fcvtzd $dst, $src \t// d2l" %}
14518
14519 ins_encode %{
14520 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14521 %}
14522
14523 ins_pipe(fp_d2l);
14524 %}
14525
14526 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14527 match(Set dst (ConvI2D src));
14528
14529 ins_cost(INSN_COST * 5);
14530 format %{ "scvtfwd $dst, $src \t// i2d" %}
14531
14532 ins_encode %{
14533 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14534 %}
14535
14536 ins_pipe(fp_i2d);
14537 %}
14538
14539 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14540 match(Set dst (ConvL2D src));
14541
14542 ins_cost(INSN_COST * 5);
14543 format %{ "scvtfd $dst, $src \t// l2d" %}
14544
14545 ins_encode %{
14546 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14547 %}
14548
14549 ins_pipe(fp_l2d);
14550 %}
14551
14552 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14553 %{
14554 match(Set dst (RoundD src));
14555 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14556 format %{ "java_round_double $dst,$src"%}
14557 ins_encode %{
14558 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14559 as_FloatRegister($ftmp$$reg));
14560 %}
14561 ins_pipe(pipe_slow);
14562 %}
14563
14564 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14565 %{
14566 match(Set dst (RoundF src));
14567 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14568 format %{ "java_round_float $dst,$src"%}
14569 ins_encode %{
14570 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14571 as_FloatRegister($ftmp$$reg));
14572 %}
14573 ins_pipe(pipe_slow);
14574 %}
14575
14576 // stack <-> reg and reg <-> reg shuffles with no conversion
14577
14578 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14579
14580 match(Set dst (MoveF2I src));
14581
14582 effect(DEF dst, USE src);
14583
14584 ins_cost(4 * INSN_COST);
14585
14586 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14587
14588 ins_encode %{
14589 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14590 %}
14591
14592 ins_pipe(iload_reg_reg);
14593
14594 %}
14595
14596 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14597
14598 match(Set dst (MoveI2F src));
14599
14600 effect(DEF dst, USE src);
14601
14602 ins_cost(4 * INSN_COST);
14603
14604 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14605
14606 ins_encode %{
14607 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14608 %}
14609
14610 ins_pipe(pipe_class_memory);
14611
14612 %}
14613
14614 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14615
14616 match(Set dst (MoveD2L src));
14617
14618 effect(DEF dst, USE src);
14619
14620 ins_cost(4 * INSN_COST);
14621
14622 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14623
14624 ins_encode %{
14625 __ ldr($dst$$Register, Address(sp, $src$$disp));
14626 %}
14627
14628 ins_pipe(iload_reg_reg);
14629
14630 %}
14631
14632 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14633
14634 match(Set dst (MoveL2D src));
14635
14636 effect(DEF dst, USE src);
14637
14638 ins_cost(4 * INSN_COST);
14639
14640 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14641
14642 ins_encode %{
14643 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14644 %}
14645
14646 ins_pipe(pipe_class_memory);
14647
14648 %}
14649
14650 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14651
14652 match(Set dst (MoveF2I src));
14653
14654 effect(DEF dst, USE src);
14655
14656 ins_cost(INSN_COST);
14657
14658 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14659
14660 ins_encode %{
14661 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14662 %}
14663
14664 ins_pipe(pipe_class_memory);
14665
14666 %}
14667
14668 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14669
14670 match(Set dst (MoveI2F src));
14671
14672 effect(DEF dst, USE src);
14673
14674 ins_cost(INSN_COST);
14675
14676 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14677
14678 ins_encode %{
14679 __ strw($src$$Register, Address(sp, $dst$$disp));
14680 %}
14681
14682 ins_pipe(istore_reg_reg);
14683
14684 %}
14685
14686 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14687
14688 match(Set dst (MoveD2L src));
14689
14690 effect(DEF dst, USE src);
14691
14692 ins_cost(INSN_COST);
14693
14694 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14695
14696 ins_encode %{
14697 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14698 %}
14699
14700 ins_pipe(pipe_class_memory);
14701
14702 %}
14703
14704 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14705
14706 match(Set dst (MoveL2D src));
14707
14708 effect(DEF dst, USE src);
14709
14710 ins_cost(INSN_COST);
14711
14712 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14713
14714 ins_encode %{
14715 __ str($src$$Register, Address(sp, $dst$$disp));
14716 %}
14717
14718 ins_pipe(istore_reg_reg);
14719
14720 %}
14721
14722 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14723
14724 match(Set dst (MoveF2I src));
14725
14726 effect(DEF dst, USE src);
14727
14728 ins_cost(INSN_COST);
14729
14730 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14731
14732 ins_encode %{
14733 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14734 %}
14735
14736 ins_pipe(fp_f2i);
14737
14738 %}
14739
14740 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14741
14742 match(Set dst (MoveI2F src));
14743
14744 effect(DEF dst, USE src);
14745
14746 ins_cost(INSN_COST);
14747
14748 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14749
14750 ins_encode %{
14751 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14752 %}
14753
14754 ins_pipe(fp_i2f);
14755
14756 %}
14757
14758 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14759
14760 match(Set dst (MoveD2L src));
14761
14762 effect(DEF dst, USE src);
14763
14764 ins_cost(INSN_COST);
14765
14766 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14767
14768 ins_encode %{
14769 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14770 %}
14771
14772 ins_pipe(fp_d2l);
14773
14774 %}
14775
14776 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14777
14778 match(Set dst (MoveL2D src));
14779
14780 effect(DEF dst, USE src);
14781
14782 ins_cost(INSN_COST);
14783
14784 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14785
14786 ins_encode %{
14787 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14788 %}
14789
14790 ins_pipe(fp_l2d);
14791
14792 %}
14793
14794 // ============================================================================
14795 // clearing of an array
14796
14797 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14798 %{
14799 match(Set dummy (ClearArray cnt base));
14800 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14801
14802 ins_cost(4 * INSN_COST);
14803 format %{ "ClearArray $cnt, $base" %}
14804
14805 ins_encode %{
14806 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14807 if (tpc == nullptr) {
14808 ciEnv::current()->record_failure("CodeCache is full");
14809 return;
14810 }
14811 %}
14812
14813 ins_pipe(pipe_class_memory);
14814 %}
14815
14816 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14817 %{
14818 predicate((uint64_t)n->in(2)->get_long()
14819 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14820 match(Set dummy (ClearArray cnt base));
14821 effect(TEMP temp, USE_KILL base, KILL cr);
14822
14823 ins_cost(4 * INSN_COST);
14824 format %{ "ClearArray $cnt, $base" %}
14825
14826 ins_encode %{
14827 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14828 if (tpc == nullptr) {
14829 ciEnv::current()->record_failure("CodeCache is full");
14830 return;
14831 }
14832 %}
14833
14834 ins_pipe(pipe_class_memory);
14835 %}
14836
14837 // ============================================================================
14838 // Overflow Math Instructions
14839
14840 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14841 %{
14842 match(Set cr (OverflowAddI op1 op2));
14843
14844 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14845 ins_cost(INSN_COST);
14846 ins_encode %{
14847 __ cmnw($op1$$Register, $op2$$Register);
14848 %}
14849
14850 ins_pipe(icmp_reg_reg);
14851 %}
14852
14853 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14854 %{
14855 match(Set cr (OverflowAddI op1 op2));
14856
14857 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14858 ins_cost(INSN_COST);
14859 ins_encode %{
14860 __ cmnw($op1$$Register, $op2$$constant);
14861 %}
14862
14863 ins_pipe(icmp_reg_imm);
14864 %}
14865
14866 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14867 %{
14868 match(Set cr (OverflowAddL op1 op2));
14869
14870 format %{ "cmn $op1, $op2\t# overflow check long" %}
14871 ins_cost(INSN_COST);
14872 ins_encode %{
14873 __ cmn($op1$$Register, $op2$$Register);
14874 %}
14875
14876 ins_pipe(icmp_reg_reg);
14877 %}
14878
14879 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14880 %{
14881 match(Set cr (OverflowAddL op1 op2));
14882
14883 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14884 ins_cost(INSN_COST);
14885 ins_encode %{
14886 __ adds(zr, $op1$$Register, $op2$$constant);
14887 %}
14888
14889 ins_pipe(icmp_reg_imm);
14890 %}
14891
14892 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14893 %{
14894 match(Set cr (OverflowSubI op1 op2));
14895
14896 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14897 ins_cost(INSN_COST);
14898 ins_encode %{
14899 __ cmpw($op1$$Register, $op2$$Register);
14900 %}
14901
14902 ins_pipe(icmp_reg_reg);
14903 %}
14904
14905 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14906 %{
14907 match(Set cr (OverflowSubI op1 op2));
14908
14909 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14910 ins_cost(INSN_COST);
14911 ins_encode %{
14912 __ cmpw($op1$$Register, $op2$$constant);
14913 %}
14914
14915 ins_pipe(icmp_reg_imm);
14916 %}
14917
14918 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14919 %{
14920 match(Set cr (OverflowSubL op1 op2));
14921
14922 format %{ "cmp $op1, $op2\t# overflow check long" %}
14923 ins_cost(INSN_COST);
14924 ins_encode %{
14925 __ cmp($op1$$Register, $op2$$Register);
14926 %}
14927
14928 ins_pipe(icmp_reg_reg);
14929 %}
14930
14931 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14932 %{
14933 match(Set cr (OverflowSubL op1 op2));
14934
14935 format %{ "cmp $op1, $op2\t# overflow check long" %}
14936 ins_cost(INSN_COST);
14937 ins_encode %{
14938 __ subs(zr, $op1$$Register, $op2$$constant);
14939 %}
14940
14941 ins_pipe(icmp_reg_imm);
14942 %}
14943
14944 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14945 %{
14946 match(Set cr (OverflowSubI zero op1));
14947
14948 format %{ "cmpw zr, $op1\t# overflow check int" %}
14949 ins_cost(INSN_COST);
14950 ins_encode %{
14951 __ cmpw(zr, $op1$$Register);
14952 %}
14953
14954 ins_pipe(icmp_reg_imm);
14955 %}
14956
14957 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14958 %{
14959 match(Set cr (OverflowSubL zero op1));
14960
14961 format %{ "cmp zr, $op1\t# overflow check long" %}
14962 ins_cost(INSN_COST);
14963 ins_encode %{
14964 __ cmp(zr, $op1$$Register);
14965 %}
14966
14967 ins_pipe(icmp_reg_imm);
14968 %}
14969
14970 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14971 %{
14972 match(Set cr (OverflowMulI op1 op2));
14973
14974 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14975 "cmp rscratch1, rscratch1, sxtw\n\t"
14976 "movw rscratch1, #0x80000000\n\t"
14977 "cselw rscratch1, rscratch1, zr, NE\n\t"
14978 "cmpw rscratch1, #1" %}
14979 ins_cost(5 * INSN_COST);
14980 ins_encode %{
14981 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14982 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14983 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14984 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14985 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14986 %}
14987
14988 ins_pipe(pipe_slow);
14989 %}
14990
14991 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14992 %{
14993 match(If cmp (OverflowMulI op1 op2));
14994 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14995 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14996 effect(USE labl, KILL cr);
14997
14998 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14999 "cmp rscratch1, rscratch1, sxtw\n\t"
15000 "b$cmp $labl" %}
15001 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15002 ins_encode %{
15003 Label* L = $labl$$label;
15004 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15005 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15006 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15007 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15008 %}
15009
15010 ins_pipe(pipe_serial);
15011 %}
15012
15013 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15014 %{
15015 match(Set cr (OverflowMulL op1 op2));
15016
15017 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15018 "smulh rscratch2, $op1, $op2\n\t"
15019 "cmp rscratch2, rscratch1, ASR #63\n\t"
15020 "movw rscratch1, #0x80000000\n\t"
15021 "cselw rscratch1, rscratch1, zr, NE\n\t"
15022 "cmpw rscratch1, #1" %}
15023 ins_cost(6 * INSN_COST);
15024 ins_encode %{
15025 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15026 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15027 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15028 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15029 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15030 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15031 %}
15032
15033 ins_pipe(pipe_slow);
15034 %}
15035
15036 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15037 %{
15038 match(If cmp (OverflowMulL op1 op2));
15039 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15040 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15041 effect(USE labl, KILL cr);
15042
15043 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15044 "smulh rscratch2, $op1, $op2\n\t"
15045 "cmp rscratch2, rscratch1, ASR #63\n\t"
15046 "b$cmp $labl" %}
15047 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15048 ins_encode %{
15049 Label* L = $labl$$label;
15050 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15051 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15052 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15053 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15054 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15055 %}
15056
15057 ins_pipe(pipe_serial);
15058 %}
15059
15060 // ============================================================================
15061 // Compare Instructions
15062
15063 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15064 %{
15065 match(Set cr (CmpI op1 op2));
15066
15067 effect(DEF cr, USE op1, USE op2);
15068
15069 ins_cost(INSN_COST);
15070 format %{ "cmpw $op1, $op2" %}
15071
15072 ins_encode(aarch64_enc_cmpw(op1, op2));
15073
15074 ins_pipe(icmp_reg_reg);
15075 %}
15076
15077 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15078 %{
15079 match(Set cr (CmpI op1 zero));
15080
15081 effect(DEF cr, USE op1);
15082
15083 ins_cost(INSN_COST);
15084 format %{ "cmpw $op1, 0" %}
15085
15086 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15087
15088 ins_pipe(icmp_reg_imm);
15089 %}
15090
15091 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15092 %{
15093 match(Set cr (CmpI op1 op2));
15094
15095 effect(DEF cr, USE op1);
15096
15097 ins_cost(INSN_COST);
15098 format %{ "cmpw $op1, $op2" %}
15099
15100 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15101
15102 ins_pipe(icmp_reg_imm);
15103 %}
15104
15105 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15106 %{
15107 match(Set cr (CmpI op1 op2));
15108
15109 effect(DEF cr, USE op1);
15110
15111 ins_cost(INSN_COST * 2);
15112 format %{ "cmpw $op1, $op2" %}
15113
15114 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15115
15116 ins_pipe(icmp_reg_imm);
15117 %}
15118
15119 // Unsigned compare Instructions; really, same as signed compare
15120 // except it should only be used to feed an If or a CMovI which takes a
15121 // cmpOpU.
15122
15123 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15124 %{
15125 match(Set cr (CmpU op1 op2));
15126
15127 effect(DEF cr, USE op1, USE op2);
15128
15129 ins_cost(INSN_COST);
15130 format %{ "cmpw $op1, $op2\t# unsigned" %}
15131
15132 ins_encode(aarch64_enc_cmpw(op1, op2));
15133
15134 ins_pipe(icmp_reg_reg);
15135 %}
15136
15137 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15138 %{
15139 match(Set cr (CmpU op1 zero));
15140
15141 effect(DEF cr, USE op1);
15142
15143 ins_cost(INSN_COST);
15144 format %{ "cmpw $op1, #0\t# unsigned" %}
15145
15146 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15147
15148 ins_pipe(icmp_reg_imm);
15149 %}
15150
15151 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15152 %{
15153 match(Set cr (CmpU op1 op2));
15154
15155 effect(DEF cr, USE op1);
15156
15157 ins_cost(INSN_COST);
15158 format %{ "cmpw $op1, $op2\t# unsigned" %}
15159
15160 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15161
15162 ins_pipe(icmp_reg_imm);
15163 %}
15164
15165 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15166 %{
15167 match(Set cr (CmpU op1 op2));
15168
15169 effect(DEF cr, USE op1);
15170
15171 ins_cost(INSN_COST * 2);
15172 format %{ "cmpw $op1, $op2\t# unsigned" %}
15173
15174 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15175
15176 ins_pipe(icmp_reg_imm);
15177 %}
15178
15179 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15180 %{
15181 match(Set cr (CmpL op1 op2));
15182
15183 effect(DEF cr, USE op1, USE op2);
15184
15185 ins_cost(INSN_COST);
15186 format %{ "cmp $op1, $op2" %}
15187
15188 ins_encode(aarch64_enc_cmp(op1, op2));
15189
15190 ins_pipe(icmp_reg_reg);
15191 %}
15192
15193 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15194 %{
15195 match(Set cr (CmpL op1 zero));
15196
15197 effect(DEF cr, USE op1);
15198
15199 ins_cost(INSN_COST);
15200 format %{ "tst $op1" %}
15201
15202 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15203
15204 ins_pipe(icmp_reg_imm);
15205 %}
15206
15207 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15208 %{
15209 match(Set cr (CmpL op1 op2));
15210
15211 effect(DEF cr, USE op1);
15212
15213 ins_cost(INSN_COST);
15214 format %{ "cmp $op1, $op2" %}
15215
15216 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15217
15218 ins_pipe(icmp_reg_imm);
15219 %}
15220
15221 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15222 %{
15223 match(Set cr (CmpL op1 op2));
15224
15225 effect(DEF cr, USE op1);
15226
15227 ins_cost(INSN_COST * 2);
15228 format %{ "cmp $op1, $op2" %}
15229
15230 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15231
15232 ins_pipe(icmp_reg_imm);
15233 %}
15234
15235 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15236 %{
15237 match(Set cr (CmpUL op1 op2));
15238
15239 effect(DEF cr, USE op1, USE op2);
15240
15241 ins_cost(INSN_COST);
15242 format %{ "cmp $op1, $op2" %}
15243
15244 ins_encode(aarch64_enc_cmp(op1, op2));
15245
15246 ins_pipe(icmp_reg_reg);
15247 %}
15248
15249 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15250 %{
15251 match(Set cr (CmpUL op1 zero));
15252
15253 effect(DEF cr, USE op1);
15254
15255 ins_cost(INSN_COST);
15256 format %{ "tst $op1" %}
15257
15258 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15259
15260 ins_pipe(icmp_reg_imm);
15261 %}
15262
15263 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15264 %{
15265 match(Set cr (CmpUL op1 op2));
15266
15267 effect(DEF cr, USE op1);
15268
15269 ins_cost(INSN_COST);
15270 format %{ "cmp $op1, $op2" %}
15271
15272 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15273
15274 ins_pipe(icmp_reg_imm);
15275 %}
15276
15277 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15278 %{
15279 match(Set cr (CmpUL op1 op2));
15280
15281 effect(DEF cr, USE op1);
15282
15283 ins_cost(INSN_COST * 2);
15284 format %{ "cmp $op1, $op2" %}
15285
15286 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15287
15288 ins_pipe(icmp_reg_imm);
15289 %}
15290
15291 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15292 %{
15293 match(Set cr (CmpP op1 op2));
15294
15295 effect(DEF cr, USE op1, USE op2);
15296
15297 ins_cost(INSN_COST);
15298 format %{ "cmp $op1, $op2\t // ptr" %}
15299
15300 ins_encode(aarch64_enc_cmpp(op1, op2));
15301
15302 ins_pipe(icmp_reg_reg);
15303 %}
15304
15305 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15306 %{
15307 match(Set cr (CmpN op1 op2));
15308
15309 effect(DEF cr, USE op1, USE op2);
15310
15311 ins_cost(INSN_COST);
15312 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15313
15314 ins_encode(aarch64_enc_cmpn(op1, op2));
15315
15316 ins_pipe(icmp_reg_reg);
15317 %}
15318
15319 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15320 %{
15321 match(Set cr (CmpP op1 zero));
15322
15323 effect(DEF cr, USE op1, USE zero);
15324
15325 ins_cost(INSN_COST);
15326 format %{ "cmp $op1, 0\t // ptr" %}
15327
15328 ins_encode(aarch64_enc_testp(op1));
15329
15330 ins_pipe(icmp_reg_imm);
15331 %}
15332
15333 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15334 %{
15335 match(Set cr (CmpN op1 zero));
15336
15337 effect(DEF cr, USE op1, USE zero);
15338
15339 ins_cost(INSN_COST);
15340 format %{ "cmp $op1, 0\t // compressed ptr" %}
15341
15342 ins_encode(aarch64_enc_testn(op1));
15343
15344 ins_pipe(icmp_reg_imm);
15345 %}
15346
15347 // FP comparisons
15348 //
15349 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15350 // using normal cmpOp. See declaration of rFlagsReg for details.
15351
15352 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15353 %{
15354 match(Set cr (CmpF src1 src2));
15355
15356 ins_cost(3 * INSN_COST);
15357 format %{ "fcmps $src1, $src2" %}
15358
15359 ins_encode %{
15360 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15361 %}
15362
15363 ins_pipe(pipe_class_compare);
15364 %}
15365
15366 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15367 %{
15368 match(Set cr (CmpF src1 src2));
15369
15370 ins_cost(3 * INSN_COST);
15371 format %{ "fcmps $src1, 0.0" %}
15372
15373 ins_encode %{
15374 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15375 %}
15376
15377 ins_pipe(pipe_class_compare);
15378 %}
15379 // FROM HERE
15380
15381 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15382 %{
15383 match(Set cr (CmpD src1 src2));
15384
15385 ins_cost(3 * INSN_COST);
15386 format %{ "fcmpd $src1, $src2" %}
15387
15388 ins_encode %{
15389 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15390 %}
15391
15392 ins_pipe(pipe_class_compare);
15393 %}
15394
15395 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15396 %{
15397 match(Set cr (CmpD src1 src2));
15398
15399 ins_cost(3 * INSN_COST);
15400 format %{ "fcmpd $src1, 0.0" %}
15401
15402 ins_encode %{
15403 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15404 %}
15405
15406 ins_pipe(pipe_class_compare);
15407 %}
15408
15409 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15410 %{
15411 match(Set dst (CmpF3 src1 src2));
15412 effect(KILL cr);
15413
15414 ins_cost(5 * INSN_COST);
15415 format %{ "fcmps $src1, $src2\n\t"
15416 "csinvw($dst, zr, zr, eq\n\t"
15417 "csnegw($dst, $dst, $dst, lt)"
15418 %}
15419
15420 ins_encode %{
15421 Label done;
15422 FloatRegister s1 = as_FloatRegister($src1$$reg);
15423 FloatRegister s2 = as_FloatRegister($src2$$reg);
15424 Register d = as_Register($dst$$reg);
15425 __ fcmps(s1, s2);
15426 // installs 0 if EQ else -1
15427 __ csinvw(d, zr, zr, Assembler::EQ);
15428 // keeps -1 if less or unordered else installs 1
15429 __ csnegw(d, d, d, Assembler::LT);
15430 __ bind(done);
15431 %}
15432
15433 ins_pipe(pipe_class_default);
15434
15435 %}
15436
15437 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15438 %{
15439 match(Set dst (CmpD3 src1 src2));
15440 effect(KILL cr);
15441
15442 ins_cost(5 * INSN_COST);
15443 format %{ "fcmpd $src1, $src2\n\t"
15444 "csinvw($dst, zr, zr, eq\n\t"
15445 "csnegw($dst, $dst, $dst, lt)"
15446 %}
15447
15448 ins_encode %{
15449 Label done;
15450 FloatRegister s1 = as_FloatRegister($src1$$reg);
15451 FloatRegister s2 = as_FloatRegister($src2$$reg);
15452 Register d = as_Register($dst$$reg);
15453 __ fcmpd(s1, s2);
15454 // installs 0 if EQ else -1
15455 __ csinvw(d, zr, zr, Assembler::EQ);
15456 // keeps -1 if less or unordered else installs 1
15457 __ csnegw(d, d, d, Assembler::LT);
15458 __ bind(done);
15459 %}
15460 ins_pipe(pipe_class_default);
15461
15462 %}
15463
15464 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15465 %{
15466 match(Set dst (CmpF3 src1 zero));
15467 effect(KILL cr);
15468
15469 ins_cost(5 * INSN_COST);
15470 format %{ "fcmps $src1, 0.0\n\t"
15471 "csinvw($dst, zr, zr, eq\n\t"
15472 "csnegw($dst, $dst, $dst, lt)"
15473 %}
15474
15475 ins_encode %{
15476 Label done;
15477 FloatRegister s1 = as_FloatRegister($src1$$reg);
15478 Register d = as_Register($dst$$reg);
15479 __ fcmps(s1, 0.0);
15480 // installs 0 if EQ else -1
15481 __ csinvw(d, zr, zr, Assembler::EQ);
15482 // keeps -1 if less or unordered else installs 1
15483 __ csnegw(d, d, d, Assembler::LT);
15484 __ bind(done);
15485 %}
15486
15487 ins_pipe(pipe_class_default);
15488
15489 %}
15490
15491 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15492 %{
15493 match(Set dst (CmpD3 src1 zero));
15494 effect(KILL cr);
15495
15496 ins_cost(5 * INSN_COST);
15497 format %{ "fcmpd $src1, 0.0\n\t"
15498 "csinvw($dst, zr, zr, eq\n\t"
15499 "csnegw($dst, $dst, $dst, lt)"
15500 %}
15501
15502 ins_encode %{
15503 Label done;
15504 FloatRegister s1 = as_FloatRegister($src1$$reg);
15505 Register d = as_Register($dst$$reg);
15506 __ fcmpd(s1, 0.0);
15507 // installs 0 if EQ else -1
15508 __ csinvw(d, zr, zr, Assembler::EQ);
15509 // keeps -1 if less or unordered else installs 1
15510 __ csnegw(d, d, d, Assembler::LT);
15511 __ bind(done);
15512 %}
15513 ins_pipe(pipe_class_default);
15514
15515 %}
15516
15517 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15518 %{
15519 match(Set dst (CmpLTMask p q));
15520 effect(KILL cr);
15521
15522 ins_cost(3 * INSN_COST);
15523
15524 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15525 "csetw $dst, lt\n\t"
15526 "subw $dst, zr, $dst"
15527 %}
15528
15529 ins_encode %{
15530 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15531 __ csetw(as_Register($dst$$reg), Assembler::LT);
15532 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15533 %}
15534
15535 ins_pipe(ialu_reg_reg);
15536 %}
15537
15538 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15539 %{
15540 match(Set dst (CmpLTMask src zero));
15541 effect(KILL cr);
15542
15543 ins_cost(INSN_COST);
15544
15545 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15546
15547 ins_encode %{
15548 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15549 %}
15550
15551 ins_pipe(ialu_reg_shift);
15552 %}
15553
15554 // ============================================================================
15555 // Max and Min
15556
15557 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15558
15559 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15560 %{
15561 effect(DEF cr, USE src);
15562 ins_cost(INSN_COST);
15563 format %{ "cmpw $src, 0" %}
15564
15565 ins_encode %{
15566 __ cmpw($src$$Register, 0);
15567 %}
15568 ins_pipe(icmp_reg_imm);
15569 %}
15570
15571 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15572 %{
15573 match(Set dst (MinI src1 src2));
15574 ins_cost(INSN_COST * 3);
15575
15576 expand %{
15577 rFlagsReg cr;
15578 compI_reg_reg(cr, src1, src2);
15579 cmovI_reg_reg_lt(dst, src1, src2, cr);
15580 %}
15581 %}
15582
15583 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15584 %{
15585 match(Set dst (MaxI src1 src2));
15586 ins_cost(INSN_COST * 3);
15587
15588 expand %{
15589 rFlagsReg cr;
15590 compI_reg_reg(cr, src1, src2);
15591 cmovI_reg_reg_gt(dst, src1, src2, cr);
15592 %}
15593 %}
15594
15595
15596 // ============================================================================
15597 // Branch Instructions
15598
15599 // Direct Branch.
15600 instruct branch(label lbl)
15601 %{
15602 match(Goto);
15603
15604 effect(USE lbl);
15605
15606 ins_cost(BRANCH_COST);
15607 format %{ "b $lbl" %}
15608
15609 ins_encode(aarch64_enc_b(lbl));
15610
15611 ins_pipe(pipe_branch);
15612 %}
15613
15614 // Conditional Near Branch
15615 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15616 %{
15617 // Same match rule as `branchConFar'.
15618 match(If cmp cr);
15619
15620 effect(USE lbl);
15621
15622 ins_cost(BRANCH_COST);
15623 // If set to 1 this indicates that the current instruction is a
15624 // short variant of a long branch. This avoids using this
15625 // instruction in first-pass matching. It will then only be used in
15626 // the `Shorten_branches' pass.
15627 // ins_short_branch(1);
15628 format %{ "b$cmp $lbl" %}
15629
15630 ins_encode(aarch64_enc_br_con(cmp, lbl));
15631
15632 ins_pipe(pipe_branch_cond);
15633 %}
15634
15635 // Conditional Near Branch Unsigned
15636 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15637 %{
15638 // Same match rule as `branchConFar'.
15639 match(If cmp cr);
15640
15641 effect(USE lbl);
15642
15643 ins_cost(BRANCH_COST);
15644 // If set to 1 this indicates that the current instruction is a
15645 // short variant of a long branch. This avoids using this
15646 // instruction in first-pass matching. It will then only be used in
15647 // the `Shorten_branches' pass.
15648 // ins_short_branch(1);
15649 format %{ "b$cmp $lbl\t# unsigned" %}
15650
15651 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15652
15653 ins_pipe(pipe_branch_cond);
15654 %}
15655
15656 // Make use of CBZ and CBNZ. These instructions, as well as being
15657 // shorter than (cmp; branch), have the additional benefit of not
15658 // killing the flags.
15659
15660 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15661 match(If cmp (CmpI op1 op2));
15662 effect(USE labl);
15663
15664 ins_cost(BRANCH_COST);
15665 format %{ "cbw$cmp $op1, $labl" %}
15666 ins_encode %{
15667 Label* L = $labl$$label;
15668 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15669 if (cond == Assembler::EQ)
15670 __ cbzw($op1$$Register, *L);
15671 else
15672 __ cbnzw($op1$$Register, *L);
15673 %}
15674 ins_pipe(pipe_cmp_branch);
15675 %}
15676
15677 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15678 match(If cmp (CmpL op1 op2));
15679 effect(USE labl);
15680
15681 ins_cost(BRANCH_COST);
15682 format %{ "cb$cmp $op1, $labl" %}
15683 ins_encode %{
15684 Label* L = $labl$$label;
15685 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15686 if (cond == Assembler::EQ)
15687 __ cbz($op1$$Register, *L);
15688 else
15689 __ cbnz($op1$$Register, *L);
15690 %}
15691 ins_pipe(pipe_cmp_branch);
15692 %}
15693
15694 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15695 match(If cmp (CmpP op1 op2));
15696 effect(USE labl);
15697
15698 ins_cost(BRANCH_COST);
15699 format %{ "cb$cmp $op1, $labl" %}
15700 ins_encode %{
15701 Label* L = $labl$$label;
15702 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15703 if (cond == Assembler::EQ)
15704 __ cbz($op1$$Register, *L);
15705 else
15706 __ cbnz($op1$$Register, *L);
15707 %}
15708 ins_pipe(pipe_cmp_branch);
15709 %}
15710
15711 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15712 match(If cmp (CmpN op1 op2));
15713 effect(USE labl);
15714
15715 ins_cost(BRANCH_COST);
15716 format %{ "cbw$cmp $op1, $labl" %}
15717 ins_encode %{
15718 Label* L = $labl$$label;
15719 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15720 if (cond == Assembler::EQ)
15721 __ cbzw($op1$$Register, *L);
15722 else
15723 __ cbnzw($op1$$Register, *L);
15724 %}
15725 ins_pipe(pipe_cmp_branch);
15726 %}
15727
15728 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15729 match(If cmp (CmpP (DecodeN oop) zero));
15730 effect(USE labl);
15731
15732 ins_cost(BRANCH_COST);
15733 format %{ "cb$cmp $oop, $labl" %}
15734 ins_encode %{
15735 Label* L = $labl$$label;
15736 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15737 if (cond == Assembler::EQ)
15738 __ cbzw($oop$$Register, *L);
15739 else
15740 __ cbnzw($oop$$Register, *L);
15741 %}
15742 ins_pipe(pipe_cmp_branch);
15743 %}
15744
15745 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15746 match(If cmp (CmpU op1 op2));
15747 effect(USE labl);
15748
15749 ins_cost(BRANCH_COST);
15750 format %{ "cbw$cmp $op1, $labl" %}
15751 ins_encode %{
15752 Label* L = $labl$$label;
15753 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15754 if (cond == Assembler::EQ || cond == Assembler::LS) {
15755 __ cbzw($op1$$Register, *L);
15756 } else {
15757 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15758 __ cbnzw($op1$$Register, *L);
15759 }
15760 %}
15761 ins_pipe(pipe_cmp_branch);
15762 %}
15763
15764 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15765 match(If cmp (CmpUL op1 op2));
15766 effect(USE labl);
15767
15768 ins_cost(BRANCH_COST);
15769 format %{ "cb$cmp $op1, $labl" %}
15770 ins_encode %{
15771 Label* L = $labl$$label;
15772 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15773 if (cond == Assembler::EQ || cond == Assembler::LS) {
15774 __ cbz($op1$$Register, *L);
15775 } else {
15776 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15777 __ cbnz($op1$$Register, *L);
15778 }
15779 %}
15780 ins_pipe(pipe_cmp_branch);
15781 %}
15782
15783 // Test bit and Branch
15784
15785 // Patterns for short (< 32KiB) variants
15786 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15787 match(If cmp (CmpL op1 op2));
15788 effect(USE labl);
15789
15790 ins_cost(BRANCH_COST);
15791 format %{ "cb$cmp $op1, $labl # long" %}
15792 ins_encode %{
15793 Label* L = $labl$$label;
15794 Assembler::Condition cond =
15795 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15796 __ tbr(cond, $op1$$Register, 63, *L);
15797 %}
15798 ins_pipe(pipe_cmp_branch);
15799 ins_short_branch(1);
15800 %}
15801
15802 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15803 match(If cmp (CmpI op1 op2));
15804 effect(USE labl);
15805
15806 ins_cost(BRANCH_COST);
15807 format %{ "cb$cmp $op1, $labl # int" %}
15808 ins_encode %{
15809 Label* L = $labl$$label;
15810 Assembler::Condition cond =
15811 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15812 __ tbr(cond, $op1$$Register, 31, *L);
15813 %}
15814 ins_pipe(pipe_cmp_branch);
15815 ins_short_branch(1);
15816 %}
15817
15818 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15819 match(If cmp (CmpL (AndL op1 op2) op3));
15820 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15821 effect(USE labl);
15822
15823 ins_cost(BRANCH_COST);
15824 format %{ "tb$cmp $op1, $op2, $labl" %}
15825 ins_encode %{
15826 Label* L = $labl$$label;
15827 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15828 int bit = exact_log2_long($op2$$constant);
15829 __ tbr(cond, $op1$$Register, bit, *L);
15830 %}
15831 ins_pipe(pipe_cmp_branch);
15832 ins_short_branch(1);
15833 %}
15834
15835 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15836 match(If cmp (CmpI (AndI op1 op2) op3));
15837 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15838 effect(USE labl);
15839
15840 ins_cost(BRANCH_COST);
15841 format %{ "tb$cmp $op1, $op2, $labl" %}
15842 ins_encode %{
15843 Label* L = $labl$$label;
15844 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15845 int bit = exact_log2((juint)$op2$$constant);
15846 __ tbr(cond, $op1$$Register, bit, *L);
15847 %}
15848 ins_pipe(pipe_cmp_branch);
15849 ins_short_branch(1);
15850 %}
15851
15852 // And far variants
15853 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15854 match(If cmp (CmpL op1 op2));
15855 effect(USE labl);
15856
15857 ins_cost(BRANCH_COST);
15858 format %{ "cb$cmp $op1, $labl # long" %}
15859 ins_encode %{
15860 Label* L = $labl$$label;
15861 Assembler::Condition cond =
15862 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15863 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15864 %}
15865 ins_pipe(pipe_cmp_branch);
15866 %}
15867
15868 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15869 match(If cmp (CmpI op1 op2));
15870 effect(USE labl);
15871
15872 ins_cost(BRANCH_COST);
15873 format %{ "cb$cmp $op1, $labl # int" %}
15874 ins_encode %{
15875 Label* L = $labl$$label;
15876 Assembler::Condition cond =
15877 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15878 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15879 %}
15880 ins_pipe(pipe_cmp_branch);
15881 %}
15882
15883 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15884 match(If cmp (CmpL (AndL op1 op2) op3));
15885 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15886 effect(USE labl);
15887
15888 ins_cost(BRANCH_COST);
15889 format %{ "tb$cmp $op1, $op2, $labl" %}
15890 ins_encode %{
15891 Label* L = $labl$$label;
15892 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15893 int bit = exact_log2_long($op2$$constant);
15894 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15895 %}
15896 ins_pipe(pipe_cmp_branch);
15897 %}
15898
15899 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15900 match(If cmp (CmpI (AndI op1 op2) op3));
15901 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15902 effect(USE labl);
15903
15904 ins_cost(BRANCH_COST);
15905 format %{ "tb$cmp $op1, $op2, $labl" %}
15906 ins_encode %{
15907 Label* L = $labl$$label;
15908 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15909 int bit = exact_log2((juint)$op2$$constant);
15910 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15911 %}
15912 ins_pipe(pipe_cmp_branch);
15913 %}
15914
15915 // Test bits
15916
15917 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15918 match(Set cr (CmpL (AndL op1 op2) op3));
15919 predicate(Assembler::operand_valid_for_logical_immediate
15920 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15921
15922 ins_cost(INSN_COST);
15923 format %{ "tst $op1, $op2 # long" %}
15924 ins_encode %{
15925 __ tst($op1$$Register, $op2$$constant);
15926 %}
15927 ins_pipe(ialu_reg_reg);
15928 %}
15929
15930 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15931 match(Set cr (CmpI (AndI op1 op2) op3));
15932 predicate(Assembler::operand_valid_for_logical_immediate
15933 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15934
15935 ins_cost(INSN_COST);
15936 format %{ "tst $op1, $op2 # int" %}
15937 ins_encode %{
15938 __ tstw($op1$$Register, $op2$$constant);
15939 %}
15940 ins_pipe(ialu_reg_reg);
15941 %}
15942
15943 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15944 match(Set cr (CmpL (AndL op1 op2) op3));
15945
15946 ins_cost(INSN_COST);
15947 format %{ "tst $op1, $op2 # long" %}
15948 ins_encode %{
15949 __ tst($op1$$Register, $op2$$Register);
15950 %}
15951 ins_pipe(ialu_reg_reg);
15952 %}
15953
15954 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15955 match(Set cr (CmpI (AndI op1 op2) op3));
15956
15957 ins_cost(INSN_COST);
15958 format %{ "tstw $op1, $op2 # int" %}
15959 ins_encode %{
15960 __ tstw($op1$$Register, $op2$$Register);
15961 %}
15962 ins_pipe(ialu_reg_reg);
15963 %}
15964
15965
15966 // Conditional Far Branch
15967 // Conditional Far Branch Unsigned
15968 // TODO: fixme
15969
15970 // counted loop end branch near
15971 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15972 %{
15973 match(CountedLoopEnd cmp cr);
15974
15975 effect(USE lbl);
15976
15977 ins_cost(BRANCH_COST);
15978 // short variant.
15979 // ins_short_branch(1);
15980 format %{ "b$cmp $lbl \t// counted loop end" %}
15981
15982 ins_encode(aarch64_enc_br_con(cmp, lbl));
15983
15984 ins_pipe(pipe_branch);
15985 %}
15986
15987 // counted loop end branch far
15988 // TODO: fixme
15989
15990 // ============================================================================
15991 // inlined locking and unlocking
15992
15993 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15994 %{
15995 predicate(LockingMode != LM_LIGHTWEIGHT);
15996 match(Set cr (FastLock object box));
15997 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15998
15999 ins_cost(5 * INSN_COST);
16000 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16001
16002 ins_encode %{
16003 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16004 %}
16005
16006 ins_pipe(pipe_serial);
16007 %}
16008
16009 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16010 %{
16011 predicate(LockingMode != LM_LIGHTWEIGHT);
16012 match(Set cr (FastUnlock object box));
16013 effect(TEMP tmp, TEMP tmp2);
16014
16015 ins_cost(5 * INSN_COST);
16016 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16017
16018 ins_encode %{
16019 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16020 %}
16021
16022 ins_pipe(pipe_serial);
16023 %}
16024
16025 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16026 %{
16027 predicate(LockingMode == LM_LIGHTWEIGHT);
16028 match(Set cr (FastLock object box));
16029 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16030
16031 ins_cost(5 * INSN_COST);
16032 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16033
16034 ins_encode %{
16035 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16036 %}
16037
16038 ins_pipe(pipe_serial);
16039 %}
16040
16041 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16042 %{
16043 predicate(LockingMode == LM_LIGHTWEIGHT);
16044 match(Set cr (FastUnlock object box));
16045 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16046
16047 ins_cost(5 * INSN_COST);
16048 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16049
16050 ins_encode %{
16051 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16052 %}
16053
16054 ins_pipe(pipe_serial);
16055 %}
16056
16057 // ============================================================================
16058 // Safepoint Instructions
16059
16060 // TODO
16061 // provide a near and far version of this code
16062
16063 instruct safePoint(rFlagsReg cr, iRegP poll)
16064 %{
16065 match(SafePoint poll);
16066 effect(KILL cr);
16067
16068 format %{
16069 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16070 %}
16071 ins_encode %{
16072 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16073 %}
16074 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16075 %}
16076
16077
16078 // ============================================================================
16079 // Procedure Call/Return Instructions
16080
16081 // Call Java Static Instruction
16082
16083 instruct CallStaticJavaDirect(method meth)
16084 %{
16085 match(CallStaticJava);
16086
16087 effect(USE meth);
16088
16089 ins_cost(CALL_COST);
16090
16091 format %{ "call,static $meth \t// ==> " %}
16092
16093 ins_encode(aarch64_enc_java_static_call(meth),
16094 aarch64_enc_call_epilog);
16095
16096 ins_pipe(pipe_class_call);
16097 %}
16098
16099 // TO HERE
16100
16101 // Call Java Dynamic Instruction
16102 instruct CallDynamicJavaDirect(method meth)
16103 %{
16104 match(CallDynamicJava);
16105
16106 effect(USE meth);
16107
16108 ins_cost(CALL_COST);
16109
16110 format %{ "CALL,dynamic $meth \t// ==> " %}
16111
16112 ins_encode(aarch64_enc_java_dynamic_call(meth),
16113 aarch64_enc_call_epilog);
16114
16115 ins_pipe(pipe_class_call);
16116 %}
16117
16118 // Call Runtime Instruction
16119
16120 instruct CallRuntimeDirect(method meth)
16121 %{
16122 match(CallRuntime);
16123
16124 effect(USE meth);
16125
16126 ins_cost(CALL_COST);
16127
16128 format %{ "CALL, runtime $meth" %}
16129
16130 ins_encode( aarch64_enc_java_to_runtime(meth) );
16131
16132 ins_pipe(pipe_class_call);
16133 %}
16134
16135 // Call Runtime Instruction
16136
16137 instruct CallLeafDirect(method meth)
16138 %{
16139 match(CallLeaf);
16140
16141 effect(USE meth);
16142
16143 ins_cost(CALL_COST);
16144
16145 format %{ "CALL, runtime leaf $meth" %}
16146
16147 ins_encode( aarch64_enc_java_to_runtime(meth) );
16148
16149 ins_pipe(pipe_class_call);
16150 %}
16151
16152 // Call Runtime Instruction without safepoint and with vector arguments
16153 instruct CallLeafDirectVector(method meth)
16154 %{
16155 match(CallLeafVector);
16156
16157 effect(USE meth);
16158
16159 ins_cost(CALL_COST);
16160
16161 format %{ "CALL, runtime leaf vector $meth" %}
16162
16163 ins_encode(aarch64_enc_java_to_runtime(meth));
16164
16165 ins_pipe(pipe_class_call);
16166 %}
16167
16168 // Call Runtime Instruction
16169
16170 instruct CallLeafNoFPDirect(method meth)
16171 %{
16172 match(CallLeafNoFP);
16173
16174 effect(USE meth);
16175
16176 ins_cost(CALL_COST);
16177
16178 format %{ "CALL, runtime leaf nofp $meth" %}
16179
16180 ins_encode( aarch64_enc_java_to_runtime(meth) );
16181
16182 ins_pipe(pipe_class_call);
16183 %}
16184
16185 // Tail Call; Jump from runtime stub to Java code.
16186 // Also known as an 'interprocedural jump'.
16187 // Target of jump will eventually return to caller.
16188 // TailJump below removes the return address.
16189 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16190 // emitted just above the TailCall which has reset rfp to the caller state.
16191 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16192 %{
16193 match(TailCall jump_target method_ptr);
16194
16195 ins_cost(CALL_COST);
16196
16197 format %{ "br $jump_target\t# $method_ptr holds method" %}
16198
16199 ins_encode(aarch64_enc_tail_call(jump_target));
16200
16201 ins_pipe(pipe_class_call);
16202 %}
16203
16204 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16205 %{
16206 match(TailJump jump_target ex_oop);
16207
16208 ins_cost(CALL_COST);
16209
16210 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16211
16212 ins_encode(aarch64_enc_tail_jmp(jump_target));
16213
16214 ins_pipe(pipe_class_call);
16215 %}
16216
16217 // Forward exception.
16218 instruct ForwardExceptionjmp()
16219 %{
16220 match(ForwardException);
16221 ins_cost(CALL_COST);
16222
16223 format %{ "b forward_exception_stub" %}
16224 ins_encode %{
16225 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16226 %}
16227 ins_pipe(pipe_class_call);
16228 %}
16229
16230 // Create exception oop: created by stack-crawling runtime code.
16231 // Created exception is now available to this handler, and is setup
16232 // just prior to jumping to this handler. No code emitted.
16233 // TODO check
16234 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16235 instruct CreateException(iRegP_R0 ex_oop)
16236 %{
16237 match(Set ex_oop (CreateEx));
16238
16239 format %{ " -- \t// exception oop; no code emitted" %}
16240
16241 size(0);
16242
16243 ins_encode( /*empty*/ );
16244
16245 ins_pipe(pipe_class_empty);
16246 %}
16247
16248 // Rethrow exception: The exception oop will come in the first
16249 // argument position. Then JUMP (not call) to the rethrow stub code.
16250 instruct RethrowException() %{
16251 match(Rethrow);
16252 ins_cost(CALL_COST);
16253
16254 format %{ "b rethrow_stub" %}
16255
16256 ins_encode( aarch64_enc_rethrow() );
16257
16258 ins_pipe(pipe_class_call);
16259 %}
16260
16261
16262 // Return Instruction
16263 // epilog node loads ret address into lr as part of frame pop
16264 instruct Ret()
16265 %{
16266 match(Return);
16267
16268 format %{ "ret\t// return register" %}
16269
16270 ins_encode( aarch64_enc_ret() );
16271
16272 ins_pipe(pipe_branch);
16273 %}
16274
16275 // Die now.
16276 instruct ShouldNotReachHere() %{
16277 match(Halt);
16278
16279 ins_cost(CALL_COST);
16280 format %{ "ShouldNotReachHere" %}
16281
16282 ins_encode %{
16283 if (is_reachable()) {
16284 __ stop(_halt_reason);
16285 }
16286 %}
16287
16288 ins_pipe(pipe_class_default);
16289 %}
16290
16291 // ============================================================================
16292 // Partial Subtype Check
16293 //
16294 // superklass array for an instance of the superklass. Set a hidden
16295 // internal cache on a hit (cache is checked with exposed code in
16296 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16297 // encoding ALSO sets flags.
16298
16299 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16300 %{
16301 match(Set result (PartialSubtypeCheck sub super));
16302 predicate(!UseSecondarySupersTable);
16303 effect(KILL cr, KILL temp);
16304
16305 ins_cost(20 * INSN_COST); // slightly larger than the next version
16306 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16307
16308 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16309
16310 opcode(0x1); // Force zero of result reg on hit
16311
16312 ins_pipe(pipe_class_memory);
16313 %}
16314
16315 // Two versions of partialSubtypeCheck, both used when we need to
16316 // search for a super class in the secondary supers array. The first
16317 // is used when we don't know _a priori_ the class being searched
16318 // for. The second, far more common, is used when we do know: this is
16319 // used for instanceof, checkcast, and any case where C2 can determine
16320 // it by constant propagation.
16321
16322 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16323 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16324 rFlagsReg cr)
16325 %{
16326 match(Set result (PartialSubtypeCheck sub super));
16327 predicate(UseSecondarySupersTable);
16328 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16329
16330 ins_cost(10 * INSN_COST); // slightly larger than the next version
16331 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16332
16333 ins_encode %{
16334 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16335 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16336 $vtemp$$FloatRegister,
16337 $result$$Register, /*L_success*/nullptr);
16338 %}
16339
16340 ins_pipe(pipe_class_memory);
16341 %}
16342
16343 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16344 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16345 rFlagsReg cr)
16346 %{
16347 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16348 predicate(UseSecondarySupersTable);
16349 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16350
16351 ins_cost(5 * INSN_COST); // smaller than the next version
16352 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16353
16354 ins_encode %{
16355 bool success = false;
16356 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16357 if (InlineSecondarySupersTest) {
16358 success =
16359 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16360 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16361 $vtemp$$FloatRegister,
16362 $result$$Register,
16363 super_klass_slot);
16364 } else {
16365 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16366 success = (call != nullptr);
16367 }
16368 if (!success) {
16369 ciEnv::current()->record_failure("CodeCache is full");
16370 return;
16371 }
16372 %}
16373
16374 ins_pipe(pipe_class_memory);
16375 %}
16376
16377 // Intrisics for String.compareTo()
16378
16379 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16380 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16381 %{
16382 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16383 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16384 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16385
16386 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16387 ins_encode %{
16388 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16389 __ string_compare($str1$$Register, $str2$$Register,
16390 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16391 $tmp1$$Register, $tmp2$$Register,
16392 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16393 %}
16394 ins_pipe(pipe_class_memory);
16395 %}
16396
16397 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16398 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16399 %{
16400 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16401 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16402 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16403
16404 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16405 ins_encode %{
16406 __ string_compare($str1$$Register, $str2$$Register,
16407 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16408 $tmp1$$Register, $tmp2$$Register,
16409 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16410 %}
16411 ins_pipe(pipe_class_memory);
16412 %}
16413
16414 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16415 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16416 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16417 %{
16418 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16419 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16420 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16421 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16422
16423 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16424 ins_encode %{
16425 __ string_compare($str1$$Register, $str2$$Register,
16426 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16427 $tmp1$$Register, $tmp2$$Register,
16428 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16429 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16430 %}
16431 ins_pipe(pipe_class_memory);
16432 %}
16433
16434 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16435 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16436 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16437 %{
16438 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16439 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16440 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16441 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16442
16443 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16444 ins_encode %{
16445 __ string_compare($str1$$Register, $str2$$Register,
16446 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16447 $tmp1$$Register, $tmp2$$Register,
16448 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16449 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16450 %}
16451 ins_pipe(pipe_class_memory);
16452 %}
16453
16454 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16455 // these string_compare variants as NEON register type for convenience so that the prototype of
16456 // string_compare can be shared with all variants.
16457
16458 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16459 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16460 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16461 pRegGov_P1 pgtmp2, rFlagsReg cr)
16462 %{
16463 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16464 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16465 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16466 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16467
16468 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16469 ins_encode %{
16470 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16471 __ string_compare($str1$$Register, $str2$$Register,
16472 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16473 $tmp1$$Register, $tmp2$$Register,
16474 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16475 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16476 StrIntrinsicNode::LL);
16477 %}
16478 ins_pipe(pipe_class_memory);
16479 %}
16480
16481 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16482 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16483 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16484 pRegGov_P1 pgtmp2, rFlagsReg cr)
16485 %{
16486 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16487 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16488 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16489 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16490
16491 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16492 ins_encode %{
16493 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16494 __ string_compare($str1$$Register, $str2$$Register,
16495 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16496 $tmp1$$Register, $tmp2$$Register,
16497 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16498 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16499 StrIntrinsicNode::LU);
16500 %}
16501 ins_pipe(pipe_class_memory);
16502 %}
16503
16504 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16505 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16506 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16507 pRegGov_P1 pgtmp2, rFlagsReg cr)
16508 %{
16509 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16510 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16511 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16512 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16513
16514 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16515 ins_encode %{
16516 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16517 __ string_compare($str1$$Register, $str2$$Register,
16518 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16519 $tmp1$$Register, $tmp2$$Register,
16520 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16521 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16522 StrIntrinsicNode::UL);
16523 %}
16524 ins_pipe(pipe_class_memory);
16525 %}
16526
16527 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16528 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16529 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16530 pRegGov_P1 pgtmp2, rFlagsReg cr)
16531 %{
16532 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16533 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16534 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16535 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16536
16537 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16538 ins_encode %{
16539 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16540 __ string_compare($str1$$Register, $str2$$Register,
16541 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16542 $tmp1$$Register, $tmp2$$Register,
16543 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16544 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16545 StrIntrinsicNode::UU);
16546 %}
16547 ins_pipe(pipe_class_memory);
16548 %}
16549
16550 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16551 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16552 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16553 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16554 %{
16555 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16556 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16557 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16558 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16559 TEMP vtmp0, TEMP vtmp1, KILL cr);
16560 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16561 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16562
16563 ins_encode %{
16564 __ string_indexof($str1$$Register, $str2$$Register,
16565 $cnt1$$Register, $cnt2$$Register,
16566 $tmp1$$Register, $tmp2$$Register,
16567 $tmp3$$Register, $tmp4$$Register,
16568 $tmp5$$Register, $tmp6$$Register,
16569 -1, $result$$Register, StrIntrinsicNode::UU);
16570 %}
16571 ins_pipe(pipe_class_memory);
16572 %}
16573
16574 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16575 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16576 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16577 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16578 %{
16579 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16580 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16581 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16582 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16583 TEMP vtmp0, TEMP vtmp1, KILL cr);
16584 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16585 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16586
16587 ins_encode %{
16588 __ string_indexof($str1$$Register, $str2$$Register,
16589 $cnt1$$Register, $cnt2$$Register,
16590 $tmp1$$Register, $tmp2$$Register,
16591 $tmp3$$Register, $tmp4$$Register,
16592 $tmp5$$Register, $tmp6$$Register,
16593 -1, $result$$Register, StrIntrinsicNode::LL);
16594 %}
16595 ins_pipe(pipe_class_memory);
16596 %}
16597
16598 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16599 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16600 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16601 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16602 %{
16603 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16604 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16605 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16606 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16607 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16608 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16609 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16610
16611 ins_encode %{
16612 __ string_indexof($str1$$Register, $str2$$Register,
16613 $cnt1$$Register, $cnt2$$Register,
16614 $tmp1$$Register, $tmp2$$Register,
16615 $tmp3$$Register, $tmp4$$Register,
16616 $tmp5$$Register, $tmp6$$Register,
16617 -1, $result$$Register, StrIntrinsicNode::UL);
16618 %}
16619 ins_pipe(pipe_class_memory);
16620 %}
16621
16622 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16623 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16624 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16625 %{
16626 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16627 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16628 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16629 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16630 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16631 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16632
16633 ins_encode %{
16634 int icnt2 = (int)$int_cnt2$$constant;
16635 __ string_indexof($str1$$Register, $str2$$Register,
16636 $cnt1$$Register, zr,
16637 $tmp1$$Register, $tmp2$$Register,
16638 $tmp3$$Register, $tmp4$$Register, zr, zr,
16639 icnt2, $result$$Register, StrIntrinsicNode::UU);
16640 %}
16641 ins_pipe(pipe_class_memory);
16642 %}
16643
16644 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16645 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16646 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16647 %{
16648 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16649 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16650 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16651 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16652 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16653 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16654
16655 ins_encode %{
16656 int icnt2 = (int)$int_cnt2$$constant;
16657 __ string_indexof($str1$$Register, $str2$$Register,
16658 $cnt1$$Register, zr,
16659 $tmp1$$Register, $tmp2$$Register,
16660 $tmp3$$Register, $tmp4$$Register, zr, zr,
16661 icnt2, $result$$Register, StrIntrinsicNode::LL);
16662 %}
16663 ins_pipe(pipe_class_memory);
16664 %}
16665
16666 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16667 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16668 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16669 %{
16670 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16671 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16672 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16673 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16674 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16675 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16676
16677 ins_encode %{
16678 int icnt2 = (int)$int_cnt2$$constant;
16679 __ string_indexof($str1$$Register, $str2$$Register,
16680 $cnt1$$Register, zr,
16681 $tmp1$$Register, $tmp2$$Register,
16682 $tmp3$$Register, $tmp4$$Register, zr, zr,
16683 icnt2, $result$$Register, StrIntrinsicNode::UL);
16684 %}
16685 ins_pipe(pipe_class_memory);
16686 %}
16687
16688 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16689 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16690 iRegINoSp tmp3, rFlagsReg cr)
16691 %{
16692 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16693 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16694 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16695 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16696
16697 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16698
16699 ins_encode %{
16700 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16701 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16702 $tmp3$$Register);
16703 %}
16704 ins_pipe(pipe_class_memory);
16705 %}
16706
16707 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16708 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16709 iRegINoSp tmp3, rFlagsReg cr)
16710 %{
16711 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16712 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16713 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16714 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16715
16716 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16717
16718 ins_encode %{
16719 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16720 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16721 $tmp3$$Register);
16722 %}
16723 ins_pipe(pipe_class_memory);
16724 %}
16725
16726 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16727 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16728 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16729 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16730 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16731 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16732 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16733 ins_encode %{
16734 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16735 $result$$Register, $ztmp1$$FloatRegister,
16736 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16737 $ptmp$$PRegister, true /* isL */);
16738 %}
16739 ins_pipe(pipe_class_memory);
16740 %}
16741
16742 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16743 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16744 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16745 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16746 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16747 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16748 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16749 ins_encode %{
16750 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16751 $result$$Register, $ztmp1$$FloatRegister,
16752 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16753 $ptmp$$PRegister, false /* isL */);
16754 %}
16755 ins_pipe(pipe_class_memory);
16756 %}
16757
16758 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16759 iRegI_R0 result, rFlagsReg cr)
16760 %{
16761 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16762 match(Set result (StrEquals (Binary str1 str2) cnt));
16763 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16764
16765 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16766 ins_encode %{
16767 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16768 __ string_equals($str1$$Register, $str2$$Register,
16769 $result$$Register, $cnt$$Register);
16770 %}
16771 ins_pipe(pipe_class_memory);
16772 %}
16773
16774 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16775 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16776 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16777 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16778 iRegP_R10 tmp, rFlagsReg cr)
16779 %{
16780 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16781 match(Set result (AryEq ary1 ary2));
16782 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16783 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16784 TEMP vtmp6, TEMP vtmp7, KILL cr);
16785
16786 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16787 ins_encode %{
16788 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16789 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16790 $result$$Register, $tmp$$Register, 1);
16791 if (tpc == nullptr) {
16792 ciEnv::current()->record_failure("CodeCache is full");
16793 return;
16794 }
16795 %}
16796 ins_pipe(pipe_class_memory);
16797 %}
16798
16799 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16800 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16801 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16802 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16803 iRegP_R10 tmp, rFlagsReg cr)
16804 %{
16805 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16806 match(Set result (AryEq ary1 ary2));
16807 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16808 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16809 TEMP vtmp6, TEMP vtmp7, KILL cr);
16810
16811 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16812 ins_encode %{
16813 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16814 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16815 $result$$Register, $tmp$$Register, 2);
16816 if (tpc == nullptr) {
16817 ciEnv::current()->record_failure("CodeCache is full");
16818 return;
16819 }
16820 %}
16821 ins_pipe(pipe_class_memory);
16822 %}
16823
16824 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16825 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16826 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16827 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16828 %{
16829 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16830 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16831 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16832
16833 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16834 ins_encode %{
16835 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16836 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16837 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16838 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16839 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16840 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16841 (BasicType)$basic_type$$constant);
16842 if (tpc == nullptr) {
16843 ciEnv::current()->record_failure("CodeCache is full");
16844 return;
16845 }
16846 %}
16847 ins_pipe(pipe_class_memory);
16848 %}
16849
16850 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16851 %{
16852 match(Set result (CountPositives ary1 len));
16853 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16854 format %{ "count positives byte[] $ary1,$len -> $result" %}
16855 ins_encode %{
16856 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16857 if (tpc == nullptr) {
16858 ciEnv::current()->record_failure("CodeCache is full");
16859 return;
16860 }
16861 %}
16862 ins_pipe( pipe_slow );
16863 %}
16864
16865 // fast char[] to byte[] compression
16866 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16867 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16868 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16869 iRegI_R0 result, rFlagsReg cr)
16870 %{
16871 match(Set result (StrCompressedCopy src (Binary dst len)));
16872 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16873 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16874
16875 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16876 ins_encode %{
16877 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16878 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16879 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16880 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16881 %}
16882 ins_pipe(pipe_slow);
16883 %}
16884
16885 // fast byte[] to char[] inflation
16886 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16887 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16888 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16889 %{
16890 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16891 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16892 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16893 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16894
16895 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16896 ins_encode %{
16897 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16898 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16899 $vtmp2$$FloatRegister, $tmp$$Register);
16900 if (tpc == nullptr) {
16901 ciEnv::current()->record_failure("CodeCache is full");
16902 return;
16903 }
16904 %}
16905 ins_pipe(pipe_class_memory);
16906 %}
16907
16908 // encode char[] to byte[] in ISO_8859_1
16909 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16910 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16911 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16912 iRegI_R0 result, rFlagsReg cr)
16913 %{
16914 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16915 match(Set result (EncodeISOArray src (Binary dst len)));
16916 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16917 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16918
16919 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16920 ins_encode %{
16921 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16922 $result$$Register, false,
16923 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16924 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16925 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16926 %}
16927 ins_pipe(pipe_class_memory);
16928 %}
16929
16930 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16931 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16932 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16933 iRegI_R0 result, rFlagsReg cr)
16934 %{
16935 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16936 match(Set result (EncodeISOArray src (Binary dst len)));
16937 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16938 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16939
16940 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16941 ins_encode %{
16942 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16943 $result$$Register, true,
16944 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16945 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16946 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16947 %}
16948 ins_pipe(pipe_class_memory);
16949 %}
16950
16951 //----------------------------- CompressBits/ExpandBits ------------------------
16952
16953 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16954 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16955 match(Set dst (CompressBits src mask));
16956 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16957 format %{ "mov $tsrc, $src\n\t"
16958 "mov $tmask, $mask\n\t"
16959 "bext $tdst, $tsrc, $tmask\n\t"
16960 "mov $dst, $tdst"
16961 %}
16962 ins_encode %{
16963 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16964 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16965 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16966 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16967 %}
16968 ins_pipe(pipe_slow);
16969 %}
16970
16971 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16972 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16973 match(Set dst (CompressBits (LoadI mem) mask));
16974 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16975 format %{ "ldrs $tsrc, $mem\n\t"
16976 "ldrs $tmask, $mask\n\t"
16977 "bext $tdst, $tsrc, $tmask\n\t"
16978 "mov $dst, $tdst"
16979 %}
16980 ins_encode %{
16981 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16983 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16984 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16985 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16986 %}
16987 ins_pipe(pipe_slow);
16988 %}
16989
16990 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16991 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16992 match(Set dst (CompressBits src mask));
16993 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16994 format %{ "mov $tsrc, $src\n\t"
16995 "mov $tmask, $mask\n\t"
16996 "bext $tdst, $tsrc, $tmask\n\t"
16997 "mov $dst, $tdst"
16998 %}
16999 ins_encode %{
17000 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17001 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17002 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17003 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17004 %}
17005 ins_pipe(pipe_slow);
17006 %}
17007
17008 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17009 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17010 match(Set dst (CompressBits (LoadL mem) mask));
17011 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17012 format %{ "ldrd $tsrc, $mem\n\t"
17013 "ldrd $tmask, $mask\n\t"
17014 "bext $tdst, $tsrc, $tmask\n\t"
17015 "mov $dst, $tdst"
17016 %}
17017 ins_encode %{
17018 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17020 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17021 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17022 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17023 %}
17024 ins_pipe(pipe_slow);
17025 %}
17026
17027 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17028 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17029 match(Set dst (ExpandBits src mask));
17030 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17031 format %{ "mov $tsrc, $src\n\t"
17032 "mov $tmask, $mask\n\t"
17033 "bdep $tdst, $tsrc, $tmask\n\t"
17034 "mov $dst, $tdst"
17035 %}
17036 ins_encode %{
17037 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17038 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17039 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17040 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17041 %}
17042 ins_pipe(pipe_slow);
17043 %}
17044
17045 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17046 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17047 match(Set dst (ExpandBits (LoadI mem) mask));
17048 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17049 format %{ "ldrs $tsrc, $mem\n\t"
17050 "ldrs $tmask, $mask\n\t"
17051 "bdep $tdst, $tsrc, $tmask\n\t"
17052 "mov $dst, $tdst"
17053 %}
17054 ins_encode %{
17055 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17056 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17057 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17058 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17059 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17060 %}
17061 ins_pipe(pipe_slow);
17062 %}
17063
17064 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17065 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17066 match(Set dst (ExpandBits src mask));
17067 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17068 format %{ "mov $tsrc, $src\n\t"
17069 "mov $tmask, $mask\n\t"
17070 "bdep $tdst, $tsrc, $tmask\n\t"
17071 "mov $dst, $tdst"
17072 %}
17073 ins_encode %{
17074 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17075 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17076 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17077 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17078 %}
17079 ins_pipe(pipe_slow);
17080 %}
17081
17082
17083 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17084 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17085 match(Set dst (ExpandBits (LoadL mem) mask));
17086 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17087 format %{ "ldrd $tsrc, $mem\n\t"
17088 "ldrd $tmask, $mask\n\t"
17089 "bdep $tdst, $tsrc, $tmask\n\t"
17090 "mov $dst, $tdst"
17091 %}
17092 ins_encode %{
17093 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17094 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17095 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17096 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17097 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17098 %}
17099 ins_pipe(pipe_slow);
17100 %}
17101
17102 // ============================================================================
17103 // This name is KNOWN by the ADLC and cannot be changed.
17104 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17105 // for this guy.
17106 instruct tlsLoadP(thread_RegP dst)
17107 %{
17108 match(Set dst (ThreadLocal));
17109
17110 ins_cost(0);
17111
17112 format %{ " -- \t// $dst=Thread::current(), empty" %}
17113
17114 size(0);
17115
17116 ins_encode( /*empty*/ );
17117
17118 ins_pipe(pipe_class_empty);
17119 %}
17120
17121 //----------PEEPHOLE RULES-----------------------------------------------------
17122 // These must follow all instruction definitions as they use the names
17123 // defined in the instructions definitions.
17124 //
17125 // peepmatch ( root_instr_name [preceding_instruction]* );
17126 //
17127 // peepconstraint %{
17128 // (instruction_number.operand_name relational_op instruction_number.operand_name
17129 // [, ...] );
17130 // // instruction numbers are zero-based using left to right order in peepmatch
17131 //
17132 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17133 // // provide an instruction_number.operand_name for each operand that appears
17134 // // in the replacement instruction's match rule
17135 //
17136 // ---------VM FLAGS---------------------------------------------------------
17137 //
17138 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17139 //
17140 // Each peephole rule is given an identifying number starting with zero and
17141 // increasing by one in the order seen by the parser. An individual peephole
17142 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17143 // on the command-line.
17144 //
17145 // ---------CURRENT LIMITATIONS----------------------------------------------
17146 //
17147 // Only match adjacent instructions in same basic block
17148 // Only equality constraints
17149 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17150 // Only one replacement instruction
17151 //
17152 // ---------EXAMPLE----------------------------------------------------------
17153 //
17154 // // pertinent parts of existing instructions in architecture description
17155 // instruct movI(iRegINoSp dst, iRegI src)
17156 // %{
17157 // match(Set dst (CopyI src));
17158 // %}
17159 //
17160 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17161 // %{
17162 // match(Set dst (AddI dst src));
17163 // effect(KILL cr);
17164 // %}
17165 //
17166 // // Change (inc mov) to lea
17167 // peephole %{
17168 // // increment preceded by register-register move
17169 // peepmatch ( incI_iReg movI );
17170 // // require that the destination register of the increment
17171 // // match the destination register of the move
17172 // peepconstraint ( 0.dst == 1.dst );
17173 // // construct a replacement instruction that sets
17174 // // the destination to ( move's source register + one )
17175 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17176 // %}
17177 //
17178
17179 // Implementation no longer uses movX instructions since
17180 // machine-independent system no longer uses CopyX nodes.
17181 //
17182 // peephole
17183 // %{
17184 // peepmatch (incI_iReg movI);
17185 // peepconstraint (0.dst == 1.dst);
17186 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17187 // %}
17188
17189 // peephole
17190 // %{
17191 // peepmatch (decI_iReg movI);
17192 // peepconstraint (0.dst == 1.dst);
17193 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17194 // %}
17195
17196 // peephole
17197 // %{
17198 // peepmatch (addI_iReg_imm movI);
17199 // peepconstraint (0.dst == 1.dst);
17200 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17201 // %}
17202
17203 // peephole
17204 // %{
17205 // peepmatch (incL_iReg movL);
17206 // peepconstraint (0.dst == 1.dst);
17207 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17208 // %}
17209
17210 // peephole
17211 // %{
17212 // peepmatch (decL_iReg movL);
17213 // peepconstraint (0.dst == 1.dst);
17214 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17215 // %}
17216
17217 // peephole
17218 // %{
17219 // peepmatch (addL_iReg_imm movL);
17220 // peepconstraint (0.dst == 1.dst);
17221 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17222 // %}
17223
17224 // peephole
17225 // %{
17226 // peepmatch (addP_iReg_imm movP);
17227 // peepconstraint (0.dst == 1.dst);
17228 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17229 // %}
17230
17231 // // Change load of spilled value to only a spill
17232 // instruct storeI(memory mem, iRegI src)
17233 // %{
17234 // match(Set mem (StoreI mem src));
17235 // %}
17236 //
17237 // instruct loadI(iRegINoSp dst, memory mem)
17238 // %{
17239 // match(Set dst (LoadI mem));
17240 // %}
17241 //
17242
17243 //----------SMARTSPILL RULES---------------------------------------------------
17244 // These must follow all instruction definitions as they use the names
17245 // defined in the instructions definitions.
17246
17247 // Local Variables:
17248 // mode: c++
17249 // End: