1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // lea(rscratch1, RuntimeAddress(addr)
1652 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 __ verified_entry(C, 0);
1773
1774 if (C->stub_function() == nullptr) {
1775 __ entry_barrier();
1776 }
1777
1778 if (!Compile::current()->output()->in_scratch_emit_size()) {
1779 __ bind(*_verified_entry);
1780 }
1781
1782 if (VerifyStackAtCalls) {
1783 Unimplemented();
1784 }
1785
1786 C->output()->set_frame_complete(__ offset());
1787
1788 if (C->has_mach_constant_base_node()) {
1789 // NOTE: We set the table base offset here because users might be
1790 // emitted before MachConstantBaseNode.
1791 ConstantTable& constant_table = C->output()->constant_table();
1792 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1793 }
1794 }
1795
1796 int MachPrologNode::reloc() const
1797 {
1798 return 0;
1799 }
1800
1801 //=============================================================================
1802
1803 #ifndef PRODUCT
1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1805 Compile* C = ra_->C;
1806 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1807
1808 st->print("# pop frame %d\n\t",framesize);
1809
1810 if (framesize == 0) {
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1813 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, #%d\n\t", framesize);
1815 } else {
1816 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1817 st->print("add sp, sp, rscratch1\n\t");
1818 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1819 }
1820 if (VM_Version::use_rop_protection()) {
1821 st->print("autiaz\n\t");
1822 st->print("ldr zr, [lr]\n\t");
1823 }
1824
1825 if (do_polling() && C->is_method_compilation()) {
1826 st->print("# test polling word\n\t");
1827 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1828 st->print("cmp sp, rscratch1\n\t");
1829 st->print("bhi #slow_path");
1830 }
1831 }
1832 #endif
1833
1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1835 Compile* C = ra_->C;
1836 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1837
1838 __ remove_frame(framesize, C->needs_stack_repair());
1839
1840 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1841 __ reserved_stack_check();
1842 }
1843
1844 if (do_polling() && C->is_method_compilation()) {
1845 Label dummy_label;
1846 Label* code_stub = &dummy_label;
1847 if (!C->output()->in_scratch_emit_size()) {
1848 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1849 C->output()->add_stub(stub);
1850 code_stub = &stub->entry();
1851 }
1852 __ relocate(relocInfo::poll_return_type);
1853 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1854 }
1855 }
1856
1857 int MachEpilogNode::reloc() const {
1858 // Return number of relocatable values contained in this instruction.
1859 return 1; // 1 for polling page.
1860 }
1861
1862 const Pipeline * MachEpilogNode::pipeline() const {
1863 return MachNode::pipeline_class();
1864 }
1865
1866 //=============================================================================
1867
1868 static enum RC rc_class(OptoReg::Name reg) {
1869
1870 if (reg == OptoReg::Bad) {
1871 return rc_bad;
1872 }
1873
1874 // we have 32 int registers * 2 halves
1875 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1876
1877 if (reg < slots_of_int_registers) {
1878 return rc_int;
1879 }
1880
1881 // we have 32 float register * 8 halves
1882 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1883 if (reg < slots_of_int_registers + slots_of_float_registers) {
1884 return rc_float;
1885 }
1886
1887 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1888 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1889 return rc_predicate;
1890 }
1891
1892 // Between predicate regs & stack is the flags.
1893 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1894
1895 return rc_stack;
1896 }
1897
1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1899 Compile* C = ra_->C;
1900
1901 // Get registers to move.
1902 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1903 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1904 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1905 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1906
1907 enum RC src_hi_rc = rc_class(src_hi);
1908 enum RC src_lo_rc = rc_class(src_lo);
1909 enum RC dst_hi_rc = rc_class(dst_hi);
1910 enum RC dst_lo_rc = rc_class(dst_lo);
1911
1912 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1913
1914 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1915 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1916 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1917 "expected aligned-adjacent pairs");
1918 }
1919
1920 if (src_lo == dst_lo && src_hi == dst_hi) {
1921 return 0; // Self copy, no move.
1922 }
1923
1924 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1925 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1926 int src_offset = ra_->reg2offset(src_lo);
1927 int dst_offset = ra_->reg2offset(dst_lo);
1928
1929 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1930 uint ireg = ideal_reg();
1931 if (ireg == Op_VecA && masm) {
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (masm) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (masm) {
1979 switch (src_lo_rc) {
1980 case rc_int:
1981 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1982 if (is64) {
1983 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 } else {
1986 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1987 as_Register(Matcher::_regEncode[src_lo]));
1988 }
1989 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1990 if (is64) {
1991 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 } else {
1994 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 }
1997 } else { // gpr --> stack spill
1998 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1999 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2000 }
2001 break;
2002 case rc_float:
2003 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2004 if (is64) {
2005 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 } else {
2008 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 }
2011 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2012 if (is64) {
2013 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 }
2019 } else { // fpr --> stack spill
2020 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2021 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 is64 ? __ D : __ S, dst_offset);
2023 }
2024 break;
2025 case rc_stack:
2026 if (dst_lo_rc == rc_int) { // stack --> gpr load
2027 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2028 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 is64 ? __ D : __ S, src_offset);
2031 } else if (dst_lo_rc == rc_predicate) {
2032 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2033 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2034 } else { // stack --> stack copy
2035 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2036 if (ideal_reg() == Op_RegVectMask) {
2037 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2038 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2039 } else {
2040 __ unspill(rscratch1, is64, src_offset);
2041 __ spill(rscratch1, is64, dst_offset);
2042 }
2043 }
2044 break;
2045 case rc_predicate:
2046 if (dst_lo_rc == rc_predicate) {
2047 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2048 } else if (dst_lo_rc == rc_stack) {
2049 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2050 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2051 } else {
2052 assert(false, "bad src and dst rc_class combination.");
2053 ShouldNotReachHere();
2054 }
2055 break;
2056 default:
2057 assert(false, "bad rc_class for spill");
2058 ShouldNotReachHere();
2059 }
2060 }
2061
2062 if (st) {
2063 st->print("spill ");
2064 if (src_lo_rc == rc_stack) {
2065 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2066 } else {
2067 st->print("%s -> ", Matcher::regName[src_lo]);
2068 }
2069 if (dst_lo_rc == rc_stack) {
2070 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2071 } else {
2072 st->print("%s", Matcher::regName[dst_lo]);
2073 }
2074 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2075 int vsize = 0;
2076 switch (ideal_reg()) {
2077 case Op_VecD:
2078 vsize = 64;
2079 break;
2080 case Op_VecX:
2081 vsize = 128;
2082 break;
2083 case Op_VecA:
2084 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2085 break;
2086 default:
2087 assert(false, "bad register type for spill");
2088 ShouldNotReachHere();
2089 }
2090 st->print("\t# vector spill size = %d", vsize);
2091 } else if (ideal_reg() == Op_RegVectMask) {
2092 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2093 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2094 st->print("\t# predicate spill size = %d", vsize);
2095 } else {
2096 st->print("\t# spill size = %d", is64 ? 64 : 32);
2097 }
2098 }
2099
2100 return 0;
2101
2102 }
2103
2104 #ifndef PRODUCT
2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2106 if (!ra_)
2107 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2108 else
2109 implementation(nullptr, ra_, false, st);
2110 }
2111 #endif
2112
2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2114 implementation(masm, ra_, false, nullptr);
2115 }
2116
2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2118 return MachNode::size(ra_);
2119 }
2120
2121 //=============================================================================
2122
2123 #ifndef PRODUCT
2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2125 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2126 int reg = ra_->get_reg_first(this);
2127 st->print("add %s, rsp, #%d]\t# box lock",
2128 Matcher::regName[reg], offset);
2129 }
2130 #endif
2131
2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2133 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2134 int reg = ra_->get_encode(this);
2135
2136 // This add will handle any 24-bit signed offset. 24 bits allows an
2137 // 8 megabyte stack frame.
2138 __ add(as_Register(reg), sp, offset);
2139 }
2140
2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2142 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2143 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2144
2145 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2146 return NativeInstruction::instruction_size;
2147 } else {
2148 return 2 * NativeInstruction::instruction_size;
2149 }
2150 }
2151
2152 ///=============================================================================
2153 #ifndef PRODUCT
2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2155 {
2156 st->print_cr("# MachVEPNode");
2157 if (!_verified) {
2158 st->print_cr("\t load_class");
2159 } else {
2160 st->print_cr("\t unpack_inline_arg");
2161 }
2162 }
2163 #endif
2164
2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2166 {
2167 if (!_verified) {
2168 __ ic_check(1);
2169 } else {
2170 // insert a nop at the start of the prolog so we can patch in a
2171 // branch if we need to invalidate the method later
2172 __ nop();
2173
2174 // TODO 8284443 Avoid creation of temporary frame
2175 if (ra_->C->stub_function() == nullptr) {
2176 __ verified_entry(ra_->C, 0);
2177 __ entry_barrier();
2178 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2179 __ remove_frame(framesize, false);
2180 }
2181 // Unpack inline type args passed as oop and then jump to
2182 // the verified entry point (skipping the unverified entry).
2183 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2184 // Emit code for verified entry and save increment for stack repair on return
2185 __ verified_entry(ra_->C, sp_inc);
2186 if (Compile::current()->output()->in_scratch_emit_size()) {
2187 Label dummy_verified_entry;
2188 __ b(dummy_verified_entry);
2189 } else {
2190 __ b(*_verified_entry);
2191 }
2192 }
2193 }
2194
2195 //=============================================================================
2196 #ifndef PRODUCT
2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2198 {
2199 st->print_cr("# MachUEPNode");
2200 if (UseCompressedClassPointers) {
2201 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2202 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2203 st->print_cr("\tcmpw rscratch1, r10");
2204 } else {
2205 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2206 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2207 st->print_cr("\tcmp rscratch1, r10");
2208 }
2209 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2210 }
2211 #endif
2212
2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2214 {
2215 __ ic_check(InteriorEntryAlignment);
2216 }
2217
2218 // REQUIRED EMIT CODE
2219
2220 //=============================================================================
2221
2222 // Emit exception handler code.
2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2224 {
2225 // mov rscratch1 #exception_blob_entry_point
2226 // br rscratch1
2227 // Note that the code buffer's insts_mark is always relative to insts.
2228 // That's why we must use the macroassembler to generate a handler.
2229 address base = __ start_a_stub(size_exception_handler());
2230 if (base == nullptr) {
2231 ciEnv::current()->record_failure("CodeCache is full");
2232 return 0; // CodeBuffer::expand failed
2233 }
2234 int offset = __ offset();
2235 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2236 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2237 __ end_a_stub();
2238 return offset;
2239 }
2240
2241 // Emit deopt handler code.
2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2243 {
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 address base = __ start_a_stub(size_deopt_handler());
2247 if (base == nullptr) {
2248 ciEnv::current()->record_failure("CodeCache is full");
2249 return 0; // CodeBuffer::expand failed
2250 }
2251 int offset = __ offset();
2252
2253 __ adr(lr, __ pc());
2254 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2255
2256 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2257 __ end_a_stub();
2258 return offset;
2259 }
2260
2261 // REQUIRED MATCHER CODE
2262
2263 //=============================================================================
2264
2265 bool Matcher::match_rule_supported(int opcode) {
2266 if (!has_match_rule(opcode))
2267 return false;
2268
2269 switch (opcode) {
2270 case Op_OnSpinWait:
2271 return VM_Version::supports_on_spin_wait();
2272 case Op_CacheWB:
2273 case Op_CacheWBPreSync:
2274 case Op_CacheWBPostSync:
2275 if (!VM_Version::supports_data_cache_line_flush()) {
2276 return false;
2277 }
2278 break;
2279 case Op_ExpandBits:
2280 case Op_CompressBits:
2281 if (!VM_Version::supports_svebitperm()) {
2282 return false;
2283 }
2284 break;
2285 case Op_FmaF:
2286 case Op_FmaD:
2287 case Op_FmaVF:
2288 case Op_FmaVD:
2289 if (!UseFMA) {
2290 return false;
2291 }
2292 break;
2293 }
2294
2295 return true; // Per default match rules are supported.
2296 }
2297
2298 const RegMask* Matcher::predicate_reg_mask(void) {
2299 return &_PR_REG_mask;
2300 }
2301
2302 // Vector calling convention not yet implemented.
2303 bool Matcher::supports_vector_calling_convention(void) {
2304 return false;
2305 }
2306
2307 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2308 Unimplemented();
2309 return OptoRegPair(0, 0);
2310 }
2311
2312 // Is this branch offset short enough that a short branch can be used?
2313 //
2314 // NOTE: If the platform does not provide any short branch variants, then
2315 // this method should return false for offset 0.
2316 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2317 // The passed offset is relative to address of the branch.
2318
2319 return (-32768 <= offset && offset < 32768);
2320 }
2321
2322 // Vector width in bytes.
2323 int Matcher::vector_width_in_bytes(BasicType bt) {
2324 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2325 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2326 // Minimum 2 values in vector
2327 if (size < 2*type2aelembytes(bt)) size = 0;
2328 // But never < 4
2329 if (size < 4) size = 0;
2330 return size;
2331 }
2332
2333 // Limits on vector size (number of elements) loaded into vector.
2334 int Matcher::max_vector_size(const BasicType bt) {
2335 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2336 }
2337
2338 int Matcher::min_vector_size(const BasicType bt) {
2339 int max_size = max_vector_size(bt);
2340 // Limit the min vector size to 8 bytes.
2341 int size = 8 / type2aelembytes(bt);
2342 if (bt == T_BYTE) {
2343 // To support vector api shuffle/rearrange.
2344 size = 4;
2345 } else if (bt == T_BOOLEAN) {
2346 // To support vector api load/store mask.
2347 size = 2;
2348 }
2349 if (size < 2) size = 2;
2350 return MIN2(size, max_size);
2351 }
2352
2353 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2354 return Matcher::max_vector_size(bt);
2355 }
2356
2357 // Actual max scalable vector register length.
2358 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2359 return Matcher::max_vector_size(bt);
2360 }
2361
2362 // Vector ideal reg.
2363 uint Matcher::vector_ideal_reg(int len) {
2364 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2365 return Op_VecA;
2366 }
2367 switch(len) {
2368 // For 16-bit/32-bit mask vector, reuse VecD.
2369 case 2:
2370 case 4:
2371 case 8: return Op_VecD;
2372 case 16: return Op_VecX;
2373 }
2374 ShouldNotReachHere();
2375 return 0;
2376 }
2377
2378 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2379 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2380 switch (ideal_reg) {
2381 case Op_VecA: return new vecAOper();
2382 case Op_VecD: return new vecDOper();
2383 case Op_VecX: return new vecXOper();
2384 }
2385 ShouldNotReachHere();
2386 return nullptr;
2387 }
2388
2389 bool Matcher::is_reg2reg_move(MachNode* m) {
2390 return false;
2391 }
2392
2393 bool Matcher::is_generic_vector(MachOper* opnd) {
2394 return opnd->opcode() == VREG;
2395 }
2396
2397 // Return whether or not this register is ever used as an argument.
2398 // This function is used on startup to build the trampoline stubs in
2399 // generateOptoStub. Registers not mentioned will be killed by the VM
2400 // call in the trampoline, and arguments in those registers not be
2401 // available to the callee.
2402 bool Matcher::can_be_java_arg(int reg)
2403 {
2404 return
2405 reg == R0_num || reg == R0_H_num ||
2406 reg == R1_num || reg == R1_H_num ||
2407 reg == R2_num || reg == R2_H_num ||
2408 reg == R3_num || reg == R3_H_num ||
2409 reg == R4_num || reg == R4_H_num ||
2410 reg == R5_num || reg == R5_H_num ||
2411 reg == R6_num || reg == R6_H_num ||
2412 reg == R7_num || reg == R7_H_num ||
2413 reg == V0_num || reg == V0_H_num ||
2414 reg == V1_num || reg == V1_H_num ||
2415 reg == V2_num || reg == V2_H_num ||
2416 reg == V3_num || reg == V3_H_num ||
2417 reg == V4_num || reg == V4_H_num ||
2418 reg == V5_num || reg == V5_H_num ||
2419 reg == V6_num || reg == V6_H_num ||
2420 reg == V7_num || reg == V7_H_num;
2421 }
2422
2423 bool Matcher::is_spillable_arg(int reg)
2424 {
2425 return can_be_java_arg(reg);
2426 }
2427
2428 uint Matcher::int_pressure_limit()
2429 {
2430 // JDK-8183543: When taking the number of available registers as int
2431 // register pressure threshold, the jtreg test:
2432 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2433 // failed due to C2 compilation failure with
2434 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2435 //
2436 // A derived pointer is live at CallNode and then is flagged by RA
2437 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2438 // derived pointers and lastly fail to spill after reaching maximum
2439 // number of iterations. Lowering the default pressure threshold to
2440 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2441 // a high register pressure area of the code so that split_DEF can
2442 // generate DefinitionSpillCopy for the derived pointer.
2443 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2444 if (!PreserveFramePointer) {
2445 // When PreserveFramePointer is off, frame pointer is allocatable,
2446 // but different from other SOC registers, it is excluded from
2447 // fatproj's mask because its save type is No-Save. Decrease 1 to
2448 // ensure high pressure at fatproj when PreserveFramePointer is off.
2449 // See check_pressure_at_fatproj().
2450 default_int_pressure_threshold--;
2451 }
2452 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2453 }
2454
2455 uint Matcher::float_pressure_limit()
2456 {
2457 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2458 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2459 }
2460
2461 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2462 return false;
2463 }
2464
2465 RegMask Matcher::divI_proj_mask() {
2466 ShouldNotReachHere();
2467 return RegMask();
2468 }
2469
2470 // Register for MODI projection of divmodI.
2471 RegMask Matcher::modI_proj_mask() {
2472 ShouldNotReachHere();
2473 return RegMask();
2474 }
2475
2476 // Register for DIVL projection of divmodL.
2477 RegMask Matcher::divL_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODL projection of divmodL.
2483 RegMask Matcher::modL_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2489 return FP_REG_mask();
2490 }
2491
2492 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2493 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2494 Node* u = addp->fast_out(i);
2495 if (u->is_LoadStore()) {
2496 // On AArch64, LoadStoreNodes (i.e. compare and swap
2497 // instructions) only take register indirect as an operand, so
2498 // any attempt to use an AddPNode as an input to a LoadStoreNode
2499 // must fail.
2500 return false;
2501 }
2502 if (u->is_Mem()) {
2503 int opsize = u->as_Mem()->memory_size();
2504 assert(opsize > 0, "unexpected memory operand size");
2505 if (u->as_Mem()->memory_size() != (1<<shift)) {
2506 return false;
2507 }
2508 }
2509 }
2510 return true;
2511 }
2512
2513 // Convert BootTest condition to Assembler condition.
2514 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2515 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2516 Assembler::Condition result;
2517 switch(cond) {
2518 case BoolTest::eq:
2519 result = Assembler::EQ; break;
2520 case BoolTest::ne:
2521 result = Assembler::NE; break;
2522 case BoolTest::le:
2523 result = Assembler::LE; break;
2524 case BoolTest::ge:
2525 result = Assembler::GE; break;
2526 case BoolTest::lt:
2527 result = Assembler::LT; break;
2528 case BoolTest::gt:
2529 result = Assembler::GT; break;
2530 case BoolTest::ule:
2531 result = Assembler::LS; break;
2532 case BoolTest::uge:
2533 result = Assembler::HS; break;
2534 case BoolTest::ult:
2535 result = Assembler::LO; break;
2536 case BoolTest::ugt:
2537 result = Assembler::HI; break;
2538 case BoolTest::overflow:
2539 result = Assembler::VS; break;
2540 case BoolTest::no_overflow:
2541 result = Assembler::VC; break;
2542 default:
2543 ShouldNotReachHere();
2544 return Assembler::Condition(-1);
2545 }
2546
2547 // Check conversion
2548 if (cond & BoolTest::unsigned_compare) {
2549 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2550 } else {
2551 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2552 }
2553
2554 return result;
2555 }
2556
2557 // Binary src (Replicate con)
2558 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2559 if (n == nullptr || m == nullptr) {
2560 return false;
2561 }
2562
2563 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2564 return false;
2565 }
2566
2567 Node* imm_node = m->in(1);
2568 if (!imm_node->is_Con()) {
2569 return false;
2570 }
2571
2572 const Type* t = imm_node->bottom_type();
2573 if (!(t->isa_int() || t->isa_long())) {
2574 return false;
2575 }
2576
2577 switch (n->Opcode()) {
2578 case Op_AndV:
2579 case Op_OrV:
2580 case Op_XorV: {
2581 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2582 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2583 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2584 }
2585 case Op_AddVB:
2586 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2587 case Op_AddVS:
2588 case Op_AddVI:
2589 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2590 case Op_AddVL:
2591 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2592 default:
2593 return false;
2594 }
2595 }
2596
2597 // (XorV src (Replicate m1))
2598 // (XorVMask src (MaskAll m1))
2599 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2600 if (n != nullptr && m != nullptr) {
2601 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2602 VectorNode::is_all_ones_vector(m);
2603 }
2604 return false;
2605 }
2606
2607 // Should the matcher clone input 'm' of node 'n'?
2608 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2609 if (is_vshift_con_pattern(n, m) ||
2610 is_vector_bitwise_not_pattern(n, m) ||
2611 is_valid_sve_arith_imm_pattern(n, m) ||
2612 is_encode_and_store_pattern(n, m)) {
2613 mstack.push(m, Visit);
2614 return true;
2615 }
2616 return false;
2617 }
2618
2619 // Should the Matcher clone shifts on addressing modes, expecting them
2620 // to be subsumed into complex addressing expressions or compute them
2621 // into registers?
2622 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2623 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2624 return true;
2625 }
2626
2627 Node *off = m->in(AddPNode::Offset);
2628 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2629 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2630 // Are there other uses besides address expressions?
2631 !is_visited(off)) {
2632 address_visited.set(off->_idx); // Flag as address_visited
2633 mstack.push(off->in(2), Visit);
2634 Node *conv = off->in(1);
2635 if (conv->Opcode() == Op_ConvI2L &&
2636 // Are there other uses besides address expressions?
2637 !is_visited(conv)) {
2638 address_visited.set(conv->_idx); // Flag as address_visited
2639 mstack.push(conv->in(1), Pre_Visit);
2640 } else {
2641 mstack.push(conv, Pre_Visit);
2642 }
2643 address_visited.test_set(m->_idx); // Flag as address_visited
2644 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2645 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2646 return true;
2647 } else if (off->Opcode() == Op_ConvI2L &&
2648 // Are there other uses besides address expressions?
2649 !is_visited(off)) {
2650 address_visited.test_set(m->_idx); // Flag as address_visited
2651 address_visited.set(off->_idx); // Flag as address_visited
2652 mstack.push(off->in(1), Pre_Visit);
2653 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2654 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2655 return true;
2656 }
2657 return false;
2658 }
2659
2660 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2661 { \
2662 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2663 guarantee(DISP == 0, "mode not permitted for volatile"); \
2664 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2665 __ INSN(REG, as_Register(BASE)); \
2666 }
2667
2668
2669 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2670 {
2671 Address::extend scale;
2672
2673 // Hooboy, this is fugly. We need a way to communicate to the
2674 // encoder that the index needs to be sign extended, so we have to
2675 // enumerate all the cases.
2676 switch (opcode) {
2677 case INDINDEXSCALEDI2L:
2678 case INDINDEXSCALEDI2LN:
2679 case INDINDEXI2L:
2680 case INDINDEXI2LN:
2681 scale = Address::sxtw(size);
2682 break;
2683 default:
2684 scale = Address::lsl(size);
2685 }
2686
2687 if (index == -1) {
2688 return Address(base, disp);
2689 } else {
2690 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2691 return Address(base, as_Register(index), scale);
2692 }
2693 }
2694
2695
2696 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2697 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2698 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2699 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2700 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2701
2702 // Used for all non-volatile memory accesses. The use of
2703 // $mem->opcode() to discover whether this pattern uses sign-extended
2704 // offsets is something of a kludge.
2705 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2706 Register reg, int opcode,
2707 Register base, int index, int scale, int disp,
2708 int size_in_memory)
2709 {
2710 Address addr = mem2address(opcode, base, index, scale, disp);
2711 if (addr.getMode() == Address::base_plus_offset) {
2712 /* Fix up any out-of-range offsets. */
2713 assert_different_registers(rscratch1, base);
2714 assert_different_registers(rscratch1, reg);
2715 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2716 }
2717 (masm->*insn)(reg, addr);
2718 }
2719
2720 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2721 FloatRegister reg, int opcode,
2722 Register base, int index, int size, int disp,
2723 int size_in_memory)
2724 {
2725 Address::extend scale;
2726
2727 switch (opcode) {
2728 case INDINDEXSCALEDI2L:
2729 case INDINDEXSCALEDI2LN:
2730 scale = Address::sxtw(size);
2731 break;
2732 default:
2733 scale = Address::lsl(size);
2734 }
2735
2736 if (index == -1) {
2737 // Fix up any out-of-range offsets.
2738 assert_different_registers(rscratch1, base);
2739 Address addr = Address(base, disp);
2740 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2741 (masm->*insn)(reg, addr);
2742 } else {
2743 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2744 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2745 }
2746 }
2747
2748 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2749 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2750 int opcode, Register base, int index, int size, int disp)
2751 {
2752 if (index == -1) {
2753 (masm->*insn)(reg, T, Address(base, disp));
2754 } else {
2755 assert(disp == 0, "unsupported address mode");
2756 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2757 }
2758 }
2759
2760 %}
2761
2762
2763
2764 //----------ENCODING BLOCK-----------------------------------------------------
2765 // This block specifies the encoding classes used by the compiler to
2766 // output byte streams. Encoding classes are parameterized macros
2767 // used by Machine Instruction Nodes in order to generate the bit
2768 // encoding of the instruction. Operands specify their base encoding
2769 // interface with the interface keyword. There are currently
2770 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2771 // COND_INTER. REG_INTER causes an operand to generate a function
2772 // which returns its register number when queried. CONST_INTER causes
2773 // an operand to generate a function which returns the value of the
2774 // constant when queried. MEMORY_INTER causes an operand to generate
2775 // four functions which return the Base Register, the Index Register,
2776 // the Scale Value, and the Offset Value of the operand when queried.
2777 // COND_INTER causes an operand to generate six functions which return
2778 // the encoding code (ie - encoding bits for the instruction)
2779 // associated with each basic boolean condition for a conditional
2780 // instruction.
2781 //
2782 // Instructions specify two basic values for encoding. Again, a
2783 // function is available to check if the constant displacement is an
2784 // oop. They use the ins_encode keyword to specify their encoding
2785 // classes (which must be a sequence of enc_class names, and their
2786 // parameters, specified in the encoding block), and they use the
2787 // opcode keyword to specify, in order, their primary, secondary, and
2788 // tertiary opcode. Only the opcode sections which a particular
2789 // instruction needs for encoding need to be specified.
2790 encode %{
2791 // Build emit functions for each basic byte or larger field in the
2792 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2793 // from C++ code in the enc_class source block. Emit functions will
2794 // live in the main source block for now. In future, we can
2795 // generalize this by adding a syntax that specifies the sizes of
2796 // fields in an order, so that the adlc can build the emit functions
2797 // automagically
2798
2799 // catch all for unimplemented encodings
2800 enc_class enc_unimplemented %{
2801 __ unimplemented("C2 catch all");
2802 %}
2803
2804 // BEGIN Non-volatile memory access
2805
2806 // This encoding class is generated automatically from ad_encode.m4.
2807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2808 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2809 Register dst_reg = as_Register($dst$$reg);
2810 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2811 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2812 %}
2813
2814 // This encoding class is generated automatically from ad_encode.m4.
2815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2816 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2817 Register dst_reg = as_Register($dst$$reg);
2818 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2819 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2820 %}
2821
2822 // This encoding class is generated automatically from ad_encode.m4.
2823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2824 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2825 Register dst_reg = as_Register($dst$$reg);
2826 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2827 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2828 %}
2829
2830 // This encoding class is generated automatically from ad_encode.m4.
2831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2832 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2833 Register dst_reg = as_Register($dst$$reg);
2834 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2835 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2836 %}
2837
2838 // This encoding class is generated automatically from ad_encode.m4.
2839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2840 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2841 Register dst_reg = as_Register($dst$$reg);
2842 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2843 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2844 %}
2845
2846 // This encoding class is generated automatically from ad_encode.m4.
2847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2848 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2849 Register dst_reg = as_Register($dst$$reg);
2850 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2851 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2852 %}
2853
2854 // This encoding class is generated automatically from ad_encode.m4.
2855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2856 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2857 Register dst_reg = as_Register($dst$$reg);
2858 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2859 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2860 %}
2861
2862 // This encoding class is generated automatically from ad_encode.m4.
2863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2864 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2865 Register dst_reg = as_Register($dst$$reg);
2866 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2867 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2868 %}
2869
2870 // This encoding class is generated automatically from ad_encode.m4.
2871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2872 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2873 Register dst_reg = as_Register($dst$$reg);
2874 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2875 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2876 %}
2877
2878 // This encoding class is generated automatically from ad_encode.m4.
2879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2880 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2881 Register dst_reg = as_Register($dst$$reg);
2882 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2883 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2884 %}
2885
2886 // This encoding class is generated automatically from ad_encode.m4.
2887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2888 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2889 Register dst_reg = as_Register($dst$$reg);
2890 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2892 %}
2893
2894 // This encoding class is generated automatically from ad_encode.m4.
2895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2896 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2897 Register dst_reg = as_Register($dst$$reg);
2898 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2899 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2900 %}
2901
2902 // This encoding class is generated automatically from ad_encode.m4.
2903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2904 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2905 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2906 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2908 %}
2909
2910 // This encoding class is generated automatically from ad_encode.m4.
2911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2912 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2913 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2914 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2915 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2916 %}
2917
2918 // This encoding class is generated automatically from ad_encode.m4.
2919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2920 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2921 Register src_reg = as_Register($src$$reg);
2922 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2924 %}
2925
2926 // This encoding class is generated automatically from ad_encode.m4.
2927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2928 enc_class aarch64_enc_strb0(memory1 mem) %{
2929 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2936 Register src_reg = as_Register($src$$reg);
2937 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_strh0(memory2 mem) %{
2944 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2945 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2946 %}
2947
2948 // This encoding class is generated automatically from ad_encode.m4.
2949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2950 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2951 Register src_reg = as_Register($src$$reg);
2952 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2953 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2954 %}
2955
2956 // This encoding class is generated automatically from ad_encode.m4.
2957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2958 enc_class aarch64_enc_strw0(memory4 mem) %{
2959 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2966 Register src_reg = as_Register($src$$reg);
2967 // we sometimes get asked to store the stack pointer into the
2968 // current thread -- we cannot do that directly on AArch64
2969 if (src_reg == r31_sp) {
2970 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2971 __ mov(rscratch2, sp);
2972 src_reg = rscratch2;
2973 }
2974 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_str0(memory8 mem) %{
2981 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2988 FloatRegister src_reg = as_FloatRegister($src$$reg);
2989 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2996 FloatRegister src_reg = as_FloatRegister($src$$reg);
2997 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3004 __ membar(Assembler::StoreStore);
3005 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3006 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3007 %}
3008
3009 // END Non-volatile memory access
3010
3011 // Vector loads and stores
3012 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3013 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3014 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3015 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3016 %}
3017
3018 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3019 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3020 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3021 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3022 %}
3023
3024 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3025 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3026 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3031 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3032 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3037 FloatRegister src_reg = as_FloatRegister($src$$reg);
3038 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3039 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3040 %}
3041
3042 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3043 FloatRegister src_reg = as_FloatRegister($src$$reg);
3044 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3045 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3046 %}
3047
3048 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3049 FloatRegister src_reg = as_FloatRegister($src$$reg);
3050 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3051 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3052 %}
3053
3054 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3055 FloatRegister src_reg = as_FloatRegister($src$$reg);
3056 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3057 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3058 %}
3059
3060 // volatile loads and stores
3061
3062 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3063 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3064 rscratch1, stlrb);
3065 %}
3066
3067 enc_class aarch64_enc_stlrb0(memory mem) %{
3068 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3069 rscratch1, stlrb);
3070 %}
3071
3072 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3073 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3074 rscratch1, stlrh);
3075 %}
3076
3077 enc_class aarch64_enc_stlrh0(memory mem) %{
3078 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3079 rscratch1, stlrh);
3080 %}
3081
3082 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3083 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3084 rscratch1, stlrw);
3085 %}
3086
3087 enc_class aarch64_enc_stlrw0(memory mem) %{
3088 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3089 rscratch1, stlrw);
3090 %}
3091
3092 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3093 Register dst_reg = as_Register($dst$$reg);
3094 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3095 rscratch1, ldarb);
3096 __ sxtbw(dst_reg, dst_reg);
3097 %}
3098
3099 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3100 Register dst_reg = as_Register($dst$$reg);
3101 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3102 rscratch1, ldarb);
3103 __ sxtb(dst_reg, dst_reg);
3104 %}
3105
3106 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3107 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, ldarb);
3109 %}
3110
3111 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3112 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, ldarb);
3114 %}
3115
3116 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3117 Register dst_reg = as_Register($dst$$reg);
3118 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3119 rscratch1, ldarh);
3120 __ sxthw(dst_reg, dst_reg);
3121 %}
3122
3123 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3124 Register dst_reg = as_Register($dst$$reg);
3125 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3126 rscratch1, ldarh);
3127 __ sxth(dst_reg, dst_reg);
3128 %}
3129
3130 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3131 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3132 rscratch1, ldarh);
3133 %}
3134
3135 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3136 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, ldarh);
3138 %}
3139
3140 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarw);
3143 %}
3144
3145 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3146 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarw);
3148 %}
3149
3150 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3151 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, ldar);
3153 %}
3154
3155 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3156 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3157 rscratch1, ldarw);
3158 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3159 %}
3160
3161 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3162 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, ldar);
3164 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3165 %}
3166
3167 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3168 Register src_reg = as_Register($src$$reg);
3169 // we sometimes get asked to store the stack pointer into the
3170 // current thread -- we cannot do that directly on AArch64
3171 if (src_reg == r31_sp) {
3172 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3173 __ mov(rscratch2, sp);
3174 src_reg = rscratch2;
3175 }
3176 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, stlr);
3178 %}
3179
3180 enc_class aarch64_enc_stlr0(memory mem) %{
3181 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, stlr);
3183 %}
3184
3185 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3186 {
3187 FloatRegister src_reg = as_FloatRegister($src$$reg);
3188 __ fmovs(rscratch2, src_reg);
3189 }
3190 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, stlrw);
3192 %}
3193
3194 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3195 {
3196 FloatRegister src_reg = as_FloatRegister($src$$reg);
3197 __ fmovd(rscratch2, src_reg);
3198 }
3199 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, stlr);
3201 %}
3202
3203 // synchronized read/update encodings
3204
3205 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3206 Register dst_reg = as_Register($dst$$reg);
3207 Register base = as_Register($mem$$base);
3208 int index = $mem$$index;
3209 int scale = $mem$$scale;
3210 int disp = $mem$$disp;
3211 if (index == -1) {
3212 if (disp != 0) {
3213 __ lea(rscratch1, Address(base, disp));
3214 __ ldaxr(dst_reg, rscratch1);
3215 } else {
3216 // TODO
3217 // should we ever get anything other than this case?
3218 __ ldaxr(dst_reg, base);
3219 }
3220 } else {
3221 Register index_reg = as_Register(index);
3222 if (disp == 0) {
3223 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3224 __ ldaxr(dst_reg, rscratch1);
3225 } else {
3226 __ lea(rscratch1, Address(base, disp));
3227 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3228 __ ldaxr(dst_reg, rscratch1);
3229 }
3230 }
3231 %}
3232
3233 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3234 Register src_reg = as_Register($src$$reg);
3235 Register base = as_Register($mem$$base);
3236 int index = $mem$$index;
3237 int scale = $mem$$scale;
3238 int disp = $mem$$disp;
3239 if (index == -1) {
3240 if (disp != 0) {
3241 __ lea(rscratch2, Address(base, disp));
3242 __ stlxr(rscratch1, src_reg, rscratch2);
3243 } else {
3244 // TODO
3245 // should we ever get anything other than this case?
3246 __ stlxr(rscratch1, src_reg, base);
3247 }
3248 } else {
3249 Register index_reg = as_Register(index);
3250 if (disp == 0) {
3251 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3252 __ stlxr(rscratch1, src_reg, rscratch2);
3253 } else {
3254 __ lea(rscratch2, Address(base, disp));
3255 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3256 __ stlxr(rscratch1, src_reg, rscratch2);
3257 }
3258 }
3259 __ cmpw(rscratch1, zr);
3260 %}
3261
3262 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3263 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3264 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3265 Assembler::xword, /*acquire*/ false, /*release*/ true,
3266 /*weak*/ false, noreg);
3267 %}
3268
3269 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3270 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3271 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3272 Assembler::word, /*acquire*/ false, /*release*/ true,
3273 /*weak*/ false, noreg);
3274 %}
3275
3276 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3277 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3278 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3279 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3280 /*weak*/ false, noreg);
3281 %}
3282
3283 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3284 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3285 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3286 Assembler::byte, /*acquire*/ false, /*release*/ true,
3287 /*weak*/ false, noreg);
3288 %}
3289
3290
3291 // The only difference between aarch64_enc_cmpxchg and
3292 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3293 // CompareAndSwap sequence to serve as a barrier on acquiring a
3294 // lock.
3295 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::xword, /*acquire*/ true, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3303 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3304 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3305 Assembler::word, /*acquire*/ true, /*release*/ true,
3306 /*weak*/ false, noreg);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::byte, /*acquire*/ true, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 // auxiliary used for CompareAndSwapX to set result register
3324 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3325 Register res_reg = as_Register($res$$reg);
3326 __ cset(res_reg, Assembler::EQ);
3327 %}
3328
3329 // prefetch encodings
3330
3331 enc_class aarch64_enc_prefetchw(memory mem) %{
3332 Register base = as_Register($mem$$base);
3333 int index = $mem$$index;
3334 int scale = $mem$$scale;
3335 int disp = $mem$$disp;
3336 if (index == -1) {
3337 // Fix up any out-of-range offsets.
3338 assert_different_registers(rscratch1, base);
3339 Address addr = Address(base, disp);
3340 addr = __ legitimize_address(addr, 8, rscratch1);
3341 __ prfm(addr, PSTL1KEEP);
3342 } else {
3343 Register index_reg = as_Register(index);
3344 if (disp == 0) {
3345 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3346 } else {
3347 __ lea(rscratch1, Address(base, disp));
3348 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3349 }
3350 }
3351 %}
3352
3353 // mov encodings
3354
3355 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3356 uint32_t con = (uint32_t)$src$$constant;
3357 Register dst_reg = as_Register($dst$$reg);
3358 if (con == 0) {
3359 __ movw(dst_reg, zr);
3360 } else {
3361 __ movw(dst_reg, con);
3362 }
3363 %}
3364
3365 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3366 Register dst_reg = as_Register($dst$$reg);
3367 uint64_t con = (uint64_t)$src$$constant;
3368 if (con == 0) {
3369 __ mov(dst_reg, zr);
3370 } else {
3371 __ mov(dst_reg, con);
3372 }
3373 %}
3374
3375 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3376 Register dst_reg = as_Register($dst$$reg);
3377 address con = (address)$src$$constant;
3378 if (con == nullptr || con == (address)1) {
3379 ShouldNotReachHere();
3380 } else {
3381 relocInfo::relocType rtype = $src->constant_reloc();
3382 if (rtype == relocInfo::oop_type) {
3383 __ movoop(dst_reg, (jobject)con);
3384 } else if (rtype == relocInfo::metadata_type) {
3385 __ mov_metadata(dst_reg, (Metadata*)con);
3386 } else {
3387 assert(rtype == relocInfo::none, "unexpected reloc type");
3388 if (! __ is_valid_AArch64_address(con) ||
3389 con < (address)(uintptr_t)os::vm_page_size()) {
3390 __ mov(dst_reg, con);
3391 } else {
3392 uint64_t offset;
3393 __ adrp(dst_reg, con, offset);
3394 __ add(dst_reg, dst_reg, offset);
3395 }
3396 }
3397 }
3398 %}
3399
3400 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3401 Register dst_reg = as_Register($dst$$reg);
3402 __ mov(dst_reg, zr);
3403 %}
3404
3405 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3406 Register dst_reg = as_Register($dst$$reg);
3407 __ mov(dst_reg, (uint64_t)1);
3408 %}
3409
3410 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3411 __ load_byte_map_base($dst$$Register);
3412 %}
3413
3414 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3415 Register dst_reg = as_Register($dst$$reg);
3416 address con = (address)$src$$constant;
3417 if (con == nullptr) {
3418 ShouldNotReachHere();
3419 } else {
3420 relocInfo::relocType rtype = $src->constant_reloc();
3421 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3422 __ set_narrow_oop(dst_reg, (jobject)con);
3423 }
3424 %}
3425
3426 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, zr);
3429 %}
3430
3431 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3432 Register dst_reg = as_Register($dst$$reg);
3433 address con = (address)$src$$constant;
3434 if (con == nullptr) {
3435 ShouldNotReachHere();
3436 } else {
3437 relocInfo::relocType rtype = $src->constant_reloc();
3438 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3439 __ set_narrow_klass(dst_reg, (Klass *)con);
3440 }
3441 %}
3442
3443 // arithmetic encodings
3444
3445 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3446 Register dst_reg = as_Register($dst$$reg);
3447 Register src_reg = as_Register($src1$$reg);
3448 int32_t con = (int32_t)$src2$$constant;
3449 // add has primary == 0, subtract has primary == 1
3450 if ($primary) { con = -con; }
3451 if (con < 0) {
3452 __ subw(dst_reg, src_reg, -con);
3453 } else {
3454 __ addw(dst_reg, src_reg, con);
3455 }
3456 %}
3457
3458 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3459 Register dst_reg = as_Register($dst$$reg);
3460 Register src_reg = as_Register($src1$$reg);
3461 int32_t con = (int32_t)$src2$$constant;
3462 // add has primary == 0, subtract has primary == 1
3463 if ($primary) { con = -con; }
3464 if (con < 0) {
3465 __ sub(dst_reg, src_reg, -con);
3466 } else {
3467 __ add(dst_reg, src_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 Register src1_reg = as_Register($src1$$reg);
3474 Register src2_reg = as_Register($src2$$reg);
3475 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3476 %}
3477
3478 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 Register src1_reg = as_Register($src1$$reg);
3481 Register src2_reg = as_Register($src2$$reg);
3482 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3483 %}
3484
3485 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3486 Register dst_reg = as_Register($dst$$reg);
3487 Register src1_reg = as_Register($src1$$reg);
3488 Register src2_reg = as_Register($src2$$reg);
3489 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3490 %}
3491
3492 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3497 %}
3498
3499 // compare instruction encodings
3500
3501 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3502 Register reg1 = as_Register($src1$$reg);
3503 Register reg2 = as_Register($src2$$reg);
3504 __ cmpw(reg1, reg2);
3505 %}
3506
3507 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3508 Register reg = as_Register($src1$$reg);
3509 int32_t val = $src2$$constant;
3510 if (val >= 0) {
3511 __ subsw(zr, reg, val);
3512 } else {
3513 __ addsw(zr, reg, -val);
3514 }
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3518 Register reg1 = as_Register($src1$$reg);
3519 uint32_t val = (uint32_t)$src2$$constant;
3520 __ movw(rscratch1, val);
3521 __ cmpw(reg1, rscratch1);
3522 %}
3523
3524 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3525 Register reg1 = as_Register($src1$$reg);
3526 Register reg2 = as_Register($src2$$reg);
3527 __ cmp(reg1, reg2);
3528 %}
3529
3530 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3531 Register reg = as_Register($src1$$reg);
3532 int64_t val = $src2$$constant;
3533 if (val >= 0) {
3534 __ subs(zr, reg, val);
3535 } else if (val != -val) {
3536 __ adds(zr, reg, -val);
3537 } else {
3538 // aargh, Long.MIN_VALUE is a special case
3539 __ orr(rscratch1, zr, (uint64_t)val);
3540 __ subs(zr, reg, rscratch1);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3545 Register reg1 = as_Register($src1$$reg);
3546 uint64_t val = (uint64_t)$src2$$constant;
3547 __ mov(rscratch1, val);
3548 __ cmp(reg1, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3552 Register reg1 = as_Register($src1$$reg);
3553 Register reg2 = as_Register($src2$$reg);
3554 __ cmp(reg1, reg2);
3555 %}
3556
3557 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmpw(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_testp(iRegP src) %{
3564 Register reg = as_Register($src$$reg);
3565 __ cmp(reg, zr);
3566 %}
3567
3568 enc_class aarch64_enc_testn(iRegN src) %{
3569 Register reg = as_Register($src$$reg);
3570 __ cmpw(reg, zr);
3571 %}
3572
3573 enc_class aarch64_enc_b(label lbl) %{
3574 Label *L = $lbl$$label;
3575 __ b(*L);
3576 %}
3577
3578 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3579 Label *L = $lbl$$label;
3580 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3581 %}
3582
3583 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3586 %}
3587
3588 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3589 %{
3590 Register sub_reg = as_Register($sub$$reg);
3591 Register super_reg = as_Register($super$$reg);
3592 Register temp_reg = as_Register($temp$$reg);
3593 Register result_reg = as_Register($result$$reg);
3594
3595 Label miss;
3596 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3597 nullptr, &miss,
3598 /*set_cond_codes:*/ true);
3599 if ($primary) {
3600 __ mov(result_reg, zr);
3601 }
3602 __ bind(miss);
3603 %}
3604
3605 enc_class aarch64_enc_java_static_call(method meth) %{
3606 address addr = (address)$meth$$method;
3607 address call;
3608 if (!_method) {
3609 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3610 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3611 if (call == nullptr) {
3612 ciEnv::current()->record_failure("CodeCache is full");
3613 return;
3614 }
3615 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3616 // The NOP here is purely to ensure that eliding a call to
3617 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3618 __ nop();
3619 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3620 } else {
3621 int method_index = resolved_method_index(masm);
3622 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3623 : static_call_Relocation::spec(method_index);
3624 call = __ trampoline_call(Address(addr, rspec));
3625 if (call == nullptr) {
3626 ciEnv::current()->record_failure("CodeCache is full");
3627 return;
3628 }
3629 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3630 // Calls of the same statically bound method can share
3631 // a stub to the interpreter.
3632 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3633 } else {
3634 // Emit stub for static call
3635 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3636 if (stub == nullptr) {
3637 ciEnv::current()->record_failure("CodeCache is full");
3638 return;
3639 }
3640 }
3641 }
3642
3643 __ post_call_nop();
3644
3645 // Only non uncommon_trap calls need to reinitialize ptrue.
3646 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3647 __ reinitialize_ptrue();
3648 }
3649 %}
3650
3651 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3652 int method_index = resolved_method_index(masm);
3653 address call = __ ic_call((address)$meth$$method, method_index);
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 __ post_call_nop();
3659 if (Compile::current()->max_vector_size() > 0) {
3660 __ reinitialize_ptrue();
3661 }
3662 %}
3663
3664 enc_class aarch64_enc_call_epilog() %{
3665 if (VerifyStackAtCalls) {
3666 // Check that stack depth is unchanged: find majik cookie on stack
3667 __ call_Unimplemented();
3668 }
3669 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3670 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3671 // Search for the corresponding projection, get the register and emit code that initialized it.
3672 uint con = (tf()->range_cc()->cnt() - 1);
3673 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3674 ProjNode* proj = fast_out(i)->as_Proj();
3675 if (proj->_con == con) {
3676 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3677 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3678 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3679 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3680 __ cmp(r0, zr);
3681 __ cset(toReg, Assembler::NE);
3682 if (reg->is_stack()) {
3683 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3684 __ str(toReg, Address(sp, st_off));
3685 }
3686 break;
3687 }
3688 }
3689 if (return_value_is_used()) {
3690 // An inline type is returned as fields in multiple registers.
3691 // R0 either contains an oop if the inline type is buffered or a pointer
3692 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3693 // if the lowest bit is set to allow C2 to use the oop after null checking.
3694 // r0 &= (r0 & 1) - 1
3695 __ andr(rscratch1, r0, 0x1);
3696 __ sub(rscratch1, rscratch1, 0x1);
3697 __ andr(r0, r0, rscratch1);
3698 }
3699 }
3700 %}
3701
3702 enc_class aarch64_enc_java_to_runtime(method meth) %{
3703 // some calls to generated routines (arraycopy code) are scheduled
3704 // by C2 as runtime calls. if so we can call them using a br (they
3705 // will be in a reachable segment) otherwise we have to use a blr
3706 // which loads the absolute address into a register.
3707 address entry = (address)$meth$$method;
3708 CodeBlob *cb = CodeCache::find_blob(entry);
3709 if (cb) {
3710 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3711 if (call == nullptr) {
3712 ciEnv::current()->record_failure("CodeCache is full");
3713 return;
3714 }
3715 __ post_call_nop();
3716 } else {
3717 Label retaddr;
3718 __ adr(rscratch2, retaddr);
3719 __ lea(rscratch1, RuntimeAddress(entry));
3720 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3721 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3722 __ blr(rscratch1);
3723 __ bind(retaddr);
3724 __ post_call_nop();
3725 __ add(sp, sp, 2 * wordSize);
3726 }
3727 if (Compile::current()->max_vector_size() > 0) {
3728 __ reinitialize_ptrue();
3729 }
3730 %}
3731
3732 enc_class aarch64_enc_rethrow() %{
3733 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3734 %}
3735
3736 enc_class aarch64_enc_ret() %{
3737 #ifdef ASSERT
3738 if (Compile::current()->max_vector_size() > 0) {
3739 __ verify_ptrue();
3740 }
3741 #endif
3742 __ ret(lr);
3743 %}
3744
3745 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3746 Register target_reg = as_Register($jump_target$$reg);
3747 __ br(target_reg);
3748 %}
3749
3750 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3751 Register target_reg = as_Register($jump_target$$reg);
3752 // exception oop should be in r0
3753 // ret addr has been popped into lr
3754 // callee expects it in r3
3755 __ mov(r3, lr);
3756 __ br(target_reg);
3757 %}
3758
3759 %}
3760
3761 //----------FRAME--------------------------------------------------------------
3762 // Definition of frame structure and management information.
3763 //
3764 // S T A C K L A Y O U T Allocators stack-slot number
3765 // | (to get allocators register number
3766 // G Owned by | | v add OptoReg::stack0())
3767 // r CALLER | |
3768 // o | +--------+ pad to even-align allocators stack-slot
3769 // w V | pad0 | numbers; owned by CALLER
3770 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3771 // h ^ | in | 5
3772 // | | args | 4 Holes in incoming args owned by SELF
3773 // | | | | 3
3774 // | | +--------+
3775 // V | | old out| Empty on Intel, window on Sparc
3776 // | old |preserve| Must be even aligned.
3777 // | SP-+--------+----> Matcher::_old_SP, even aligned
3778 // | | in | 3 area for Intel ret address
3779 // Owned by |preserve| Empty on Sparc.
3780 // SELF +--------+
3781 // | | pad2 | 2 pad to align old SP
3782 // | +--------+ 1
3783 // | | locks | 0
3784 // | +--------+----> OptoReg::stack0(), even aligned
3785 // | | pad1 | 11 pad to align new SP
3786 // | +--------+
3787 // | | | 10
3788 // | | spills | 9 spills
3789 // V | | 8 (pad0 slot for callee)
3790 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3791 // ^ | out | 7
3792 // | | args | 6 Holes in outgoing args owned by CALLEE
3793 // Owned by +--------+
3794 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3795 // | new |preserve| Must be even-aligned.
3796 // | SP-+--------+----> Matcher::_new_SP, even aligned
3797 // | | |
3798 //
3799 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3800 // known from SELF's arguments and the Java calling convention.
3801 // Region 6-7 is determined per call site.
3802 // Note 2: If the calling convention leaves holes in the incoming argument
3803 // area, those holes are owned by SELF. Holes in the outgoing area
3804 // are owned by the CALLEE. Holes should not be necessary in the
3805 // incoming area, as the Java calling convention is completely under
3806 // the control of the AD file. Doubles can be sorted and packed to
3807 // avoid holes. Holes in the outgoing arguments may be necessary for
3808 // varargs C calling conventions.
3809 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3810 // even aligned with pad0 as needed.
3811 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3812 // (the latter is true on Intel but is it false on AArch64?)
3813 // region 6-11 is even aligned; it may be padded out more so that
3814 // the region from SP to FP meets the minimum stack alignment.
3815 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3816 // alignment. Region 11, pad1, may be dynamically extended so that
3817 // SP meets the minimum alignment.
3818
3819 frame %{
3820 // These three registers define part of the calling convention
3821 // between compiled code and the interpreter.
3822
3823 // Inline Cache Register or Method for I2C.
3824 inline_cache_reg(R12);
3825
3826 // Number of stack slots consumed by locking an object
3827 sync_stack_slots(2);
3828
3829 // Compiled code's Frame Pointer
3830 frame_pointer(R31);
3831
3832 // Interpreter stores its frame pointer in a register which is
3833 // stored to the stack by I2CAdaptors.
3834 // I2CAdaptors convert from interpreted java to compiled java.
3835 interpreter_frame_pointer(R29);
3836
3837 // Stack alignment requirement
3838 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3839
3840 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3841 // for calls to C. Supports the var-args backing area for register parms.
3842 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3843
3844 // The after-PROLOG location of the return address. Location of
3845 // return address specifies a type (REG or STACK) and a number
3846 // representing the register number (i.e. - use a register name) or
3847 // stack slot.
3848 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3849 // Otherwise, it is above the locks and verification slot and alignment word
3850 // TODO this may well be correct but need to check why that - 2 is there
3851 // ppc port uses 0 but we definitely need to allow for fixed_slots
3852 // which folds in the space used for monitors
3853 return_addr(STACK - 2 +
3854 align_up((Compile::current()->in_preserve_stack_slots() +
3855 Compile::current()->fixed_slots()),
3856 stack_alignment_in_slots()));
3857
3858 // Location of compiled Java return values. Same as C for now.
3859 return_value
3860 %{
3861 // TODO do we allow ideal_reg == Op_RegN???
3862 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3863 "only return normal values");
3864
3865 static const int lo[Op_RegL + 1] = { // enum name
3866 0, // Op_Node
3867 0, // Op_Set
3868 R0_num, // Op_RegN
3869 R0_num, // Op_RegI
3870 R0_num, // Op_RegP
3871 V0_num, // Op_RegF
3872 V0_num, // Op_RegD
3873 R0_num // Op_RegL
3874 };
3875
3876 static const int hi[Op_RegL + 1] = { // enum name
3877 0, // Op_Node
3878 0, // Op_Set
3879 OptoReg::Bad, // Op_RegN
3880 OptoReg::Bad, // Op_RegI
3881 R0_H_num, // Op_RegP
3882 OptoReg::Bad, // Op_RegF
3883 V0_H_num, // Op_RegD
3884 R0_H_num // Op_RegL
3885 };
3886
3887 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3888 %}
3889 %}
3890
3891 //----------ATTRIBUTES---------------------------------------------------------
3892 //----------Operand Attributes-------------------------------------------------
3893 op_attrib op_cost(1); // Required cost attribute
3894
3895 //----------Instruction Attributes---------------------------------------------
3896 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3897 ins_attrib ins_size(32); // Required size attribute (in bits)
3898 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3899 // a non-matching short branch variant
3900 // of some long branch?
3901 ins_attrib ins_alignment(4); // Required alignment attribute (must
3902 // be a power of 2) specifies the
3903 // alignment that some part of the
3904 // instruction (not necessarily the
3905 // start) requires. If > 1, a
3906 // compute_padding() function must be
3907 // provided for the instruction
3908
3909 //----------OPERANDS-----------------------------------------------------------
3910 // Operand definitions must precede instruction definitions for correct parsing
3911 // in the ADLC because operands constitute user defined types which are used in
3912 // instruction definitions.
3913
3914 //----------Simple Operands----------------------------------------------------
3915
3916 // Integer operands 32 bit
3917 // 32 bit immediate
3918 operand immI()
3919 %{
3920 match(ConI);
3921
3922 op_cost(0);
3923 format %{ %}
3924 interface(CONST_INTER);
3925 %}
3926
3927 // 32 bit zero
3928 operand immI0()
3929 %{
3930 predicate(n->get_int() == 0);
3931 match(ConI);
3932
3933 op_cost(0);
3934 format %{ %}
3935 interface(CONST_INTER);
3936 %}
3937
3938 // 32 bit unit increment
3939 operand immI_1()
3940 %{
3941 predicate(n->get_int() == 1);
3942 match(ConI);
3943
3944 op_cost(0);
3945 format %{ %}
3946 interface(CONST_INTER);
3947 %}
3948
3949 // 32 bit unit decrement
3950 operand immI_M1()
3951 %{
3952 predicate(n->get_int() == -1);
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 // Shift values for add/sub extension shift
3961 operand immIExt()
3962 %{
3963 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3964 match(ConI);
3965
3966 op_cost(0);
3967 format %{ %}
3968 interface(CONST_INTER);
3969 %}
3970
3971 operand immI_gt_1()
3972 %{
3973 predicate(n->get_int() > 1);
3974 match(ConI);
3975
3976 op_cost(0);
3977 format %{ %}
3978 interface(CONST_INTER);
3979 %}
3980
3981 operand immI_le_4()
3982 %{
3983 predicate(n->get_int() <= 4);
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 operand immI_16()
3992 %{
3993 predicate(n->get_int() == 16);
3994 match(ConI);
3995
3996 op_cost(0);
3997 format %{ %}
3998 interface(CONST_INTER);
3999 %}
4000
4001 operand immI_24()
4002 %{
4003 predicate(n->get_int() == 24);
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_32()
4012 %{
4013 predicate(n->get_int() == 32);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_48()
4022 %{
4023 predicate(n->get_int() == 48);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_56()
4032 %{
4033 predicate(n->get_int() == 56);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_255()
4042 %{
4043 predicate(n->get_int() == 255);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immI_65535()
4052 %{
4053 predicate(n->get_int() == 65535);
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immI_positive()
4062 %{
4063 predicate(n->get_int() > 0);
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 // BoolTest condition for signed compare
4072 operand immI_cmp_cond()
4073 %{
4074 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4075 match(ConI);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 // BoolTest condition for unsigned compare
4083 operand immI_cmpU_cond()
4084 %{
4085 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4086 match(ConI);
4087
4088 op_cost(0);
4089 format %{ %}
4090 interface(CONST_INTER);
4091 %}
4092
4093 operand immL_255()
4094 %{
4095 predicate(n->get_long() == 255L);
4096 match(ConL);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 operand immL_65535()
4104 %{
4105 predicate(n->get_long() == 65535L);
4106 match(ConL);
4107
4108 op_cost(0);
4109 format %{ %}
4110 interface(CONST_INTER);
4111 %}
4112
4113 operand immL_4294967295()
4114 %{
4115 predicate(n->get_long() == 4294967295L);
4116 match(ConL);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immL_bitmask()
4124 %{
4125 predicate((n->get_long() != 0)
4126 && ((n->get_long() & 0xc000000000000000l) == 0)
4127 && is_power_of_2(n->get_long() + 1));
4128 match(ConL);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 operand immI_bitmask()
4136 %{
4137 predicate((n->get_int() != 0)
4138 && ((n->get_int() & 0xc0000000) == 0)
4139 && is_power_of_2(n->get_int() + 1));
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 operand immL_positive_bitmaskI()
4148 %{
4149 predicate((n->get_long() != 0)
4150 && ((julong)n->get_long() < 0x80000000ULL)
4151 && is_power_of_2(n->get_long() + 1));
4152 match(ConL);
4153
4154 op_cost(0);
4155 format %{ %}
4156 interface(CONST_INTER);
4157 %}
4158
4159 // Scale values for scaled offset addressing modes (up to long but not quad)
4160 operand immIScale()
4161 %{
4162 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 // 5 bit signed integer
4171 operand immI5()
4172 %{
4173 predicate(Assembler::is_simm(n->get_int(), 5));
4174 match(ConI);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 // 7 bit unsigned integer
4182 operand immIU7()
4183 %{
4184 predicate(Assembler::is_uimm(n->get_int(), 7));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 // Offset for scaled or unscaled immediate loads and stores
4193 operand immIOffset()
4194 %{
4195 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4196 match(ConI);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 operand immIOffset1()
4204 %{
4205 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4206 match(ConI);
4207
4208 op_cost(0);
4209 format %{ %}
4210 interface(CONST_INTER);
4211 %}
4212
4213 operand immIOffset2()
4214 %{
4215 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4216 match(ConI);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 operand immIOffset4()
4224 %{
4225 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4226 match(ConI);
4227
4228 op_cost(0);
4229 format %{ %}
4230 interface(CONST_INTER);
4231 %}
4232
4233 operand immIOffset8()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immIOffset16()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4246 match(ConI);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immLOffset()
4254 %{
4255 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4256 match(ConL);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immLoffset1()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4266 match(ConL);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immLoffset2()
4274 %{
4275 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4276 match(ConL);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immLoffset4()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4286 match(ConL);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 operand immLoffset8()
4294 %{
4295 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 operand immLoffset16()
4304 %{
4305 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 5 bit signed long integer
4314 operand immL5()
4315 %{
4316 predicate(Assembler::is_simm(n->get_long(), 5));
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 7 bit unsigned long integer
4325 operand immLU7()
4326 %{
4327 predicate(Assembler::is_uimm(n->get_long(), 7));
4328 match(ConL);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value.
4336 operand immI8()
4337 %{
4338 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4339 match(ConI);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 // 8 bit signed value (simm8), or #simm8 LSL 8.
4347 operand immI8_shift8()
4348 %{
4349 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4350 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4351 match(ConI);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 // 8 bit signed value (simm8), or #simm8 LSL 8.
4359 operand immL8_shift8()
4360 %{
4361 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4362 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4363 match(ConL);
4364
4365 op_cost(0);
4366 format %{ %}
4367 interface(CONST_INTER);
4368 %}
4369
4370 // 8 bit integer valid for vector add sub immediate
4371 operand immBAddSubV()
4372 %{
4373 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4374 match(ConI);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 32 bit integer valid for add sub immediate
4382 operand immIAddSub()
4383 %{
4384 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4385 match(ConI);
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 // 32 bit integer valid for vector add sub immediate
4392 operand immIAddSubV()
4393 %{
4394 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4395 match(ConI);
4396
4397 op_cost(0);
4398 format %{ %}
4399 interface(CONST_INTER);
4400 %}
4401
4402 // 32 bit unsigned integer valid for logical immediate
4403
4404 operand immBLog()
4405 %{
4406 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 operand immSLog()
4415 %{
4416 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4417 match(ConI);
4418
4419 op_cost(0);
4420 format %{ %}
4421 interface(CONST_INTER);
4422 %}
4423
4424 operand immILog()
4425 %{
4426 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4427 match(ConI);
4428
4429 op_cost(0);
4430 format %{ %}
4431 interface(CONST_INTER);
4432 %}
4433
4434 // Integer operands 64 bit
4435 // 64 bit immediate
4436 operand immL()
4437 %{
4438 match(ConL);
4439
4440 op_cost(0);
4441 format %{ %}
4442 interface(CONST_INTER);
4443 %}
4444
4445 // 64 bit zero
4446 operand immL0()
4447 %{
4448 predicate(n->get_long() == 0);
4449 match(ConL);
4450
4451 op_cost(0);
4452 format %{ %}
4453 interface(CONST_INTER);
4454 %}
4455
4456 // 64 bit unit decrement
4457 operand immL_M1()
4458 %{
4459 predicate(n->get_long() == -1);
4460 match(ConL);
4461
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // 64 bit integer valid for add sub immediate
4468 operand immLAddSub()
4469 %{
4470 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4471 match(ConL);
4472 op_cost(0);
4473 format %{ %}
4474 interface(CONST_INTER);
4475 %}
4476
4477 // 64 bit integer valid for addv subv immediate
4478 operand immLAddSubV()
4479 %{
4480 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4481 match(ConL);
4482
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // 64 bit integer valid for logical immediate
4489 operand immLLog()
4490 %{
4491 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4492 match(ConL);
4493 op_cost(0);
4494 format %{ %}
4495 interface(CONST_INTER);
4496 %}
4497
4498 // Long Immediate: low 32-bit mask
4499 operand immL_32bits()
4500 %{
4501 predicate(n->get_long() == 0xFFFFFFFFL);
4502 match(ConL);
4503 op_cost(0);
4504 format %{ %}
4505 interface(CONST_INTER);
4506 %}
4507
4508 // Pointer operands
4509 // Pointer Immediate
4510 operand immP()
4511 %{
4512 match(ConP);
4513
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 // nullptr Pointer Immediate
4520 operand immP0()
4521 %{
4522 predicate(n->get_ptr() == 0);
4523 match(ConP);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 // Pointer Immediate One
4531 // this is used in object initialization (initial object header)
4532 operand immP_1()
4533 %{
4534 predicate(n->get_ptr() == 1);
4535 match(ConP);
4536
4537 op_cost(0);
4538 format %{ %}
4539 interface(CONST_INTER);
4540 %}
4541
4542 // Card Table Byte Map Base
4543 operand immByteMapBase()
4544 %{
4545 // Get base of card map
4546 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4547 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4548 match(ConP);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // Float and Double operands
4556 // Double Immediate
4557 operand immD()
4558 %{
4559 match(ConD);
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Double Immediate: +0.0d
4566 operand immD0()
4567 %{
4568 predicate(jlong_cast(n->getd()) == 0);
4569 match(ConD);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // constant 'double +0.0'.
4577 operand immDPacked()
4578 %{
4579 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4580 match(ConD);
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // Float Immediate
4587 operand immF()
4588 %{
4589 match(ConF);
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 // Float Immediate: +0.0f.
4596 operand immF0()
4597 %{
4598 predicate(jint_cast(n->getf()) == 0);
4599 match(ConF);
4600
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 //
4607 operand immFPacked()
4608 %{
4609 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4610 match(ConF);
4611 op_cost(0);
4612 format %{ %}
4613 interface(CONST_INTER);
4614 %}
4615
4616 // Narrow pointer operands
4617 // Narrow Pointer Immediate
4618 operand immN()
4619 %{
4620 match(ConN);
4621
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Narrow nullptr Pointer Immediate
4628 operand immN0()
4629 %{
4630 predicate(n->get_narrowcon() == 0);
4631 match(ConN);
4632
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 operand immNKlass()
4639 %{
4640 match(ConNKlass);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // Integer 32 bit Register Operands
4648 // Integer 32 bitRegister (excludes SP)
4649 operand iRegI()
4650 %{
4651 constraint(ALLOC_IN_RC(any_reg32));
4652 match(RegI);
4653 match(iRegINoSp);
4654 op_cost(0);
4655 format %{ %}
4656 interface(REG_INTER);
4657 %}
4658
4659 // Integer 32 bit Register not Special
4660 operand iRegINoSp()
4661 %{
4662 constraint(ALLOC_IN_RC(no_special_reg32));
4663 match(RegI);
4664 op_cost(0);
4665 format %{ %}
4666 interface(REG_INTER);
4667 %}
4668
4669 // Integer 64 bit Register Operands
4670 // Integer 64 bit Register (includes SP)
4671 operand iRegL()
4672 %{
4673 constraint(ALLOC_IN_RC(any_reg));
4674 match(RegL);
4675 match(iRegLNoSp);
4676 op_cost(0);
4677 format %{ %}
4678 interface(REG_INTER);
4679 %}
4680
4681 // Integer 64 bit Register not Special
4682 operand iRegLNoSp()
4683 %{
4684 constraint(ALLOC_IN_RC(no_special_reg));
4685 match(RegL);
4686 match(iRegL_R0);
4687 format %{ %}
4688 interface(REG_INTER);
4689 %}
4690
4691 // Pointer Register Operands
4692 // Pointer Register
4693 operand iRegP()
4694 %{
4695 constraint(ALLOC_IN_RC(ptr_reg));
4696 match(RegP);
4697 match(iRegPNoSp);
4698 match(iRegP_R0);
4699 //match(iRegP_R2);
4700 //match(iRegP_R4);
4701 match(iRegP_R5);
4702 match(thread_RegP);
4703 op_cost(0);
4704 format %{ %}
4705 interface(REG_INTER);
4706 %}
4707
4708 // Pointer 64 bit Register not Special
4709 operand iRegPNoSp()
4710 %{
4711 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4712 match(RegP);
4713 // match(iRegP);
4714 // match(iRegP_R0);
4715 // match(iRegP_R2);
4716 // match(iRegP_R4);
4717 // match(iRegP_R5);
4718 // match(thread_RegP);
4719 op_cost(0);
4720 format %{ %}
4721 interface(REG_INTER);
4722 %}
4723
4724 // This operand is not allowed to use rfp even if
4725 // rfp is not used to hold the frame pointer.
4726 operand iRegPNoSpNoRfp()
4727 %{
4728 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4729 match(RegP);
4730 match(iRegPNoSp);
4731 op_cost(0);
4732 format %{ %}
4733 interface(REG_INTER);
4734 %}
4735
4736 // Pointer 64 bit Register R0 only
4737 operand iRegP_R0()
4738 %{
4739 constraint(ALLOC_IN_RC(r0_reg));
4740 match(RegP);
4741 // match(iRegP);
4742 match(iRegPNoSp);
4743 op_cost(0);
4744 format %{ %}
4745 interface(REG_INTER);
4746 %}
4747
4748 // Pointer 64 bit Register R1 only
4749 operand iRegP_R1()
4750 %{
4751 constraint(ALLOC_IN_RC(r1_reg));
4752 match(RegP);
4753 // match(iRegP);
4754 match(iRegPNoSp);
4755 op_cost(0);
4756 format %{ %}
4757 interface(REG_INTER);
4758 %}
4759
4760 // Pointer 64 bit Register R2 only
4761 operand iRegP_R2()
4762 %{
4763 constraint(ALLOC_IN_RC(r2_reg));
4764 match(RegP);
4765 // match(iRegP);
4766 match(iRegPNoSp);
4767 op_cost(0);
4768 format %{ %}
4769 interface(REG_INTER);
4770 %}
4771
4772 // Pointer 64 bit Register R3 only
4773 operand iRegP_R3()
4774 %{
4775 constraint(ALLOC_IN_RC(r3_reg));
4776 match(RegP);
4777 // match(iRegP);
4778 match(iRegPNoSp);
4779 op_cost(0);
4780 format %{ %}
4781 interface(REG_INTER);
4782 %}
4783
4784 // Pointer 64 bit Register R4 only
4785 operand iRegP_R4()
4786 %{
4787 constraint(ALLOC_IN_RC(r4_reg));
4788 match(RegP);
4789 // match(iRegP);
4790 match(iRegPNoSp);
4791 op_cost(0);
4792 format %{ %}
4793 interface(REG_INTER);
4794 %}
4795
4796 // Pointer 64 bit Register R5 only
4797 operand iRegP_R5()
4798 %{
4799 constraint(ALLOC_IN_RC(r5_reg));
4800 match(RegP);
4801 // match(iRegP);
4802 match(iRegPNoSp);
4803 op_cost(0);
4804 format %{ %}
4805 interface(REG_INTER);
4806 %}
4807
4808 // Pointer 64 bit Register R10 only
4809 operand iRegP_R10()
4810 %{
4811 constraint(ALLOC_IN_RC(r10_reg));
4812 match(RegP);
4813 // match(iRegP);
4814 match(iRegPNoSp);
4815 op_cost(0);
4816 format %{ %}
4817 interface(REG_INTER);
4818 %}
4819
4820 // Long 64 bit Register R0 only
4821 operand iRegL_R0()
4822 %{
4823 constraint(ALLOC_IN_RC(r0_reg));
4824 match(RegL);
4825 match(iRegLNoSp);
4826 op_cost(0);
4827 format %{ %}
4828 interface(REG_INTER);
4829 %}
4830
4831 // Long 64 bit Register R11 only
4832 operand iRegL_R11()
4833 %{
4834 constraint(ALLOC_IN_RC(r11_reg));
4835 match(RegL);
4836 match(iRegLNoSp);
4837 op_cost(0);
4838 format %{ %}
4839 interface(REG_INTER);
4840 %}
4841
4842 // Register R0 only
4843 operand iRegI_R0()
4844 %{
4845 constraint(ALLOC_IN_RC(int_r0_reg));
4846 match(RegI);
4847 match(iRegINoSp);
4848 op_cost(0);
4849 format %{ %}
4850 interface(REG_INTER);
4851 %}
4852
4853 // Register R2 only
4854 operand iRegI_R2()
4855 %{
4856 constraint(ALLOC_IN_RC(int_r2_reg));
4857 match(RegI);
4858 match(iRegINoSp);
4859 op_cost(0);
4860 format %{ %}
4861 interface(REG_INTER);
4862 %}
4863
4864 // Register R3 only
4865 operand iRegI_R3()
4866 %{
4867 constraint(ALLOC_IN_RC(int_r3_reg));
4868 match(RegI);
4869 match(iRegINoSp);
4870 op_cost(0);
4871 format %{ %}
4872 interface(REG_INTER);
4873 %}
4874
4875
4876 // Register R4 only
4877 operand iRegI_R4()
4878 %{
4879 constraint(ALLOC_IN_RC(int_r4_reg));
4880 match(RegI);
4881 match(iRegINoSp);
4882 op_cost(0);
4883 format %{ %}
4884 interface(REG_INTER);
4885 %}
4886
4887
4888 // Pointer Register Operands
4889 // Narrow Pointer Register
4890 operand iRegN()
4891 %{
4892 constraint(ALLOC_IN_RC(any_reg32));
4893 match(RegN);
4894 match(iRegNNoSp);
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Integer 64 bit Register not Special
4901 operand iRegNNoSp()
4902 %{
4903 constraint(ALLOC_IN_RC(no_special_reg32));
4904 match(RegN);
4905 op_cost(0);
4906 format %{ %}
4907 interface(REG_INTER);
4908 %}
4909
4910 // Float Register
4911 // Float register operands
4912 operand vRegF()
4913 %{
4914 constraint(ALLOC_IN_RC(float_reg));
4915 match(RegF);
4916
4917 op_cost(0);
4918 format %{ %}
4919 interface(REG_INTER);
4920 %}
4921
4922 // Double Register
4923 // Double register operands
4924 operand vRegD()
4925 %{
4926 constraint(ALLOC_IN_RC(double_reg));
4927 match(RegD);
4928
4929 op_cost(0);
4930 format %{ %}
4931 interface(REG_INTER);
4932 %}
4933
4934 // Generic vector class. This will be used for
4935 // all vector operands, including NEON and SVE.
4936 operand vReg()
4937 %{
4938 constraint(ALLOC_IN_RC(dynamic));
4939 match(VecA);
4940 match(VecD);
4941 match(VecX);
4942
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 operand vecA()
4949 %{
4950 constraint(ALLOC_IN_RC(vectora_reg));
4951 match(VecA);
4952
4953 op_cost(0);
4954 format %{ %}
4955 interface(REG_INTER);
4956 %}
4957
4958 operand vecD()
4959 %{
4960 constraint(ALLOC_IN_RC(vectord_reg));
4961 match(VecD);
4962
4963 op_cost(0);
4964 format %{ %}
4965 interface(REG_INTER);
4966 %}
4967
4968 operand vecX()
4969 %{
4970 constraint(ALLOC_IN_RC(vectorx_reg));
4971 match(VecX);
4972
4973 op_cost(0);
4974 format %{ %}
4975 interface(REG_INTER);
4976 %}
4977
4978 operand vRegD_V0()
4979 %{
4980 constraint(ALLOC_IN_RC(v0_reg));
4981 match(RegD);
4982 op_cost(0);
4983 format %{ %}
4984 interface(REG_INTER);
4985 %}
4986
4987 operand vRegD_V1()
4988 %{
4989 constraint(ALLOC_IN_RC(v1_reg));
4990 match(RegD);
4991 op_cost(0);
4992 format %{ %}
4993 interface(REG_INTER);
4994 %}
4995
4996 operand vRegD_V2()
4997 %{
4998 constraint(ALLOC_IN_RC(v2_reg));
4999 match(RegD);
5000 op_cost(0);
5001 format %{ %}
5002 interface(REG_INTER);
5003 %}
5004
5005 operand vRegD_V3()
5006 %{
5007 constraint(ALLOC_IN_RC(v3_reg));
5008 match(RegD);
5009 op_cost(0);
5010 format %{ %}
5011 interface(REG_INTER);
5012 %}
5013
5014 operand vRegD_V4()
5015 %{
5016 constraint(ALLOC_IN_RC(v4_reg));
5017 match(RegD);
5018 op_cost(0);
5019 format %{ %}
5020 interface(REG_INTER);
5021 %}
5022
5023 operand vRegD_V5()
5024 %{
5025 constraint(ALLOC_IN_RC(v5_reg));
5026 match(RegD);
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vRegD_V6()
5033 %{
5034 constraint(ALLOC_IN_RC(v6_reg));
5035 match(RegD);
5036 op_cost(0);
5037 format %{ %}
5038 interface(REG_INTER);
5039 %}
5040
5041 operand vRegD_V7()
5042 %{
5043 constraint(ALLOC_IN_RC(v7_reg));
5044 match(RegD);
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 operand pReg()
5051 %{
5052 constraint(ALLOC_IN_RC(pr_reg));
5053 match(RegVectMask);
5054 match(pRegGov);
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand pRegGov()
5061 %{
5062 constraint(ALLOC_IN_RC(gov_pr));
5063 match(RegVectMask);
5064 match(pReg);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand pRegGov_P0()
5071 %{
5072 constraint(ALLOC_IN_RC(p0_reg));
5073 match(RegVectMask);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand pRegGov_P1()
5080 %{
5081 constraint(ALLOC_IN_RC(p1_reg));
5082 match(RegVectMask);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 // Flags register, used as output of signed compare instructions
5089
5090 // note that on AArch64 we also use this register as the output for
5091 // for floating point compare instructions (CmpF CmpD). this ensures
5092 // that ordered inequality tests use GT, GE, LT or LE none of which
5093 // pass through cases where the result is unordered i.e. one or both
5094 // inputs to the compare is a NaN. this means that the ideal code can
5095 // replace e.g. a GT with an LE and not end up capturing the NaN case
5096 // (where the comparison should always fail). EQ and NE tests are
5097 // always generated in ideal code so that unordered folds into the NE
5098 // case, matching the behaviour of AArch64 NE.
5099 //
5100 // This differs from x86 where the outputs of FP compares use a
5101 // special FP flags registers and where compares based on this
5102 // register are distinguished into ordered inequalities (cmpOpUCF) and
5103 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5104 // to explicitly handle the unordered case in branches. x86 also has
5105 // to include extra CMoveX rules to accept a cmpOpUCF input.
5106
5107 operand rFlagsReg()
5108 %{
5109 constraint(ALLOC_IN_RC(int_flags));
5110 match(RegFlags);
5111
5112 op_cost(0);
5113 format %{ "RFLAGS" %}
5114 interface(REG_INTER);
5115 %}
5116
5117 // Flags register, used as output of unsigned compare instructions
5118 operand rFlagsRegU()
5119 %{
5120 constraint(ALLOC_IN_RC(int_flags));
5121 match(RegFlags);
5122
5123 op_cost(0);
5124 format %{ "RFLAGSU" %}
5125 interface(REG_INTER);
5126 %}
5127
5128 // Special Registers
5129
5130 // Method Register
5131 operand inline_cache_RegP(iRegP reg)
5132 %{
5133 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5134 match(reg);
5135 match(iRegPNoSp);
5136 op_cost(0);
5137 format %{ %}
5138 interface(REG_INTER);
5139 %}
5140
5141 // Thread Register
5142 operand thread_RegP(iRegP reg)
5143 %{
5144 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5145 match(reg);
5146 op_cost(0);
5147 format %{ %}
5148 interface(REG_INTER);
5149 %}
5150
5151 //----------Memory Operands----------------------------------------------------
5152
5153 operand indirect(iRegP reg)
5154 %{
5155 constraint(ALLOC_IN_RC(ptr_reg));
5156 match(reg);
5157 op_cost(0);
5158 format %{ "[$reg]" %}
5159 interface(MEMORY_INTER) %{
5160 base($reg);
5161 index(0xffffffff);
5162 scale(0x0);
5163 disp(0x0);
5164 %}
5165 %}
5166
5167 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5168 %{
5169 constraint(ALLOC_IN_RC(ptr_reg));
5170 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5171 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5172 op_cost(0);
5173 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5174 interface(MEMORY_INTER) %{
5175 base($reg);
5176 index($ireg);
5177 scale($scale);
5178 disp(0x0);
5179 %}
5180 %}
5181
5182 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5183 %{
5184 constraint(ALLOC_IN_RC(ptr_reg));
5185 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5186 match(AddP reg (LShiftL lreg scale));
5187 op_cost(0);
5188 format %{ "$reg, $lreg lsl($scale)" %}
5189 interface(MEMORY_INTER) %{
5190 base($reg);
5191 index($lreg);
5192 scale($scale);
5193 disp(0x0);
5194 %}
5195 %}
5196
5197 operand indIndexI2L(iRegP reg, iRegI ireg)
5198 %{
5199 constraint(ALLOC_IN_RC(ptr_reg));
5200 match(AddP reg (ConvI2L ireg));
5201 op_cost(0);
5202 format %{ "$reg, $ireg, 0, I2L" %}
5203 interface(MEMORY_INTER) %{
5204 base($reg);
5205 index($ireg);
5206 scale(0x0);
5207 disp(0x0);
5208 %}
5209 %}
5210
5211 operand indIndex(iRegP reg, iRegL lreg)
5212 %{
5213 constraint(ALLOC_IN_RC(ptr_reg));
5214 match(AddP reg lreg);
5215 op_cost(0);
5216 format %{ "$reg, $lreg" %}
5217 interface(MEMORY_INTER) %{
5218 base($reg);
5219 index($lreg);
5220 scale(0x0);
5221 disp(0x0);
5222 %}
5223 %}
5224
5225 operand indOffI1(iRegP reg, immIOffset1 off)
5226 %{
5227 constraint(ALLOC_IN_RC(ptr_reg));
5228 match(AddP reg off);
5229 op_cost(0);
5230 format %{ "[$reg, $off]" %}
5231 interface(MEMORY_INTER) %{
5232 base($reg);
5233 index(0xffffffff);
5234 scale(0x0);
5235 disp($off);
5236 %}
5237 %}
5238
5239 operand indOffI2(iRegP reg, immIOffset2 off)
5240 %{
5241 constraint(ALLOC_IN_RC(ptr_reg));
5242 match(AddP reg off);
5243 op_cost(0);
5244 format %{ "[$reg, $off]" %}
5245 interface(MEMORY_INTER) %{
5246 base($reg);
5247 index(0xffffffff);
5248 scale(0x0);
5249 disp($off);
5250 %}
5251 %}
5252
5253 operand indOffI4(iRegP reg, immIOffset4 off)
5254 %{
5255 constraint(ALLOC_IN_RC(ptr_reg));
5256 match(AddP reg off);
5257 op_cost(0);
5258 format %{ "[$reg, $off]" %}
5259 interface(MEMORY_INTER) %{
5260 base($reg);
5261 index(0xffffffff);
5262 scale(0x0);
5263 disp($off);
5264 %}
5265 %}
5266
5267 operand indOffI8(iRegP reg, immIOffset8 off)
5268 %{
5269 constraint(ALLOC_IN_RC(ptr_reg));
5270 match(AddP reg off);
5271 op_cost(0);
5272 format %{ "[$reg, $off]" %}
5273 interface(MEMORY_INTER) %{
5274 base($reg);
5275 index(0xffffffff);
5276 scale(0x0);
5277 disp($off);
5278 %}
5279 %}
5280
5281 operand indOffI16(iRegP reg, immIOffset16 off)
5282 %{
5283 constraint(ALLOC_IN_RC(ptr_reg));
5284 match(AddP reg off);
5285 op_cost(0);
5286 format %{ "[$reg, $off]" %}
5287 interface(MEMORY_INTER) %{
5288 base($reg);
5289 index(0xffffffff);
5290 scale(0x0);
5291 disp($off);
5292 %}
5293 %}
5294
5295 operand indOffL1(iRegP reg, immLoffset1 off)
5296 %{
5297 constraint(ALLOC_IN_RC(ptr_reg));
5298 match(AddP reg off);
5299 op_cost(0);
5300 format %{ "[$reg, $off]" %}
5301 interface(MEMORY_INTER) %{
5302 base($reg);
5303 index(0xffffffff);
5304 scale(0x0);
5305 disp($off);
5306 %}
5307 %}
5308
5309 operand indOffL2(iRegP reg, immLoffset2 off)
5310 %{
5311 constraint(ALLOC_IN_RC(ptr_reg));
5312 match(AddP reg off);
5313 op_cost(0);
5314 format %{ "[$reg, $off]" %}
5315 interface(MEMORY_INTER) %{
5316 base($reg);
5317 index(0xffffffff);
5318 scale(0x0);
5319 disp($off);
5320 %}
5321 %}
5322
5323 operand indOffL4(iRegP reg, immLoffset4 off)
5324 %{
5325 constraint(ALLOC_IN_RC(ptr_reg));
5326 match(AddP reg off);
5327 op_cost(0);
5328 format %{ "[$reg, $off]" %}
5329 interface(MEMORY_INTER) %{
5330 base($reg);
5331 index(0xffffffff);
5332 scale(0x0);
5333 disp($off);
5334 %}
5335 %}
5336
5337 operand indOffL8(iRegP reg, immLoffset8 off)
5338 %{
5339 constraint(ALLOC_IN_RC(ptr_reg));
5340 match(AddP reg off);
5341 op_cost(0);
5342 format %{ "[$reg, $off]" %}
5343 interface(MEMORY_INTER) %{
5344 base($reg);
5345 index(0xffffffff);
5346 scale(0x0);
5347 disp($off);
5348 %}
5349 %}
5350
5351 operand indOffL16(iRegP reg, immLoffset16 off)
5352 %{
5353 constraint(ALLOC_IN_RC(ptr_reg));
5354 match(AddP reg off);
5355 op_cost(0);
5356 format %{ "[$reg, $off]" %}
5357 interface(MEMORY_INTER) %{
5358 base($reg);
5359 index(0xffffffff);
5360 scale(0x0);
5361 disp($off);
5362 %}
5363 %}
5364
5365 operand indirectX2P(iRegL reg)
5366 %{
5367 constraint(ALLOC_IN_RC(ptr_reg));
5368 match(CastX2P reg);
5369 op_cost(0);
5370 format %{ "[$reg]\t# long -> ptr" %}
5371 interface(MEMORY_INTER) %{
5372 base($reg);
5373 index(0xffffffff);
5374 scale(0x0);
5375 disp(0x0);
5376 %}
5377 %}
5378
5379 operand indOffX2P(iRegL reg, immLOffset off)
5380 %{
5381 constraint(ALLOC_IN_RC(ptr_reg));
5382 match(AddP (CastX2P reg) off);
5383 op_cost(0);
5384 format %{ "[$reg, $off]\t# long -> ptr" %}
5385 interface(MEMORY_INTER) %{
5386 base($reg);
5387 index(0xffffffff);
5388 scale(0x0);
5389 disp($off);
5390 %}
5391 %}
5392
5393 operand indirectN(iRegN reg)
5394 %{
5395 predicate(CompressedOops::shift() == 0);
5396 constraint(ALLOC_IN_RC(ptr_reg));
5397 match(DecodeN reg);
5398 op_cost(0);
5399 format %{ "[$reg]\t# narrow" %}
5400 interface(MEMORY_INTER) %{
5401 base($reg);
5402 index(0xffffffff);
5403 scale(0x0);
5404 disp(0x0);
5405 %}
5406 %}
5407
5408 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5409 %{
5410 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5413 op_cost(0);
5414 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index($ireg);
5418 scale($scale);
5419 disp(0x0);
5420 %}
5421 %}
5422
5423 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5424 %{
5425 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5426 constraint(ALLOC_IN_RC(ptr_reg));
5427 match(AddP (DecodeN reg) (LShiftL lreg scale));
5428 op_cost(0);
5429 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5430 interface(MEMORY_INTER) %{
5431 base($reg);
5432 index($lreg);
5433 scale($scale);
5434 disp(0x0);
5435 %}
5436 %}
5437
5438 operand indIndexI2LN(iRegN reg, iRegI ireg)
5439 %{
5440 predicate(CompressedOops::shift() == 0);
5441 constraint(ALLOC_IN_RC(ptr_reg));
5442 match(AddP (DecodeN reg) (ConvI2L ireg));
5443 op_cost(0);
5444 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5445 interface(MEMORY_INTER) %{
5446 base($reg);
5447 index($ireg);
5448 scale(0x0);
5449 disp(0x0);
5450 %}
5451 %}
5452
5453 operand indIndexN(iRegN reg, iRegL lreg)
5454 %{
5455 predicate(CompressedOops::shift() == 0);
5456 constraint(ALLOC_IN_RC(ptr_reg));
5457 match(AddP (DecodeN reg) lreg);
5458 op_cost(0);
5459 format %{ "$reg, $lreg\t# narrow" %}
5460 interface(MEMORY_INTER) %{
5461 base($reg);
5462 index($lreg);
5463 scale(0x0);
5464 disp(0x0);
5465 %}
5466 %}
5467
5468 operand indOffIN(iRegN reg, immIOffset off)
5469 %{
5470 predicate(CompressedOops::shift() == 0);
5471 constraint(ALLOC_IN_RC(ptr_reg));
5472 match(AddP (DecodeN reg) off);
5473 op_cost(0);
5474 format %{ "[$reg, $off]\t# narrow" %}
5475 interface(MEMORY_INTER) %{
5476 base($reg);
5477 index(0xffffffff);
5478 scale(0x0);
5479 disp($off);
5480 %}
5481 %}
5482
5483 operand indOffLN(iRegN reg, immLOffset off)
5484 %{
5485 predicate(CompressedOops::shift() == 0);
5486 constraint(ALLOC_IN_RC(ptr_reg));
5487 match(AddP (DecodeN reg) off);
5488 op_cost(0);
5489 format %{ "[$reg, $off]\t# narrow" %}
5490 interface(MEMORY_INTER) %{
5491 base($reg);
5492 index(0xffffffff);
5493 scale(0x0);
5494 disp($off);
5495 %}
5496 %}
5497
5498
5499 //----------Special Memory Operands--------------------------------------------
5500 // Stack Slot Operand - This operand is used for loading and storing temporary
5501 // values on the stack where a match requires a value to
5502 // flow through memory.
5503 operand stackSlotP(sRegP reg)
5504 %{
5505 constraint(ALLOC_IN_RC(stack_slots));
5506 op_cost(100);
5507 // No match rule because this operand is only generated in matching
5508 // match(RegP);
5509 format %{ "[$reg]" %}
5510 interface(MEMORY_INTER) %{
5511 base(0x1e); // RSP
5512 index(0x0); // No Index
5513 scale(0x0); // No Scale
5514 disp($reg); // Stack Offset
5515 %}
5516 %}
5517
5518 operand stackSlotI(sRegI reg)
5519 %{
5520 constraint(ALLOC_IN_RC(stack_slots));
5521 // No match rule because this operand is only generated in matching
5522 // match(RegI);
5523 format %{ "[$reg]" %}
5524 interface(MEMORY_INTER) %{
5525 base(0x1e); // RSP
5526 index(0x0); // No Index
5527 scale(0x0); // No Scale
5528 disp($reg); // Stack Offset
5529 %}
5530 %}
5531
5532 operand stackSlotF(sRegF reg)
5533 %{
5534 constraint(ALLOC_IN_RC(stack_slots));
5535 // No match rule because this operand is only generated in matching
5536 // match(RegF);
5537 format %{ "[$reg]" %}
5538 interface(MEMORY_INTER) %{
5539 base(0x1e); // RSP
5540 index(0x0); // No Index
5541 scale(0x0); // No Scale
5542 disp($reg); // Stack Offset
5543 %}
5544 %}
5545
5546 operand stackSlotD(sRegD reg)
5547 %{
5548 constraint(ALLOC_IN_RC(stack_slots));
5549 // No match rule because this operand is only generated in matching
5550 // match(RegD);
5551 format %{ "[$reg]" %}
5552 interface(MEMORY_INTER) %{
5553 base(0x1e); // RSP
5554 index(0x0); // No Index
5555 scale(0x0); // No Scale
5556 disp($reg); // Stack Offset
5557 %}
5558 %}
5559
5560 operand stackSlotL(sRegL reg)
5561 %{
5562 constraint(ALLOC_IN_RC(stack_slots));
5563 // No match rule because this operand is only generated in matching
5564 // match(RegL);
5565 format %{ "[$reg]" %}
5566 interface(MEMORY_INTER) %{
5567 base(0x1e); // RSP
5568 index(0x0); // No Index
5569 scale(0x0); // No Scale
5570 disp($reg); // Stack Offset
5571 %}
5572 %}
5573
5574 // Operands for expressing Control Flow
5575 // NOTE: Label is a predefined operand which should not be redefined in
5576 // the AD file. It is generically handled within the ADLC.
5577
5578 //----------Conditional Branch Operands----------------------------------------
5579 // Comparison Op - This is the operation of the comparison, and is limited to
5580 // the following set of codes:
5581 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5582 //
5583 // Other attributes of the comparison, such as unsignedness, are specified
5584 // by the comparison instruction that sets a condition code flags register.
5585 // That result is represented by a flags operand whose subtype is appropriate
5586 // to the unsignedness (etc.) of the comparison.
5587 //
5588 // Later, the instruction which matches both the Comparison Op (a Bool) and
5589 // the flags (produced by the Cmp) specifies the coding of the comparison op
5590 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5591
5592 // used for signed integral comparisons and fp comparisons
5593
5594 operand cmpOp()
5595 %{
5596 match(Bool);
5597
5598 format %{ "" %}
5599 interface(COND_INTER) %{
5600 equal(0x0, "eq");
5601 not_equal(0x1, "ne");
5602 less(0xb, "lt");
5603 greater_equal(0xa, "ge");
5604 less_equal(0xd, "le");
5605 greater(0xc, "gt");
5606 overflow(0x6, "vs");
5607 no_overflow(0x7, "vc");
5608 %}
5609 %}
5610
5611 // used for unsigned integral comparisons
5612
5613 operand cmpOpU()
5614 %{
5615 match(Bool);
5616
5617 format %{ "" %}
5618 interface(COND_INTER) %{
5619 equal(0x0, "eq");
5620 not_equal(0x1, "ne");
5621 less(0x3, "lo");
5622 greater_equal(0x2, "hs");
5623 less_equal(0x9, "ls");
5624 greater(0x8, "hi");
5625 overflow(0x6, "vs");
5626 no_overflow(0x7, "vc");
5627 %}
5628 %}
5629
5630 // used for certain integral comparisons which can be
5631 // converted to cbxx or tbxx instructions
5632
5633 operand cmpOpEqNe()
5634 %{
5635 match(Bool);
5636 op_cost(0);
5637 predicate(n->as_Bool()->_test._test == BoolTest::ne
5638 || n->as_Bool()->_test._test == BoolTest::eq);
5639
5640 format %{ "" %}
5641 interface(COND_INTER) %{
5642 equal(0x0, "eq");
5643 not_equal(0x1, "ne");
5644 less(0xb, "lt");
5645 greater_equal(0xa, "ge");
5646 less_equal(0xd, "le");
5647 greater(0xc, "gt");
5648 overflow(0x6, "vs");
5649 no_overflow(0x7, "vc");
5650 %}
5651 %}
5652
5653 // used for certain integral comparisons which can be
5654 // converted to cbxx or tbxx instructions
5655
5656 operand cmpOpLtGe()
5657 %{
5658 match(Bool);
5659 op_cost(0);
5660
5661 predicate(n->as_Bool()->_test._test == BoolTest::lt
5662 || n->as_Bool()->_test._test == BoolTest::ge);
5663
5664 format %{ "" %}
5665 interface(COND_INTER) %{
5666 equal(0x0, "eq");
5667 not_equal(0x1, "ne");
5668 less(0xb, "lt");
5669 greater_equal(0xa, "ge");
5670 less_equal(0xd, "le");
5671 greater(0xc, "gt");
5672 overflow(0x6, "vs");
5673 no_overflow(0x7, "vc");
5674 %}
5675 %}
5676
5677 // used for certain unsigned integral comparisons which can be
5678 // converted to cbxx or tbxx instructions
5679
5680 operand cmpOpUEqNeLeGt()
5681 %{
5682 match(Bool);
5683 op_cost(0);
5684
5685 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5686 n->as_Bool()->_test._test == BoolTest::ne ||
5687 n->as_Bool()->_test._test == BoolTest::le ||
5688 n->as_Bool()->_test._test == BoolTest::gt);
5689
5690 format %{ "" %}
5691 interface(COND_INTER) %{
5692 equal(0x0, "eq");
5693 not_equal(0x1, "ne");
5694 less(0x3, "lo");
5695 greater_equal(0x2, "hs");
5696 less_equal(0x9, "ls");
5697 greater(0x8, "hi");
5698 overflow(0x6, "vs");
5699 no_overflow(0x7, "vc");
5700 %}
5701 %}
5702
5703 // Special operand allowing long args to int ops to be truncated for free
5704
5705 operand iRegL2I(iRegL reg) %{
5706
5707 op_cost(0);
5708
5709 match(ConvL2I reg);
5710
5711 format %{ "l2i($reg)" %}
5712
5713 interface(REG_INTER)
5714 %}
5715
5716 operand iRegL2P(iRegL reg) %{
5717
5718 op_cost(0);
5719
5720 match(CastX2P reg);
5721
5722 format %{ "l2p($reg)" %}
5723
5724 interface(REG_INTER)
5725 %}
5726
5727 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5728 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5729 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5730 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5731
5732 //----------OPERAND CLASSES----------------------------------------------------
5733 // Operand Classes are groups of operands that are used as to simplify
5734 // instruction definitions by not requiring the AD writer to specify
5735 // separate instructions for every form of operand when the
5736 // instruction accepts multiple operand types with the same basic
5737 // encoding and format. The classic case of this is memory operands.
5738
5739 // memory is used to define read/write location for load/store
5740 // instruction defs. we can turn a memory op into an Address
5741
5742 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5743 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5744
5745 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5746 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5747
5748 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5749 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5750
5751 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5752 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5753
5754 // All of the memory operands. For the pipeline description.
5755 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5756 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5757 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5758
5759
5760 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5761 // operations. it allows the src to be either an iRegI or a (ConvL2I
5762 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5763 // can be elided because the 32-bit instruction will just employ the
5764 // lower 32 bits anyway.
5765 //
5766 // n.b. this does not elide all L2I conversions. if the truncated
5767 // value is consumed by more than one operation then the ConvL2I
5768 // cannot be bundled into the consuming nodes so an l2i gets planted
5769 // (actually a movw $dst $src) and the downstream instructions consume
5770 // the result of the l2i as an iRegI input. That's a shame since the
5771 // movw is actually redundant but its not too costly.
5772
5773 opclass iRegIorL2I(iRegI, iRegL2I);
5774 opclass iRegPorL2P(iRegP, iRegL2P);
5775
5776 //----------PIPELINE-----------------------------------------------------------
5777 // Rules which define the behavior of the target architectures pipeline.
5778
5779 // For specific pipelines, eg A53, define the stages of that pipeline
5780 //pipe_desc(ISS, EX1, EX2, WR);
5781 #define ISS S0
5782 #define EX1 S1
5783 #define EX2 S2
5784 #define WR S3
5785
5786 // Integer ALU reg operation
5787 pipeline %{
5788
5789 attributes %{
5790 // ARM instructions are of fixed length
5791 fixed_size_instructions; // Fixed size instructions TODO does
5792 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5793 // ARM instructions come in 32-bit word units
5794 instruction_unit_size = 4; // An instruction is 4 bytes long
5795 instruction_fetch_unit_size = 64; // The processor fetches one line
5796 instruction_fetch_units = 1; // of 64 bytes
5797
5798 // List of nop instructions
5799 nops( MachNop );
5800 %}
5801
5802 // We don't use an actual pipeline model so don't care about resources
5803 // or description. we do use pipeline classes to introduce fixed
5804 // latencies
5805
5806 //----------RESOURCES----------------------------------------------------------
5807 // Resources are the functional units available to the machine
5808
5809 resources( INS0, INS1, INS01 = INS0 | INS1,
5810 ALU0, ALU1, ALU = ALU0 | ALU1,
5811 MAC,
5812 DIV,
5813 BRANCH,
5814 LDST,
5815 NEON_FP);
5816
5817 //----------PIPELINE DESCRIPTION-----------------------------------------------
5818 // Pipeline Description specifies the stages in the machine's pipeline
5819
5820 // Define the pipeline as a generic 6 stage pipeline
5821 pipe_desc(S0, S1, S2, S3, S4, S5);
5822
5823 //----------PIPELINE CLASSES---------------------------------------------------
5824 // Pipeline Classes describe the stages in which input and output are
5825 // referenced by the hardware pipeline.
5826
5827 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5828 %{
5829 single_instruction;
5830 src1 : S1(read);
5831 src2 : S2(read);
5832 dst : S5(write);
5833 INS01 : ISS;
5834 NEON_FP : S5;
5835 %}
5836
5837 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5838 %{
5839 single_instruction;
5840 src1 : S1(read);
5841 src2 : S2(read);
5842 dst : S5(write);
5843 INS01 : ISS;
5844 NEON_FP : S5;
5845 %}
5846
5847 pipe_class fp_uop_s(vRegF dst, vRegF src)
5848 %{
5849 single_instruction;
5850 src : S1(read);
5851 dst : S5(write);
5852 INS01 : ISS;
5853 NEON_FP : S5;
5854 %}
5855
5856 pipe_class fp_uop_d(vRegD dst, vRegD src)
5857 %{
5858 single_instruction;
5859 src : S1(read);
5860 dst : S5(write);
5861 INS01 : ISS;
5862 NEON_FP : S5;
5863 %}
5864
5865 pipe_class fp_d2f(vRegF dst, vRegD src)
5866 %{
5867 single_instruction;
5868 src : S1(read);
5869 dst : S5(write);
5870 INS01 : ISS;
5871 NEON_FP : S5;
5872 %}
5873
5874 pipe_class fp_f2d(vRegD dst, vRegF src)
5875 %{
5876 single_instruction;
5877 src : S1(read);
5878 dst : S5(write);
5879 INS01 : ISS;
5880 NEON_FP : S5;
5881 %}
5882
5883 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5884 %{
5885 single_instruction;
5886 src : S1(read);
5887 dst : S5(write);
5888 INS01 : ISS;
5889 NEON_FP : S5;
5890 %}
5891
5892 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5893 %{
5894 single_instruction;
5895 src : S1(read);
5896 dst : S5(write);
5897 INS01 : ISS;
5898 NEON_FP : S5;
5899 %}
5900
5901 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5902 %{
5903 single_instruction;
5904 src : S1(read);
5905 dst : S5(write);
5906 INS01 : ISS;
5907 NEON_FP : S5;
5908 %}
5909
5910 pipe_class fp_l2f(vRegF dst, iRegL src)
5911 %{
5912 single_instruction;
5913 src : S1(read);
5914 dst : S5(write);
5915 INS01 : ISS;
5916 NEON_FP : S5;
5917 %}
5918
5919 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5920 %{
5921 single_instruction;
5922 src : S1(read);
5923 dst : S5(write);
5924 INS01 : ISS;
5925 NEON_FP : S5;
5926 %}
5927
5928 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5929 %{
5930 single_instruction;
5931 src : S1(read);
5932 dst : S5(write);
5933 INS01 : ISS;
5934 NEON_FP : S5;
5935 %}
5936
5937 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5938 %{
5939 single_instruction;
5940 src : S1(read);
5941 dst : S5(write);
5942 INS01 : ISS;
5943 NEON_FP : S5;
5944 %}
5945
5946 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5947 %{
5948 single_instruction;
5949 src : S1(read);
5950 dst : S5(write);
5951 INS01 : ISS;
5952 NEON_FP : S5;
5953 %}
5954
5955 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5956 %{
5957 single_instruction;
5958 src1 : S1(read);
5959 src2 : S2(read);
5960 dst : S5(write);
5961 INS0 : ISS;
5962 NEON_FP : S5;
5963 %}
5964
5965 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5966 %{
5967 single_instruction;
5968 src1 : S1(read);
5969 src2 : S2(read);
5970 dst : S5(write);
5971 INS0 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5976 %{
5977 single_instruction;
5978 cr : S1(read);
5979 src1 : S1(read);
5980 src2 : S1(read);
5981 dst : S3(write);
5982 INS01 : ISS;
5983 NEON_FP : S3;
5984 %}
5985
5986 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5987 %{
5988 single_instruction;
5989 cr : S1(read);
5990 src1 : S1(read);
5991 src2 : S1(read);
5992 dst : S3(write);
5993 INS01 : ISS;
5994 NEON_FP : S3;
5995 %}
5996
5997 pipe_class fp_imm_s(vRegF dst)
5998 %{
5999 single_instruction;
6000 dst : S3(write);
6001 INS01 : ISS;
6002 NEON_FP : S3;
6003 %}
6004
6005 pipe_class fp_imm_d(vRegD dst)
6006 %{
6007 single_instruction;
6008 dst : S3(write);
6009 INS01 : ISS;
6010 NEON_FP : S3;
6011 %}
6012
6013 pipe_class fp_load_constant_s(vRegF dst)
6014 %{
6015 single_instruction;
6016 dst : S4(write);
6017 INS01 : ISS;
6018 NEON_FP : S4;
6019 %}
6020
6021 pipe_class fp_load_constant_d(vRegD dst)
6022 %{
6023 single_instruction;
6024 dst : S4(write);
6025 INS01 : ISS;
6026 NEON_FP : S4;
6027 %}
6028
6029 //------- Integer ALU operations --------------------------
6030
6031 // Integer ALU reg-reg operation
6032 // Operands needed in EX1, result generated in EX2
6033 // Eg. ADD x0, x1, x2
6034 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6035 %{
6036 single_instruction;
6037 dst : EX2(write);
6038 src1 : EX1(read);
6039 src2 : EX1(read);
6040 INS01 : ISS; // Dual issue as instruction 0 or 1
6041 ALU : EX2;
6042 %}
6043
6044 // Integer ALU reg-reg operation with constant shift
6045 // Shifted register must be available in LATE_ISS instead of EX1
6046 // Eg. ADD x0, x1, x2, LSL #2
6047 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6048 %{
6049 single_instruction;
6050 dst : EX2(write);
6051 src1 : EX1(read);
6052 src2 : ISS(read);
6053 INS01 : ISS;
6054 ALU : EX2;
6055 %}
6056
6057 // Integer ALU reg operation with constant shift
6058 // Eg. LSL x0, x1, #shift
6059 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6060 %{
6061 single_instruction;
6062 dst : EX2(write);
6063 src1 : ISS(read);
6064 INS01 : ISS;
6065 ALU : EX2;
6066 %}
6067
6068 // Integer ALU reg-reg operation with variable shift
6069 // Both operands must be available in LATE_ISS instead of EX1
6070 // Result is available in EX1 instead of EX2
6071 // Eg. LSLV x0, x1, x2
6072 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6073 %{
6074 single_instruction;
6075 dst : EX1(write);
6076 src1 : ISS(read);
6077 src2 : ISS(read);
6078 INS01 : ISS;
6079 ALU : EX1;
6080 %}
6081
6082 // Integer ALU reg-reg operation with extract
6083 // As for _vshift above, but result generated in EX2
6084 // Eg. EXTR x0, x1, x2, #N
6085 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6086 %{
6087 single_instruction;
6088 dst : EX2(write);
6089 src1 : ISS(read);
6090 src2 : ISS(read);
6091 INS1 : ISS; // Can only dual issue as Instruction 1
6092 ALU : EX1;
6093 %}
6094
6095 // Integer ALU reg operation
6096 // Eg. NEG x0, x1
6097 pipe_class ialu_reg(iRegI dst, iRegI src)
6098 %{
6099 single_instruction;
6100 dst : EX2(write);
6101 src : EX1(read);
6102 INS01 : ISS;
6103 ALU : EX2;
6104 %}
6105
6106 // Integer ALU reg mmediate operation
6107 // Eg. ADD x0, x1, #N
6108 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6109 %{
6110 single_instruction;
6111 dst : EX2(write);
6112 src1 : EX1(read);
6113 INS01 : ISS;
6114 ALU : EX2;
6115 %}
6116
6117 // Integer ALU immediate operation (no source operands)
6118 // Eg. MOV x0, #N
6119 pipe_class ialu_imm(iRegI dst)
6120 %{
6121 single_instruction;
6122 dst : EX1(write);
6123 INS01 : ISS;
6124 ALU : EX1;
6125 %}
6126
6127 //------- Compare operation -------------------------------
6128
6129 // Compare reg-reg
6130 // Eg. CMP x0, x1
6131 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6132 %{
6133 single_instruction;
6134 // fixed_latency(16);
6135 cr : EX2(write);
6136 op1 : EX1(read);
6137 op2 : EX1(read);
6138 INS01 : ISS;
6139 ALU : EX2;
6140 %}
6141
6142 // Compare reg-reg
6143 // Eg. CMP x0, #N
6144 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6145 %{
6146 single_instruction;
6147 // fixed_latency(16);
6148 cr : EX2(write);
6149 op1 : EX1(read);
6150 INS01 : ISS;
6151 ALU : EX2;
6152 %}
6153
6154 //------- Conditional instructions ------------------------
6155
6156 // Conditional no operands
6157 // Eg. CSINC x0, zr, zr, <cond>
6158 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6159 %{
6160 single_instruction;
6161 cr : EX1(read);
6162 dst : EX2(write);
6163 INS01 : ISS;
6164 ALU : EX2;
6165 %}
6166
6167 // Conditional 2 operand
6168 // EG. CSEL X0, X1, X2, <cond>
6169 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6170 %{
6171 single_instruction;
6172 cr : EX1(read);
6173 src1 : EX1(read);
6174 src2 : EX1(read);
6175 dst : EX2(write);
6176 INS01 : ISS;
6177 ALU : EX2;
6178 %}
6179
6180 // Conditional 2 operand
6181 // EG. CSEL X0, X1, X2, <cond>
6182 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6183 %{
6184 single_instruction;
6185 cr : EX1(read);
6186 src : EX1(read);
6187 dst : EX2(write);
6188 INS01 : ISS;
6189 ALU : EX2;
6190 %}
6191
6192 //------- Multiply pipeline operations --------------------
6193
6194 // Multiply reg-reg
6195 // Eg. MUL w0, w1, w2
6196 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6197 %{
6198 single_instruction;
6199 dst : WR(write);
6200 src1 : ISS(read);
6201 src2 : ISS(read);
6202 INS01 : ISS;
6203 MAC : WR;
6204 %}
6205
6206 // Multiply accumulate
6207 // Eg. MADD w0, w1, w2, w3
6208 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6209 %{
6210 single_instruction;
6211 dst : WR(write);
6212 src1 : ISS(read);
6213 src2 : ISS(read);
6214 src3 : ISS(read);
6215 INS01 : ISS;
6216 MAC : WR;
6217 %}
6218
6219 // Eg. MUL w0, w1, w2
6220 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6221 %{
6222 single_instruction;
6223 fixed_latency(3); // Maximum latency for 64 bit mul
6224 dst : WR(write);
6225 src1 : ISS(read);
6226 src2 : ISS(read);
6227 INS01 : ISS;
6228 MAC : WR;
6229 %}
6230
6231 // Multiply accumulate
6232 // Eg. MADD w0, w1, w2, w3
6233 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6234 %{
6235 single_instruction;
6236 fixed_latency(3); // Maximum latency for 64 bit mul
6237 dst : WR(write);
6238 src1 : ISS(read);
6239 src2 : ISS(read);
6240 src3 : ISS(read);
6241 INS01 : ISS;
6242 MAC : WR;
6243 %}
6244
6245 //------- Divide pipeline operations --------------------
6246
6247 // Eg. SDIV w0, w1, w2
6248 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6249 %{
6250 single_instruction;
6251 fixed_latency(8); // Maximum latency for 32 bit divide
6252 dst : WR(write);
6253 src1 : ISS(read);
6254 src2 : ISS(read);
6255 INS0 : ISS; // Can only dual issue as instruction 0
6256 DIV : WR;
6257 %}
6258
6259 // Eg. SDIV x0, x1, x2
6260 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6261 %{
6262 single_instruction;
6263 fixed_latency(16); // Maximum latency for 64 bit divide
6264 dst : WR(write);
6265 src1 : ISS(read);
6266 src2 : ISS(read);
6267 INS0 : ISS; // Can only dual issue as instruction 0
6268 DIV : WR;
6269 %}
6270
6271 //------- Load pipeline operations ------------------------
6272
6273 // Load - prefetch
6274 // Eg. PFRM <mem>
6275 pipe_class iload_prefetch(memory mem)
6276 %{
6277 single_instruction;
6278 mem : ISS(read);
6279 INS01 : ISS;
6280 LDST : WR;
6281 %}
6282
6283 // Load - reg, mem
6284 // Eg. LDR x0, <mem>
6285 pipe_class iload_reg_mem(iRegI dst, memory mem)
6286 %{
6287 single_instruction;
6288 dst : WR(write);
6289 mem : ISS(read);
6290 INS01 : ISS;
6291 LDST : WR;
6292 %}
6293
6294 // Load - reg, reg
6295 // Eg. LDR x0, [sp, x1]
6296 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6297 %{
6298 single_instruction;
6299 dst : WR(write);
6300 src : ISS(read);
6301 INS01 : ISS;
6302 LDST : WR;
6303 %}
6304
6305 //------- Store pipeline operations -----------------------
6306
6307 // Store - zr, mem
6308 // Eg. STR zr, <mem>
6309 pipe_class istore_mem(memory mem)
6310 %{
6311 single_instruction;
6312 mem : ISS(read);
6313 INS01 : ISS;
6314 LDST : WR;
6315 %}
6316
6317 // Store - reg, mem
6318 // Eg. STR x0, <mem>
6319 pipe_class istore_reg_mem(iRegI src, memory mem)
6320 %{
6321 single_instruction;
6322 mem : ISS(read);
6323 src : EX2(read);
6324 INS01 : ISS;
6325 LDST : WR;
6326 %}
6327
6328 // Store - reg, reg
6329 // Eg. STR x0, [sp, x1]
6330 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6331 %{
6332 single_instruction;
6333 dst : ISS(read);
6334 src : EX2(read);
6335 INS01 : ISS;
6336 LDST : WR;
6337 %}
6338
6339 //------- Store pipeline operations -----------------------
6340
6341 // Branch
6342 pipe_class pipe_branch()
6343 %{
6344 single_instruction;
6345 INS01 : ISS;
6346 BRANCH : EX1;
6347 %}
6348
6349 // Conditional branch
6350 pipe_class pipe_branch_cond(rFlagsReg cr)
6351 %{
6352 single_instruction;
6353 cr : EX1(read);
6354 INS01 : ISS;
6355 BRANCH : EX1;
6356 %}
6357
6358 // Compare & Branch
6359 // EG. CBZ/CBNZ
6360 pipe_class pipe_cmp_branch(iRegI op1)
6361 %{
6362 single_instruction;
6363 op1 : EX1(read);
6364 INS01 : ISS;
6365 BRANCH : EX1;
6366 %}
6367
6368 //------- Synchronisation operations ----------------------
6369
6370 // Any operation requiring serialization.
6371 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6372 pipe_class pipe_serial()
6373 %{
6374 single_instruction;
6375 force_serialization;
6376 fixed_latency(16);
6377 INS01 : ISS(2); // Cannot dual issue with any other instruction
6378 LDST : WR;
6379 %}
6380
6381 // Generic big/slow expanded idiom - also serialized
6382 pipe_class pipe_slow()
6383 %{
6384 instruction_count(10);
6385 multiple_bundles;
6386 force_serialization;
6387 fixed_latency(16);
6388 INS01 : ISS(2); // Cannot dual issue with any other instruction
6389 LDST : WR;
6390 %}
6391
6392 // Empty pipeline class
6393 pipe_class pipe_class_empty()
6394 %{
6395 single_instruction;
6396 fixed_latency(0);
6397 %}
6398
6399 // Default pipeline class.
6400 pipe_class pipe_class_default()
6401 %{
6402 single_instruction;
6403 fixed_latency(2);
6404 %}
6405
6406 // Pipeline class for compares.
6407 pipe_class pipe_class_compare()
6408 %{
6409 single_instruction;
6410 fixed_latency(16);
6411 %}
6412
6413 // Pipeline class for memory operations.
6414 pipe_class pipe_class_memory()
6415 %{
6416 single_instruction;
6417 fixed_latency(16);
6418 %}
6419
6420 // Pipeline class for call.
6421 pipe_class pipe_class_call()
6422 %{
6423 single_instruction;
6424 fixed_latency(100);
6425 %}
6426
6427 // Define the class for the Nop node.
6428 define %{
6429 MachNop = pipe_class_empty;
6430 %}
6431
6432 %}
6433 //----------INSTRUCTIONS-------------------------------------------------------
6434 //
6435 // match -- States which machine-independent subtree may be replaced
6436 // by this instruction.
6437 // ins_cost -- The estimated cost of this instruction is used by instruction
6438 // selection to identify a minimum cost tree of machine
6439 // instructions that matches a tree of machine-independent
6440 // instructions.
6441 // format -- A string providing the disassembly for this instruction.
6442 // The value of an instruction's operand may be inserted
6443 // by referring to it with a '$' prefix.
6444 // opcode -- Three instruction opcodes may be provided. These are referred
6445 // to within an encode class as $primary, $secondary, and $tertiary
6446 // rrspectively. The primary opcode is commonly used to
6447 // indicate the type of machine instruction, while secondary
6448 // and tertiary are often used for prefix options or addressing
6449 // modes.
6450 // ins_encode -- A list of encode classes with parameters. The encode class
6451 // name must have been defined in an 'enc_class' specification
6452 // in the encode section of the architecture description.
6453
6454 // ============================================================================
6455 // Memory (Load/Store) Instructions
6456
6457 // Load Instructions
6458
6459 // Load Byte (8 bit signed)
6460 instruct loadB(iRegINoSp dst, memory1 mem)
6461 %{
6462 match(Set dst (LoadB mem));
6463 predicate(!needs_acquiring_load(n));
6464
6465 ins_cost(4 * INSN_COST);
6466 format %{ "ldrsbw $dst, $mem\t# byte" %}
6467
6468 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6469
6470 ins_pipe(iload_reg_mem);
6471 %}
6472
6473 // Load Byte (8 bit signed) into long
6474 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6475 %{
6476 match(Set dst (ConvI2L (LoadB mem)));
6477 predicate(!needs_acquiring_load(n->in(1)));
6478
6479 ins_cost(4 * INSN_COST);
6480 format %{ "ldrsb $dst, $mem\t# byte" %}
6481
6482 ins_encode(aarch64_enc_ldrsb(dst, mem));
6483
6484 ins_pipe(iload_reg_mem);
6485 %}
6486
6487 // Load Byte (8 bit unsigned)
6488 instruct loadUB(iRegINoSp dst, memory1 mem)
6489 %{
6490 match(Set dst (LoadUB mem));
6491 predicate(!needs_acquiring_load(n));
6492
6493 ins_cost(4 * INSN_COST);
6494 format %{ "ldrbw $dst, $mem\t# byte" %}
6495
6496 ins_encode(aarch64_enc_ldrb(dst, mem));
6497
6498 ins_pipe(iload_reg_mem);
6499 %}
6500
6501 // Load Byte (8 bit unsigned) into long
6502 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6503 %{
6504 match(Set dst (ConvI2L (LoadUB mem)));
6505 predicate(!needs_acquiring_load(n->in(1)));
6506
6507 ins_cost(4 * INSN_COST);
6508 format %{ "ldrb $dst, $mem\t# byte" %}
6509
6510 ins_encode(aarch64_enc_ldrb(dst, mem));
6511
6512 ins_pipe(iload_reg_mem);
6513 %}
6514
6515 // Load Short (16 bit signed)
6516 instruct loadS(iRegINoSp dst, memory2 mem)
6517 %{
6518 match(Set dst (LoadS mem));
6519 predicate(!needs_acquiring_load(n));
6520
6521 ins_cost(4 * INSN_COST);
6522 format %{ "ldrshw $dst, $mem\t# short" %}
6523
6524 ins_encode(aarch64_enc_ldrshw(dst, mem));
6525
6526 ins_pipe(iload_reg_mem);
6527 %}
6528
6529 // Load Short (16 bit signed) into long
6530 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6531 %{
6532 match(Set dst (ConvI2L (LoadS mem)));
6533 predicate(!needs_acquiring_load(n->in(1)));
6534
6535 ins_cost(4 * INSN_COST);
6536 format %{ "ldrsh $dst, $mem\t# short" %}
6537
6538 ins_encode(aarch64_enc_ldrsh(dst, mem));
6539
6540 ins_pipe(iload_reg_mem);
6541 %}
6542
6543 // Load Char (16 bit unsigned)
6544 instruct loadUS(iRegINoSp dst, memory2 mem)
6545 %{
6546 match(Set dst (LoadUS mem));
6547 predicate(!needs_acquiring_load(n));
6548
6549 ins_cost(4 * INSN_COST);
6550 format %{ "ldrh $dst, $mem\t# short" %}
6551
6552 ins_encode(aarch64_enc_ldrh(dst, mem));
6553
6554 ins_pipe(iload_reg_mem);
6555 %}
6556
6557 // Load Short/Char (16 bit unsigned) into long
6558 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6559 %{
6560 match(Set dst (ConvI2L (LoadUS mem)));
6561 predicate(!needs_acquiring_load(n->in(1)));
6562
6563 ins_cost(4 * INSN_COST);
6564 format %{ "ldrh $dst, $mem\t# short" %}
6565
6566 ins_encode(aarch64_enc_ldrh(dst, mem));
6567
6568 ins_pipe(iload_reg_mem);
6569 %}
6570
6571 // Load Integer (32 bit signed)
6572 instruct loadI(iRegINoSp dst, memory4 mem)
6573 %{
6574 match(Set dst (LoadI mem));
6575 predicate(!needs_acquiring_load(n));
6576
6577 ins_cost(4 * INSN_COST);
6578 format %{ "ldrw $dst, $mem\t# int" %}
6579
6580 ins_encode(aarch64_enc_ldrw(dst, mem));
6581
6582 ins_pipe(iload_reg_mem);
6583 %}
6584
6585 // Load Integer (32 bit signed) into long
6586 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6587 %{
6588 match(Set dst (ConvI2L (LoadI mem)));
6589 predicate(!needs_acquiring_load(n->in(1)));
6590
6591 ins_cost(4 * INSN_COST);
6592 format %{ "ldrsw $dst, $mem\t# int" %}
6593
6594 ins_encode(aarch64_enc_ldrsw(dst, mem));
6595
6596 ins_pipe(iload_reg_mem);
6597 %}
6598
6599 // Load Integer (32 bit unsigned) into long
6600 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6601 %{
6602 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6603 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6604
6605 ins_cost(4 * INSN_COST);
6606 format %{ "ldrw $dst, $mem\t# int" %}
6607
6608 ins_encode(aarch64_enc_ldrw(dst, mem));
6609
6610 ins_pipe(iload_reg_mem);
6611 %}
6612
6613 // Load Long (64 bit signed)
6614 instruct loadL(iRegLNoSp dst, memory8 mem)
6615 %{
6616 match(Set dst (LoadL mem));
6617 predicate(!needs_acquiring_load(n));
6618
6619 ins_cost(4 * INSN_COST);
6620 format %{ "ldr $dst, $mem\t# int" %}
6621
6622 ins_encode(aarch64_enc_ldr(dst, mem));
6623
6624 ins_pipe(iload_reg_mem);
6625 %}
6626
6627 // Load Range
6628 instruct loadRange(iRegINoSp dst, memory4 mem)
6629 %{
6630 match(Set dst (LoadRange mem));
6631
6632 ins_cost(4 * INSN_COST);
6633 format %{ "ldrw $dst, $mem\t# range" %}
6634
6635 ins_encode(aarch64_enc_ldrw(dst, mem));
6636
6637 ins_pipe(iload_reg_mem);
6638 %}
6639
6640 // Load Pointer
6641 instruct loadP(iRegPNoSp dst, memory8 mem)
6642 %{
6643 match(Set dst (LoadP mem));
6644 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6645
6646 ins_cost(4 * INSN_COST);
6647 format %{ "ldr $dst, $mem\t# ptr" %}
6648
6649 ins_encode(aarch64_enc_ldr(dst, mem));
6650
6651 ins_pipe(iload_reg_mem);
6652 %}
6653
6654 // Load Compressed Pointer
6655 instruct loadN(iRegNNoSp dst, memory4 mem)
6656 %{
6657 match(Set dst (LoadN mem));
6658 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6659
6660 ins_cost(4 * INSN_COST);
6661 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6662
6663 ins_encode(aarch64_enc_ldrw(dst, mem));
6664
6665 ins_pipe(iload_reg_mem);
6666 %}
6667
6668 // Load Klass Pointer
6669 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6670 %{
6671 match(Set dst (LoadKlass mem));
6672 predicate(!needs_acquiring_load(n));
6673
6674 ins_cost(4 * INSN_COST);
6675 format %{ "ldr $dst, $mem\t# class" %}
6676
6677 ins_encode(aarch64_enc_ldr(dst, mem));
6678
6679 ins_pipe(iload_reg_mem);
6680 %}
6681
6682 // Load Narrow Klass Pointer
6683 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6684 %{
6685 match(Set dst (LoadNKlass mem));
6686 predicate(!needs_acquiring_load(n));
6687
6688 ins_cost(4 * INSN_COST);
6689 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6690
6691 ins_encode(aarch64_enc_ldrw(dst, mem));
6692
6693 ins_pipe(iload_reg_mem);
6694 %}
6695
6696 // Load Float
6697 instruct loadF(vRegF dst, memory4 mem)
6698 %{
6699 match(Set dst (LoadF mem));
6700 predicate(!needs_acquiring_load(n));
6701
6702 ins_cost(4 * INSN_COST);
6703 format %{ "ldrs $dst, $mem\t# float" %}
6704
6705 ins_encode( aarch64_enc_ldrs(dst, mem) );
6706
6707 ins_pipe(pipe_class_memory);
6708 %}
6709
6710 // Load Double
6711 instruct loadD(vRegD dst, memory8 mem)
6712 %{
6713 match(Set dst (LoadD mem));
6714 predicate(!needs_acquiring_load(n));
6715
6716 ins_cost(4 * INSN_COST);
6717 format %{ "ldrd $dst, $mem\t# double" %}
6718
6719 ins_encode( aarch64_enc_ldrd(dst, mem) );
6720
6721 ins_pipe(pipe_class_memory);
6722 %}
6723
6724
6725 // Load Int Constant
6726 instruct loadConI(iRegINoSp dst, immI src)
6727 %{
6728 match(Set dst src);
6729
6730 ins_cost(INSN_COST);
6731 format %{ "mov $dst, $src\t# int" %}
6732
6733 ins_encode( aarch64_enc_movw_imm(dst, src) );
6734
6735 ins_pipe(ialu_imm);
6736 %}
6737
6738 // Load Long Constant
6739 instruct loadConL(iRegLNoSp dst, immL src)
6740 %{
6741 match(Set dst src);
6742
6743 ins_cost(INSN_COST);
6744 format %{ "mov $dst, $src\t# long" %}
6745
6746 ins_encode( aarch64_enc_mov_imm(dst, src) );
6747
6748 ins_pipe(ialu_imm);
6749 %}
6750
6751 // Load Pointer Constant
6752
6753 instruct loadConP(iRegPNoSp dst, immP con)
6754 %{
6755 match(Set dst con);
6756
6757 ins_cost(INSN_COST * 4);
6758 format %{
6759 "mov $dst, $con\t# ptr"
6760 %}
6761
6762 ins_encode(aarch64_enc_mov_p(dst, con));
6763
6764 ins_pipe(ialu_imm);
6765 %}
6766
6767 // Load Null Pointer Constant
6768
6769 instruct loadConP0(iRegPNoSp dst, immP0 con)
6770 %{
6771 match(Set dst con);
6772
6773 ins_cost(INSN_COST);
6774 format %{ "mov $dst, $con\t# nullptr ptr" %}
6775
6776 ins_encode(aarch64_enc_mov_p0(dst, con));
6777
6778 ins_pipe(ialu_imm);
6779 %}
6780
6781 // Load Pointer Constant One
6782
6783 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6784 %{
6785 match(Set dst con);
6786
6787 ins_cost(INSN_COST);
6788 format %{ "mov $dst, $con\t# nullptr ptr" %}
6789
6790 ins_encode(aarch64_enc_mov_p1(dst, con));
6791
6792 ins_pipe(ialu_imm);
6793 %}
6794
6795 // Load Byte Map Base Constant
6796
6797 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6798 %{
6799 match(Set dst con);
6800
6801 ins_cost(INSN_COST);
6802 format %{ "adr $dst, $con\t# Byte Map Base" %}
6803
6804 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6805
6806 ins_pipe(ialu_imm);
6807 %}
6808
6809 // Load Narrow Pointer Constant
6810
6811 instruct loadConN(iRegNNoSp dst, immN con)
6812 %{
6813 match(Set dst con);
6814
6815 ins_cost(INSN_COST * 4);
6816 format %{ "mov $dst, $con\t# compressed ptr" %}
6817
6818 ins_encode(aarch64_enc_mov_n(dst, con));
6819
6820 ins_pipe(ialu_imm);
6821 %}
6822
6823 // Load Narrow Null Pointer Constant
6824
6825 instruct loadConN0(iRegNNoSp dst, immN0 con)
6826 %{
6827 match(Set dst con);
6828
6829 ins_cost(INSN_COST);
6830 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6831
6832 ins_encode(aarch64_enc_mov_n0(dst, con));
6833
6834 ins_pipe(ialu_imm);
6835 %}
6836
6837 // Load Narrow Klass Constant
6838
6839 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6840 %{
6841 match(Set dst con);
6842
6843 ins_cost(INSN_COST);
6844 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6845
6846 ins_encode(aarch64_enc_mov_nk(dst, con));
6847
6848 ins_pipe(ialu_imm);
6849 %}
6850
6851 // Load Packed Float Constant
6852
6853 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6854 match(Set dst con);
6855 ins_cost(INSN_COST * 4);
6856 format %{ "fmovs $dst, $con"%}
6857 ins_encode %{
6858 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6859 %}
6860
6861 ins_pipe(fp_imm_s);
6862 %}
6863
6864 // Load Float Constant
6865
6866 instruct loadConF(vRegF dst, immF con) %{
6867 match(Set dst con);
6868
6869 ins_cost(INSN_COST * 4);
6870
6871 format %{
6872 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6873 %}
6874
6875 ins_encode %{
6876 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6877 %}
6878
6879 ins_pipe(fp_load_constant_s);
6880 %}
6881
6882 // Load Packed Double Constant
6883
6884 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6885 match(Set dst con);
6886 ins_cost(INSN_COST);
6887 format %{ "fmovd $dst, $con"%}
6888 ins_encode %{
6889 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6890 %}
6891
6892 ins_pipe(fp_imm_d);
6893 %}
6894
6895 // Load Double Constant
6896
6897 instruct loadConD(vRegD dst, immD con) %{
6898 match(Set dst con);
6899
6900 ins_cost(INSN_COST * 5);
6901 format %{
6902 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6903 %}
6904
6905 ins_encode %{
6906 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6907 %}
6908
6909 ins_pipe(fp_load_constant_d);
6910 %}
6911
6912 // Store Instructions
6913
6914 // Store Byte
6915 instruct storeB(iRegIorL2I src, memory1 mem)
6916 %{
6917 match(Set mem (StoreB mem src));
6918 predicate(!needs_releasing_store(n));
6919
6920 ins_cost(INSN_COST);
6921 format %{ "strb $src, $mem\t# byte" %}
6922
6923 ins_encode(aarch64_enc_strb(src, mem));
6924
6925 ins_pipe(istore_reg_mem);
6926 %}
6927
6928
6929 instruct storeimmB0(immI0 zero, memory1 mem)
6930 %{
6931 match(Set mem (StoreB mem zero));
6932 predicate(!needs_releasing_store(n));
6933
6934 ins_cost(INSN_COST);
6935 format %{ "strb rscractch2, $mem\t# byte" %}
6936
6937 ins_encode(aarch64_enc_strb0(mem));
6938
6939 ins_pipe(istore_mem);
6940 %}
6941
6942 // Store Char/Short
6943 instruct storeC(iRegIorL2I src, memory2 mem)
6944 %{
6945 match(Set mem (StoreC mem src));
6946 predicate(!needs_releasing_store(n));
6947
6948 ins_cost(INSN_COST);
6949 format %{ "strh $src, $mem\t# short" %}
6950
6951 ins_encode(aarch64_enc_strh(src, mem));
6952
6953 ins_pipe(istore_reg_mem);
6954 %}
6955
6956 instruct storeimmC0(immI0 zero, memory2 mem)
6957 %{
6958 match(Set mem (StoreC mem zero));
6959 predicate(!needs_releasing_store(n));
6960
6961 ins_cost(INSN_COST);
6962 format %{ "strh zr, $mem\t# short" %}
6963
6964 ins_encode(aarch64_enc_strh0(mem));
6965
6966 ins_pipe(istore_mem);
6967 %}
6968
6969 // Store Integer
6970
6971 instruct storeI(iRegIorL2I src, memory4 mem)
6972 %{
6973 match(Set mem(StoreI mem src));
6974 predicate(!needs_releasing_store(n));
6975
6976 ins_cost(INSN_COST);
6977 format %{ "strw $src, $mem\t# int" %}
6978
6979 ins_encode(aarch64_enc_strw(src, mem));
6980
6981 ins_pipe(istore_reg_mem);
6982 %}
6983
6984 instruct storeimmI0(immI0 zero, memory4 mem)
6985 %{
6986 match(Set mem(StoreI mem zero));
6987 predicate(!needs_releasing_store(n));
6988
6989 ins_cost(INSN_COST);
6990 format %{ "strw zr, $mem\t# int" %}
6991
6992 ins_encode(aarch64_enc_strw0(mem));
6993
6994 ins_pipe(istore_mem);
6995 %}
6996
6997 // Store Long (64 bit signed)
6998 instruct storeL(iRegL src, memory8 mem)
6999 %{
7000 match(Set mem (StoreL mem src));
7001 predicate(!needs_releasing_store(n));
7002
7003 ins_cost(INSN_COST);
7004 format %{ "str $src, $mem\t# int" %}
7005
7006 ins_encode(aarch64_enc_str(src, mem));
7007
7008 ins_pipe(istore_reg_mem);
7009 %}
7010
7011 // Store Long (64 bit signed)
7012 instruct storeimmL0(immL0 zero, memory8 mem)
7013 %{
7014 match(Set mem (StoreL mem zero));
7015 predicate(!needs_releasing_store(n));
7016
7017 ins_cost(INSN_COST);
7018 format %{ "str zr, $mem\t# int" %}
7019
7020 ins_encode(aarch64_enc_str0(mem));
7021
7022 ins_pipe(istore_mem);
7023 %}
7024
7025 // Store Pointer
7026 instruct storeP(iRegP src, memory8 mem)
7027 %{
7028 match(Set mem (StoreP mem src));
7029 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7030
7031 ins_cost(INSN_COST);
7032 format %{ "str $src, $mem\t# ptr" %}
7033
7034 ins_encode(aarch64_enc_str(src, mem));
7035
7036 ins_pipe(istore_reg_mem);
7037 %}
7038
7039 // Store Pointer
7040 instruct storeimmP0(immP0 zero, memory8 mem)
7041 %{
7042 match(Set mem (StoreP mem zero));
7043 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7044
7045 ins_cost(INSN_COST);
7046 format %{ "str zr, $mem\t# ptr" %}
7047
7048 ins_encode(aarch64_enc_str0(mem));
7049
7050 ins_pipe(istore_mem);
7051 %}
7052
7053 // Store Compressed Pointer
7054 instruct storeN(iRegN src, memory4 mem)
7055 %{
7056 match(Set mem (StoreN mem src));
7057 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7058
7059 ins_cost(INSN_COST);
7060 format %{ "strw $src, $mem\t# compressed ptr" %}
7061
7062 ins_encode(aarch64_enc_strw(src, mem));
7063
7064 ins_pipe(istore_reg_mem);
7065 %}
7066
7067 instruct storeImmN0(immN0 zero, memory4 mem)
7068 %{
7069 match(Set mem (StoreN mem zero));
7070 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7071
7072 ins_cost(INSN_COST);
7073 format %{ "strw zr, $mem\t# compressed ptr" %}
7074
7075 ins_encode(aarch64_enc_strw0(mem));
7076
7077 ins_pipe(istore_mem);
7078 %}
7079
7080 // Store Float
7081 instruct storeF(vRegF src, memory4 mem)
7082 %{
7083 match(Set mem (StoreF mem src));
7084 predicate(!needs_releasing_store(n));
7085
7086 ins_cost(INSN_COST);
7087 format %{ "strs $src, $mem\t# float" %}
7088
7089 ins_encode( aarch64_enc_strs(src, mem) );
7090
7091 ins_pipe(pipe_class_memory);
7092 %}
7093
7094 // TODO
7095 // implement storeImmF0 and storeFImmPacked
7096
7097 // Store Double
7098 instruct storeD(vRegD src, memory8 mem)
7099 %{
7100 match(Set mem (StoreD mem src));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "strd $src, $mem\t# double" %}
7105
7106 ins_encode( aarch64_enc_strd(src, mem) );
7107
7108 ins_pipe(pipe_class_memory);
7109 %}
7110
7111 // Store Compressed Klass Pointer
7112 instruct storeNKlass(iRegN src, memory4 mem)
7113 %{
7114 predicate(!needs_releasing_store(n));
7115 match(Set mem (StoreNKlass mem src));
7116
7117 ins_cost(INSN_COST);
7118 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7119
7120 ins_encode(aarch64_enc_strw(src, mem));
7121
7122 ins_pipe(istore_reg_mem);
7123 %}
7124
7125 // TODO
7126 // implement storeImmD0 and storeDImmPacked
7127
7128 // prefetch instructions
7129 // Must be safe to execute with invalid address (cannot fault).
7130
7131 instruct prefetchalloc( memory8 mem ) %{
7132 match(PrefetchAllocation mem);
7133
7134 ins_cost(INSN_COST);
7135 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7136
7137 ins_encode( aarch64_enc_prefetchw(mem) );
7138
7139 ins_pipe(iload_prefetch);
7140 %}
7141
7142 // ---------------- volatile loads and stores ----------------
7143
7144 // Load Byte (8 bit signed)
7145 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7146 %{
7147 match(Set dst (LoadB mem));
7148
7149 ins_cost(VOLATILE_REF_COST);
7150 format %{ "ldarsb $dst, $mem\t# byte" %}
7151
7152 ins_encode(aarch64_enc_ldarsb(dst, mem));
7153
7154 ins_pipe(pipe_serial);
7155 %}
7156
7157 // Load Byte (8 bit signed) into long
7158 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7159 %{
7160 match(Set dst (ConvI2L (LoadB mem)));
7161
7162 ins_cost(VOLATILE_REF_COST);
7163 format %{ "ldarsb $dst, $mem\t# byte" %}
7164
7165 ins_encode(aarch64_enc_ldarsb(dst, mem));
7166
7167 ins_pipe(pipe_serial);
7168 %}
7169
7170 // Load Byte (8 bit unsigned)
7171 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7172 %{
7173 match(Set dst (LoadUB mem));
7174
7175 ins_cost(VOLATILE_REF_COST);
7176 format %{ "ldarb $dst, $mem\t# byte" %}
7177
7178 ins_encode(aarch64_enc_ldarb(dst, mem));
7179
7180 ins_pipe(pipe_serial);
7181 %}
7182
7183 // Load Byte (8 bit unsigned) into long
7184 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7185 %{
7186 match(Set dst (ConvI2L (LoadUB mem)));
7187
7188 ins_cost(VOLATILE_REF_COST);
7189 format %{ "ldarb $dst, $mem\t# byte" %}
7190
7191 ins_encode(aarch64_enc_ldarb(dst, mem));
7192
7193 ins_pipe(pipe_serial);
7194 %}
7195
7196 // Load Short (16 bit signed)
7197 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7198 %{
7199 match(Set dst (LoadS mem));
7200
7201 ins_cost(VOLATILE_REF_COST);
7202 format %{ "ldarshw $dst, $mem\t# short" %}
7203
7204 ins_encode(aarch64_enc_ldarshw(dst, mem));
7205
7206 ins_pipe(pipe_serial);
7207 %}
7208
7209 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7210 %{
7211 match(Set dst (LoadUS mem));
7212
7213 ins_cost(VOLATILE_REF_COST);
7214 format %{ "ldarhw $dst, $mem\t# short" %}
7215
7216 ins_encode(aarch64_enc_ldarhw(dst, mem));
7217
7218 ins_pipe(pipe_serial);
7219 %}
7220
7221 // Load Short/Char (16 bit unsigned) into long
7222 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7223 %{
7224 match(Set dst (ConvI2L (LoadUS mem)));
7225
7226 ins_cost(VOLATILE_REF_COST);
7227 format %{ "ldarh $dst, $mem\t# short" %}
7228
7229 ins_encode(aarch64_enc_ldarh(dst, mem));
7230
7231 ins_pipe(pipe_serial);
7232 %}
7233
7234 // Load Short/Char (16 bit signed) into long
7235 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7236 %{
7237 match(Set dst (ConvI2L (LoadS mem)));
7238
7239 ins_cost(VOLATILE_REF_COST);
7240 format %{ "ldarh $dst, $mem\t# short" %}
7241
7242 ins_encode(aarch64_enc_ldarsh(dst, mem));
7243
7244 ins_pipe(pipe_serial);
7245 %}
7246
7247 // Load Integer (32 bit signed)
7248 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7249 %{
7250 match(Set dst (LoadI mem));
7251
7252 ins_cost(VOLATILE_REF_COST);
7253 format %{ "ldarw $dst, $mem\t# int" %}
7254
7255 ins_encode(aarch64_enc_ldarw(dst, mem));
7256
7257 ins_pipe(pipe_serial);
7258 %}
7259
7260 // Load Integer (32 bit unsigned) into long
7261 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7262 %{
7263 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7264
7265 ins_cost(VOLATILE_REF_COST);
7266 format %{ "ldarw $dst, $mem\t# int" %}
7267
7268 ins_encode(aarch64_enc_ldarw(dst, mem));
7269
7270 ins_pipe(pipe_serial);
7271 %}
7272
7273 // Load Long (64 bit signed)
7274 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7275 %{
7276 match(Set dst (LoadL mem));
7277
7278 ins_cost(VOLATILE_REF_COST);
7279 format %{ "ldar $dst, $mem\t# int" %}
7280
7281 ins_encode(aarch64_enc_ldar(dst, mem));
7282
7283 ins_pipe(pipe_serial);
7284 %}
7285
7286 // Load Pointer
7287 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7288 %{
7289 match(Set dst (LoadP mem));
7290 predicate(n->as_Load()->barrier_data() == 0);
7291
7292 ins_cost(VOLATILE_REF_COST);
7293 format %{ "ldar $dst, $mem\t# ptr" %}
7294
7295 ins_encode(aarch64_enc_ldar(dst, mem));
7296
7297 ins_pipe(pipe_serial);
7298 %}
7299
7300 // Load Compressed Pointer
7301 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7302 %{
7303 match(Set dst (LoadN mem));
7304 predicate(n->as_Load()->barrier_data() == 0);
7305
7306 ins_cost(VOLATILE_REF_COST);
7307 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7308
7309 ins_encode(aarch64_enc_ldarw(dst, mem));
7310
7311 ins_pipe(pipe_serial);
7312 %}
7313
7314 // Load Float
7315 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7316 %{
7317 match(Set dst (LoadF mem));
7318
7319 ins_cost(VOLATILE_REF_COST);
7320 format %{ "ldars $dst, $mem\t# float" %}
7321
7322 ins_encode( aarch64_enc_fldars(dst, mem) );
7323
7324 ins_pipe(pipe_serial);
7325 %}
7326
7327 // Load Double
7328 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7329 %{
7330 match(Set dst (LoadD mem));
7331
7332 ins_cost(VOLATILE_REF_COST);
7333 format %{ "ldard $dst, $mem\t# double" %}
7334
7335 ins_encode( aarch64_enc_fldard(dst, mem) );
7336
7337 ins_pipe(pipe_serial);
7338 %}
7339
7340 // Store Byte
7341 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7342 %{
7343 match(Set mem (StoreB mem src));
7344
7345 ins_cost(VOLATILE_REF_COST);
7346 format %{ "stlrb $src, $mem\t# byte" %}
7347
7348 ins_encode(aarch64_enc_stlrb(src, mem));
7349
7350 ins_pipe(pipe_class_memory);
7351 %}
7352
7353 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set mem (StoreB mem zero));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "stlrb zr, $mem\t# byte" %}
7359
7360 ins_encode(aarch64_enc_stlrb0(mem));
7361
7362 ins_pipe(pipe_class_memory);
7363 %}
7364
7365 // Store Char/Short
7366 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set mem (StoreC mem src));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "stlrh $src, $mem\t# short" %}
7372
7373 ins_encode(aarch64_enc_stlrh(src, mem));
7374
7375 ins_pipe(pipe_class_memory);
7376 %}
7377
7378 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7379 %{
7380 match(Set mem (StoreC mem zero));
7381
7382 ins_cost(VOLATILE_REF_COST);
7383 format %{ "stlrh zr, $mem\t# short" %}
7384
7385 ins_encode(aarch64_enc_stlrh0(mem));
7386
7387 ins_pipe(pipe_class_memory);
7388 %}
7389
7390 // Store Integer
7391
7392 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set mem(StoreI mem src));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "stlrw $src, $mem\t# int" %}
7398
7399 ins_encode(aarch64_enc_stlrw(src, mem));
7400
7401 ins_pipe(pipe_class_memory);
7402 %}
7403
7404 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7405 %{
7406 match(Set mem(StoreI mem zero));
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "stlrw zr, $mem\t# int" %}
7410
7411 ins_encode(aarch64_enc_stlrw0(mem));
7412
7413 ins_pipe(pipe_class_memory);
7414 %}
7415
7416 // Store Long (64 bit signed)
7417 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set mem (StoreL mem src));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "stlr $src, $mem\t# int" %}
7423
7424 ins_encode(aarch64_enc_stlr(src, mem));
7425
7426 ins_pipe(pipe_class_memory);
7427 %}
7428
7429 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7430 %{
7431 match(Set mem (StoreL mem zero));
7432
7433 ins_cost(VOLATILE_REF_COST);
7434 format %{ "stlr zr, $mem\t# int" %}
7435
7436 ins_encode(aarch64_enc_stlr0(mem));
7437
7438 ins_pipe(pipe_class_memory);
7439 %}
7440
7441 // Store Pointer
7442 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7443 %{
7444 match(Set mem (StoreP mem src));
7445 predicate(n->as_Store()->barrier_data() == 0);
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "stlr $src, $mem\t# ptr" %}
7449
7450 ins_encode(aarch64_enc_stlr(src, mem));
7451
7452 ins_pipe(pipe_class_memory);
7453 %}
7454
7455 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set mem (StoreP mem zero));
7458 predicate(n->as_Store()->barrier_data() == 0);
7459
7460 ins_cost(VOLATILE_REF_COST);
7461 format %{ "stlr zr, $mem\t# ptr" %}
7462
7463 ins_encode(aarch64_enc_stlr0(mem));
7464
7465 ins_pipe(pipe_class_memory);
7466 %}
7467
7468 // Store Compressed Pointer
7469 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7470 %{
7471 match(Set mem (StoreN mem src));
7472 predicate(n->as_Store()->barrier_data() == 0);
7473
7474 ins_cost(VOLATILE_REF_COST);
7475 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7476
7477 ins_encode(aarch64_enc_stlrw(src, mem));
7478
7479 ins_pipe(pipe_class_memory);
7480 %}
7481
7482 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7483 %{
7484 match(Set mem (StoreN mem zero));
7485 predicate(n->as_Store()->barrier_data() == 0);
7486
7487 ins_cost(VOLATILE_REF_COST);
7488 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7489
7490 ins_encode(aarch64_enc_stlrw0(mem));
7491
7492 ins_pipe(pipe_class_memory);
7493 %}
7494
7495 // Store Float
7496 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7497 %{
7498 match(Set mem (StoreF mem src));
7499
7500 ins_cost(VOLATILE_REF_COST);
7501 format %{ "stlrs $src, $mem\t# float" %}
7502
7503 ins_encode( aarch64_enc_fstlrs(src, mem) );
7504
7505 ins_pipe(pipe_class_memory);
7506 %}
7507
7508 // TODO
7509 // implement storeImmF0 and storeFImmPacked
7510
7511 // Store Double
7512 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7513 %{
7514 match(Set mem (StoreD mem src));
7515
7516 ins_cost(VOLATILE_REF_COST);
7517 format %{ "stlrd $src, $mem\t# double" %}
7518
7519 ins_encode( aarch64_enc_fstlrd(src, mem) );
7520
7521 ins_pipe(pipe_class_memory);
7522 %}
7523
7524 // ---------------- end of volatile loads and stores ----------------
7525
7526 instruct cacheWB(indirect addr)
7527 %{
7528 predicate(VM_Version::supports_data_cache_line_flush());
7529 match(CacheWB addr);
7530
7531 ins_cost(100);
7532 format %{"cache wb $addr" %}
7533 ins_encode %{
7534 assert($addr->index_position() < 0, "should be");
7535 assert($addr$$disp == 0, "should be");
7536 __ cache_wb(Address($addr$$base$$Register, 0));
7537 %}
7538 ins_pipe(pipe_slow); // XXX
7539 %}
7540
7541 instruct cacheWBPreSync()
7542 %{
7543 predicate(VM_Version::supports_data_cache_line_flush());
7544 match(CacheWBPreSync);
7545
7546 ins_cost(100);
7547 format %{"cache wb presync" %}
7548 ins_encode %{
7549 __ cache_wbsync(true);
7550 %}
7551 ins_pipe(pipe_slow); // XXX
7552 %}
7553
7554 instruct cacheWBPostSync()
7555 %{
7556 predicate(VM_Version::supports_data_cache_line_flush());
7557 match(CacheWBPostSync);
7558
7559 ins_cost(100);
7560 format %{"cache wb postsync" %}
7561 ins_encode %{
7562 __ cache_wbsync(false);
7563 %}
7564 ins_pipe(pipe_slow); // XXX
7565 %}
7566
7567 // ============================================================================
7568 // BSWAP Instructions
7569
7570 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7571 match(Set dst (ReverseBytesI src));
7572
7573 ins_cost(INSN_COST);
7574 format %{ "revw $dst, $src" %}
7575
7576 ins_encode %{
7577 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7578 %}
7579
7580 ins_pipe(ialu_reg);
7581 %}
7582
7583 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7584 match(Set dst (ReverseBytesL src));
7585
7586 ins_cost(INSN_COST);
7587 format %{ "rev $dst, $src" %}
7588
7589 ins_encode %{
7590 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7591 %}
7592
7593 ins_pipe(ialu_reg);
7594 %}
7595
7596 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7597 match(Set dst (ReverseBytesUS src));
7598
7599 ins_cost(INSN_COST);
7600 format %{ "rev16w $dst, $src" %}
7601
7602 ins_encode %{
7603 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7604 %}
7605
7606 ins_pipe(ialu_reg);
7607 %}
7608
7609 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7610 match(Set dst (ReverseBytesS src));
7611
7612 ins_cost(INSN_COST);
7613 format %{ "rev16w $dst, $src\n\t"
7614 "sbfmw $dst, $dst, #0, #15" %}
7615
7616 ins_encode %{
7617 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7618 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7619 %}
7620
7621 ins_pipe(ialu_reg);
7622 %}
7623
7624 // ============================================================================
7625 // Zero Count Instructions
7626
7627 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7628 match(Set dst (CountLeadingZerosI src));
7629
7630 ins_cost(INSN_COST);
7631 format %{ "clzw $dst, $src" %}
7632 ins_encode %{
7633 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7634 %}
7635
7636 ins_pipe(ialu_reg);
7637 %}
7638
7639 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7640 match(Set dst (CountLeadingZerosL src));
7641
7642 ins_cost(INSN_COST);
7643 format %{ "clz $dst, $src" %}
7644 ins_encode %{
7645 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7646 %}
7647
7648 ins_pipe(ialu_reg);
7649 %}
7650
7651 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7652 match(Set dst (CountTrailingZerosI src));
7653
7654 ins_cost(INSN_COST * 2);
7655 format %{ "rbitw $dst, $src\n\t"
7656 "clzw $dst, $dst" %}
7657 ins_encode %{
7658 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7659 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7660 %}
7661
7662 ins_pipe(ialu_reg);
7663 %}
7664
7665 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7666 match(Set dst (CountTrailingZerosL src));
7667
7668 ins_cost(INSN_COST * 2);
7669 format %{ "rbit $dst, $src\n\t"
7670 "clz $dst, $dst" %}
7671 ins_encode %{
7672 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7673 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7674 %}
7675
7676 ins_pipe(ialu_reg);
7677 %}
7678
7679 //---------- Population Count Instructions -------------------------------------
7680 //
7681
7682 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7683 match(Set dst (PopCountI src));
7684 effect(TEMP tmp);
7685 ins_cost(INSN_COST * 13);
7686
7687 format %{ "movw $src, $src\n\t"
7688 "mov $tmp, $src\t# vector (1D)\n\t"
7689 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7690 "addv $tmp, $tmp\t# vector (8B)\n\t"
7691 "mov $dst, $tmp\t# vector (1D)" %}
7692 ins_encode %{
7693 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7694 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7695 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7696 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7697 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7698 %}
7699
7700 ins_pipe(pipe_class_default);
7701 %}
7702
7703 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7704 match(Set dst (PopCountI (LoadI mem)));
7705 effect(TEMP tmp);
7706 ins_cost(INSN_COST * 13);
7707
7708 format %{ "ldrs $tmp, $mem\n\t"
7709 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7710 "addv $tmp, $tmp\t# vector (8B)\n\t"
7711 "mov $dst, $tmp\t# vector (1D)" %}
7712 ins_encode %{
7713 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7714 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7715 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7716 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7717 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7718 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7719 %}
7720
7721 ins_pipe(pipe_class_default);
7722 %}
7723
7724 // Note: Long.bitCount(long) returns an int.
7725 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7726 match(Set dst (PopCountL src));
7727 effect(TEMP tmp);
7728 ins_cost(INSN_COST * 13);
7729
7730 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7731 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7732 "addv $tmp, $tmp\t# vector (8B)\n\t"
7733 "mov $dst, $tmp\t# vector (1D)" %}
7734 ins_encode %{
7735 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7736 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7737 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7738 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7739 %}
7740
7741 ins_pipe(pipe_class_default);
7742 %}
7743
7744 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7745 match(Set dst (PopCountL (LoadL mem)));
7746 effect(TEMP tmp);
7747 ins_cost(INSN_COST * 13);
7748
7749 format %{ "ldrd $tmp, $mem\n\t"
7750 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7751 "addv $tmp, $tmp\t# vector (8B)\n\t"
7752 "mov $dst, $tmp\t# vector (1D)" %}
7753 ins_encode %{
7754 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7755 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7756 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7757 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7758 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7759 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7760 %}
7761
7762 ins_pipe(pipe_class_default);
7763 %}
7764
7765 // ============================================================================
7766 // VerifyVectorAlignment Instruction
7767
7768 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7769 match(Set addr (VerifyVectorAlignment addr mask));
7770 effect(KILL cr);
7771 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7772 ins_encode %{
7773 Label Lskip;
7774 // check if masked bits of addr are zero
7775 __ tst($addr$$Register, $mask$$constant);
7776 __ br(Assembler::EQ, Lskip);
7777 __ stop("verify_vector_alignment found a misaligned vector memory access");
7778 __ bind(Lskip);
7779 %}
7780 ins_pipe(pipe_slow);
7781 %}
7782
7783 // ============================================================================
7784 // MemBar Instruction
7785
7786 instruct load_fence() %{
7787 match(LoadFence);
7788 ins_cost(VOLATILE_REF_COST);
7789
7790 format %{ "load_fence" %}
7791
7792 ins_encode %{
7793 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7794 %}
7795 ins_pipe(pipe_serial);
7796 %}
7797
7798 instruct unnecessary_membar_acquire() %{
7799 predicate(unnecessary_acquire(n));
7800 match(MemBarAcquire);
7801 ins_cost(0);
7802
7803 format %{ "membar_acquire (elided)" %}
7804
7805 ins_encode %{
7806 __ block_comment("membar_acquire (elided)");
7807 %}
7808
7809 ins_pipe(pipe_class_empty);
7810 %}
7811
7812 instruct membar_acquire() %{
7813 match(MemBarAcquire);
7814 ins_cost(VOLATILE_REF_COST);
7815
7816 format %{ "membar_acquire\n\t"
7817 "dmb ishld" %}
7818
7819 ins_encode %{
7820 __ block_comment("membar_acquire");
7821 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7822 %}
7823
7824 ins_pipe(pipe_serial);
7825 %}
7826
7827
7828 instruct membar_acquire_lock() %{
7829 match(MemBarAcquireLock);
7830 ins_cost(VOLATILE_REF_COST);
7831
7832 format %{ "membar_acquire_lock (elided)" %}
7833
7834 ins_encode %{
7835 __ block_comment("membar_acquire_lock (elided)");
7836 %}
7837
7838 ins_pipe(pipe_serial);
7839 %}
7840
7841 instruct store_fence() %{
7842 match(StoreFence);
7843 ins_cost(VOLATILE_REF_COST);
7844
7845 format %{ "store_fence" %}
7846
7847 ins_encode %{
7848 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7849 %}
7850 ins_pipe(pipe_serial);
7851 %}
7852
7853 instruct unnecessary_membar_release() %{
7854 predicate(unnecessary_release(n));
7855 match(MemBarRelease);
7856 ins_cost(0);
7857
7858 format %{ "membar_release (elided)" %}
7859
7860 ins_encode %{
7861 __ block_comment("membar_release (elided)");
7862 %}
7863 ins_pipe(pipe_serial);
7864 %}
7865
7866 instruct membar_release() %{
7867 match(MemBarRelease);
7868 ins_cost(VOLATILE_REF_COST);
7869
7870 format %{ "membar_release\n\t"
7871 "dmb ishst\n\tdmb ishld" %}
7872
7873 ins_encode %{
7874 __ block_comment("membar_release");
7875 // These will be merged if AlwaysMergeDMB is enabled.
7876 __ membar(Assembler::StoreStore);
7877 __ membar(Assembler::LoadStore);
7878 %}
7879 ins_pipe(pipe_serial);
7880 %}
7881
7882 instruct membar_storestore() %{
7883 match(MemBarStoreStore);
7884 match(StoreStoreFence);
7885 ins_cost(VOLATILE_REF_COST);
7886
7887 format %{ "MEMBAR-store-store" %}
7888
7889 ins_encode %{
7890 __ membar(Assembler::StoreStore);
7891 %}
7892 ins_pipe(pipe_serial);
7893 %}
7894
7895 instruct membar_release_lock() %{
7896 match(MemBarReleaseLock);
7897 ins_cost(VOLATILE_REF_COST);
7898
7899 format %{ "membar_release_lock (elided)" %}
7900
7901 ins_encode %{
7902 __ block_comment("membar_release_lock (elided)");
7903 %}
7904
7905 ins_pipe(pipe_serial);
7906 %}
7907
7908 instruct unnecessary_membar_volatile() %{
7909 predicate(unnecessary_volatile(n));
7910 match(MemBarVolatile);
7911 ins_cost(0);
7912
7913 format %{ "membar_volatile (elided)" %}
7914
7915 ins_encode %{
7916 __ block_comment("membar_volatile (elided)");
7917 %}
7918
7919 ins_pipe(pipe_serial);
7920 %}
7921
7922 instruct membar_volatile() %{
7923 match(MemBarVolatile);
7924 ins_cost(VOLATILE_REF_COST*100);
7925
7926 format %{ "membar_volatile\n\t"
7927 "dmb ish"%}
7928
7929 ins_encode %{
7930 __ block_comment("membar_volatile");
7931 __ membar(Assembler::StoreLoad);
7932 %}
7933
7934 ins_pipe(pipe_serial);
7935 %}
7936
7937 // ============================================================================
7938 // Cast/Convert Instructions
7939
7940 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7941 match(Set dst (CastX2P src));
7942
7943 ins_cost(INSN_COST);
7944 format %{ "mov $dst, $src\t# long -> ptr" %}
7945
7946 ins_encode %{
7947 if ($dst$$reg != $src$$reg) {
7948 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7949 }
7950 %}
7951
7952 ins_pipe(ialu_reg);
7953 %}
7954
7955 instruct castN2X(iRegLNoSp dst, iRegN src) %{
7956 match(Set dst (CastP2X src));
7957
7958 ins_cost(INSN_COST);
7959 format %{ "mov $dst, $src\t# ptr -> long" %}
7960
7961 ins_encode %{
7962 if ($dst$$reg != $src$$reg) {
7963 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7964 }
7965 %}
7966
7967 ins_pipe(ialu_reg);
7968 %}
7969
7970 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7971 match(Set dst (CastP2X src));
7972
7973 ins_cost(INSN_COST);
7974 format %{ "mov $dst, $src\t# ptr -> long" %}
7975
7976 ins_encode %{
7977 if ($dst$$reg != $src$$reg) {
7978 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7979 }
7980 %}
7981
7982 ins_pipe(ialu_reg);
7983 %}
7984
7985 // Convert oop into int for vectors alignment masking
7986 instruct convP2I(iRegINoSp dst, iRegP src) %{
7987 match(Set dst (ConvL2I (CastP2X src)));
7988
7989 ins_cost(INSN_COST);
7990 format %{ "movw $dst, $src\t# ptr -> int" %}
7991 ins_encode %{
7992 __ movw($dst$$Register, $src$$Register);
7993 %}
7994
7995 ins_pipe(ialu_reg);
7996 %}
7997
7998 // Convert compressed oop into int for vectors alignment masking
7999 // in case of 32bit oops (heap < 4Gb).
8000 instruct convN2I(iRegINoSp dst, iRegN src)
8001 %{
8002 predicate(CompressedOops::shift() == 0);
8003 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8004
8005 ins_cost(INSN_COST);
8006 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8007 ins_encode %{
8008 __ movw($dst$$Register, $src$$Register);
8009 %}
8010
8011 ins_pipe(ialu_reg);
8012 %}
8013
8014
8015 // Convert oop pointer into compressed form
8016 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8017 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8018 match(Set dst (EncodeP src));
8019 effect(KILL cr);
8020 ins_cost(INSN_COST * 3);
8021 format %{ "encode_heap_oop $dst, $src" %}
8022 ins_encode %{
8023 Register s = $src$$Register;
8024 Register d = $dst$$Register;
8025 __ encode_heap_oop(d, s);
8026 %}
8027 ins_pipe(ialu_reg);
8028 %}
8029
8030 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8031 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8032 match(Set dst (EncodeP src));
8033 ins_cost(INSN_COST * 3);
8034 format %{ "encode_heap_oop_not_null $dst, $src" %}
8035 ins_encode %{
8036 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8037 %}
8038 ins_pipe(ialu_reg);
8039 %}
8040
8041 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8042 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8043 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8044 match(Set dst (DecodeN src));
8045 ins_cost(INSN_COST * 3);
8046 format %{ "decode_heap_oop $dst, $src" %}
8047 ins_encode %{
8048 Register s = $src$$Register;
8049 Register d = $dst$$Register;
8050 __ decode_heap_oop(d, s);
8051 %}
8052 ins_pipe(ialu_reg);
8053 %}
8054
8055 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8056 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8057 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8058 match(Set dst (DecodeN src));
8059 ins_cost(INSN_COST * 3);
8060 format %{ "decode_heap_oop_not_null $dst, $src" %}
8061 ins_encode %{
8062 Register s = $src$$Register;
8063 Register d = $dst$$Register;
8064 __ decode_heap_oop_not_null(d, s);
8065 %}
8066 ins_pipe(ialu_reg);
8067 %}
8068
8069 // n.b. AArch64 implementations of encode_klass_not_null and
8070 // decode_klass_not_null do not modify the flags register so, unlike
8071 // Intel, we don't kill CR as a side effect here
8072
8073 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8074 match(Set dst (EncodePKlass src));
8075
8076 ins_cost(INSN_COST * 3);
8077 format %{ "encode_klass_not_null $dst,$src" %}
8078
8079 ins_encode %{
8080 Register src_reg = as_Register($src$$reg);
8081 Register dst_reg = as_Register($dst$$reg);
8082 __ encode_klass_not_null(dst_reg, src_reg);
8083 %}
8084
8085 ins_pipe(ialu_reg);
8086 %}
8087
8088 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8089 match(Set dst (DecodeNKlass src));
8090
8091 ins_cost(INSN_COST * 3);
8092 format %{ "decode_klass_not_null $dst,$src" %}
8093
8094 ins_encode %{
8095 Register src_reg = as_Register($src$$reg);
8096 Register dst_reg = as_Register($dst$$reg);
8097 if (dst_reg != src_reg) {
8098 __ decode_klass_not_null(dst_reg, src_reg);
8099 } else {
8100 __ decode_klass_not_null(dst_reg);
8101 }
8102 %}
8103
8104 ins_pipe(ialu_reg);
8105 %}
8106
8107 instruct checkCastPP(iRegPNoSp dst)
8108 %{
8109 match(Set dst (CheckCastPP dst));
8110
8111 size(0);
8112 format %{ "# checkcastPP of $dst" %}
8113 ins_encode(/* empty encoding */);
8114 ins_pipe(pipe_class_empty);
8115 %}
8116
8117 instruct castPP(iRegPNoSp dst)
8118 %{
8119 match(Set dst (CastPP dst));
8120
8121 size(0);
8122 format %{ "# castPP of $dst" %}
8123 ins_encode(/* empty encoding */);
8124 ins_pipe(pipe_class_empty);
8125 %}
8126
8127 instruct castII(iRegI dst)
8128 %{
8129 match(Set dst (CastII dst));
8130
8131 size(0);
8132 format %{ "# castII of $dst" %}
8133 ins_encode(/* empty encoding */);
8134 ins_cost(0);
8135 ins_pipe(pipe_class_empty);
8136 %}
8137
8138 instruct castLL(iRegL dst)
8139 %{
8140 match(Set dst (CastLL dst));
8141
8142 size(0);
8143 format %{ "# castLL of $dst" %}
8144 ins_encode(/* empty encoding */);
8145 ins_cost(0);
8146 ins_pipe(pipe_class_empty);
8147 %}
8148
8149 instruct castFF(vRegF dst)
8150 %{
8151 match(Set dst (CastFF dst));
8152
8153 size(0);
8154 format %{ "# castFF of $dst" %}
8155 ins_encode(/* empty encoding */);
8156 ins_cost(0);
8157 ins_pipe(pipe_class_empty);
8158 %}
8159
8160 instruct castDD(vRegD dst)
8161 %{
8162 match(Set dst (CastDD dst));
8163
8164 size(0);
8165 format %{ "# castDD of $dst" %}
8166 ins_encode(/* empty encoding */);
8167 ins_cost(0);
8168 ins_pipe(pipe_class_empty);
8169 %}
8170
8171 instruct castVV(vReg dst)
8172 %{
8173 match(Set dst (CastVV dst));
8174
8175 size(0);
8176 format %{ "# castVV of $dst" %}
8177 ins_encode(/* empty encoding */);
8178 ins_cost(0);
8179 ins_pipe(pipe_class_empty);
8180 %}
8181
8182 instruct castVVMask(pRegGov dst)
8183 %{
8184 match(Set dst (CastVV dst));
8185
8186 size(0);
8187 format %{ "# castVV of $dst" %}
8188 ins_encode(/* empty encoding */);
8189 ins_cost(0);
8190 ins_pipe(pipe_class_empty);
8191 %}
8192
8193 // ============================================================================
8194 // Atomic operation instructions
8195 //
8196
8197 // standard CompareAndSwapX when we are using barriers
8198 // these have higher priority than the rules selected by a predicate
8199
8200 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8201 // can't match them
8202
8203 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8204
8205 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8206 ins_cost(2 * VOLATILE_REF_COST);
8207
8208 effect(KILL cr);
8209
8210 format %{
8211 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8212 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8213 %}
8214
8215 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8216 aarch64_enc_cset_eq(res));
8217
8218 ins_pipe(pipe_slow);
8219 %}
8220
8221 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8222
8223 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8224 ins_cost(2 * VOLATILE_REF_COST);
8225
8226 effect(KILL cr);
8227
8228 format %{
8229 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8230 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8231 %}
8232
8233 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8234 aarch64_enc_cset_eq(res));
8235
8236 ins_pipe(pipe_slow);
8237 %}
8238
8239 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8240
8241 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8242 ins_cost(2 * VOLATILE_REF_COST);
8243
8244 effect(KILL cr);
8245
8246 format %{
8247 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8248 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8249 %}
8250
8251 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8252 aarch64_enc_cset_eq(res));
8253
8254 ins_pipe(pipe_slow);
8255 %}
8256
8257 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8258
8259 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8260 ins_cost(2 * VOLATILE_REF_COST);
8261
8262 effect(KILL cr);
8263
8264 format %{
8265 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8266 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8267 %}
8268
8269 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8270 aarch64_enc_cset_eq(res));
8271
8272 ins_pipe(pipe_slow);
8273 %}
8274
8275 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8276
8277 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8278 predicate(n->as_LoadStore()->barrier_data() == 0);
8279 ins_cost(2 * VOLATILE_REF_COST);
8280
8281 effect(KILL cr);
8282
8283 format %{
8284 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8285 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8286 %}
8287
8288 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8289 aarch64_enc_cset_eq(res));
8290
8291 ins_pipe(pipe_slow);
8292 %}
8293
8294 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8295
8296 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8297 predicate(n->as_LoadStore()->barrier_data() == 0);
8298 ins_cost(2 * VOLATILE_REF_COST);
8299
8300 effect(KILL cr);
8301
8302 format %{
8303 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8304 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8305 %}
8306
8307 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8308 aarch64_enc_cset_eq(res));
8309
8310 ins_pipe(pipe_slow);
8311 %}
8312
8313 // alternative CompareAndSwapX when we are eliding barriers
8314
8315 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8316
8317 predicate(needs_acquiring_load_exclusive(n));
8318 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8319 ins_cost(VOLATILE_REF_COST);
8320
8321 effect(KILL cr);
8322
8323 format %{
8324 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8325 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8326 %}
8327
8328 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8329 aarch64_enc_cset_eq(res));
8330
8331 ins_pipe(pipe_slow);
8332 %}
8333
8334 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8335
8336 predicate(needs_acquiring_load_exclusive(n));
8337 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8338 ins_cost(VOLATILE_REF_COST);
8339
8340 effect(KILL cr);
8341
8342 format %{
8343 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8344 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8345 %}
8346
8347 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8348 aarch64_enc_cset_eq(res));
8349
8350 ins_pipe(pipe_slow);
8351 %}
8352
8353 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8354
8355 predicate(needs_acquiring_load_exclusive(n));
8356 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8357 ins_cost(VOLATILE_REF_COST);
8358
8359 effect(KILL cr);
8360
8361 format %{
8362 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8363 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8364 %}
8365
8366 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8367 aarch64_enc_cset_eq(res));
8368
8369 ins_pipe(pipe_slow);
8370 %}
8371
8372 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8373
8374 predicate(needs_acquiring_load_exclusive(n));
8375 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8376 ins_cost(VOLATILE_REF_COST);
8377
8378 effect(KILL cr);
8379
8380 format %{
8381 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8382 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8383 %}
8384
8385 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8386 aarch64_enc_cset_eq(res));
8387
8388 ins_pipe(pipe_slow);
8389 %}
8390
8391 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8392
8393 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8394 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8395 ins_cost(VOLATILE_REF_COST);
8396
8397 effect(KILL cr);
8398
8399 format %{
8400 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8401 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8402 %}
8403
8404 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8405 aarch64_enc_cset_eq(res));
8406
8407 ins_pipe(pipe_slow);
8408 %}
8409
8410 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8411
8412 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8413 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8414 ins_cost(VOLATILE_REF_COST);
8415
8416 effect(KILL cr);
8417
8418 format %{
8419 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8420 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8421 %}
8422
8423 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8424 aarch64_enc_cset_eq(res));
8425
8426 ins_pipe(pipe_slow);
8427 %}
8428
8429
8430 // ---------------------------------------------------------------------
8431
8432 // BEGIN This section of the file is automatically generated. Do not edit --------------
8433
8434 // Sundry CAS operations. Note that release is always true,
8435 // regardless of the memory ordering of the CAS. This is because we
8436 // need the volatile case to be sequentially consistent but there is
8437 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8438 // can't check the type of memory ordering here, so we always emit a
8439 // STLXR.
8440
8441 // This section is generated from cas.m4
8442
8443
8444 // This pattern is generated automatically from cas.m4.
8445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8446 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8447 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8448 ins_cost(2 * VOLATILE_REF_COST);
8449 effect(TEMP_DEF res, KILL cr);
8450 format %{
8451 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8452 %}
8453 ins_encode %{
8454 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8455 Assembler::byte, /*acquire*/ false, /*release*/ true,
8456 /*weak*/ false, $res$$Register);
8457 __ sxtbw($res$$Register, $res$$Register);
8458 %}
8459 ins_pipe(pipe_slow);
8460 %}
8461
8462 // This pattern is generated automatically from cas.m4.
8463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8464 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8465 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8466 ins_cost(2 * VOLATILE_REF_COST);
8467 effect(TEMP_DEF res, KILL cr);
8468 format %{
8469 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8470 %}
8471 ins_encode %{
8472 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8473 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8474 /*weak*/ false, $res$$Register);
8475 __ sxthw($res$$Register, $res$$Register);
8476 %}
8477 ins_pipe(pipe_slow);
8478 %}
8479
8480 // This pattern is generated automatically from cas.m4.
8481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8482 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8483 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8484 ins_cost(2 * VOLATILE_REF_COST);
8485 effect(TEMP_DEF res, KILL cr);
8486 format %{
8487 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8488 %}
8489 ins_encode %{
8490 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8491 Assembler::word, /*acquire*/ false, /*release*/ true,
8492 /*weak*/ false, $res$$Register);
8493 %}
8494 ins_pipe(pipe_slow);
8495 %}
8496
8497 // This pattern is generated automatically from cas.m4.
8498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8499 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8500 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8501 ins_cost(2 * VOLATILE_REF_COST);
8502 effect(TEMP_DEF res, KILL cr);
8503 format %{
8504 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8505 %}
8506 ins_encode %{
8507 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8508 Assembler::xword, /*acquire*/ false, /*release*/ true,
8509 /*weak*/ false, $res$$Register);
8510 %}
8511 ins_pipe(pipe_slow);
8512 %}
8513
8514 // This pattern is generated automatically from cas.m4.
8515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8516 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8517 predicate(n->as_LoadStore()->barrier_data() == 0);
8518 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8519 ins_cost(2 * VOLATILE_REF_COST);
8520 effect(TEMP_DEF res, KILL cr);
8521 format %{
8522 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8523 %}
8524 ins_encode %{
8525 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8526 Assembler::word, /*acquire*/ false, /*release*/ true,
8527 /*weak*/ false, $res$$Register);
8528 %}
8529 ins_pipe(pipe_slow);
8530 %}
8531
8532 // This pattern is generated automatically from cas.m4.
8533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8534 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8535 predicate(n->as_LoadStore()->barrier_data() == 0);
8536 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8537 ins_cost(2 * VOLATILE_REF_COST);
8538 effect(TEMP_DEF res, KILL cr);
8539 format %{
8540 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8541 %}
8542 ins_encode %{
8543 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8544 Assembler::xword, /*acquire*/ false, /*release*/ true,
8545 /*weak*/ false, $res$$Register);
8546 %}
8547 ins_pipe(pipe_slow);
8548 %}
8549
8550 // This pattern is generated automatically from cas.m4.
8551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8552 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8553 predicate(needs_acquiring_load_exclusive(n));
8554 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8555 ins_cost(VOLATILE_REF_COST);
8556 effect(TEMP_DEF res, KILL cr);
8557 format %{
8558 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8559 %}
8560 ins_encode %{
8561 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8562 Assembler::byte, /*acquire*/ true, /*release*/ true,
8563 /*weak*/ false, $res$$Register);
8564 __ sxtbw($res$$Register, $res$$Register);
8565 %}
8566 ins_pipe(pipe_slow);
8567 %}
8568
8569 // This pattern is generated automatically from cas.m4.
8570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8571 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8572 predicate(needs_acquiring_load_exclusive(n));
8573 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8574 ins_cost(VOLATILE_REF_COST);
8575 effect(TEMP_DEF res, KILL cr);
8576 format %{
8577 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8578 %}
8579 ins_encode %{
8580 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8581 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8582 /*weak*/ false, $res$$Register);
8583 __ sxthw($res$$Register, $res$$Register);
8584 %}
8585 ins_pipe(pipe_slow);
8586 %}
8587
8588 // This pattern is generated automatically from cas.m4.
8589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8590 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8591 predicate(needs_acquiring_load_exclusive(n));
8592 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8593 ins_cost(VOLATILE_REF_COST);
8594 effect(TEMP_DEF res, KILL cr);
8595 format %{
8596 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8597 %}
8598 ins_encode %{
8599 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8600 Assembler::word, /*acquire*/ true, /*release*/ true,
8601 /*weak*/ false, $res$$Register);
8602 %}
8603 ins_pipe(pipe_slow);
8604 %}
8605
8606 // This pattern is generated automatically from cas.m4.
8607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8608 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8609 predicate(needs_acquiring_load_exclusive(n));
8610 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8611 ins_cost(VOLATILE_REF_COST);
8612 effect(TEMP_DEF res, KILL cr);
8613 format %{
8614 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8615 %}
8616 ins_encode %{
8617 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8618 Assembler::xword, /*acquire*/ true, /*release*/ true,
8619 /*weak*/ false, $res$$Register);
8620 %}
8621 ins_pipe(pipe_slow);
8622 %}
8623
8624 // This pattern is generated automatically from cas.m4.
8625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8626 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8627 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8628 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8629 ins_cost(VOLATILE_REF_COST);
8630 effect(TEMP_DEF res, KILL cr);
8631 format %{
8632 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8633 %}
8634 ins_encode %{
8635 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8636 Assembler::word, /*acquire*/ true, /*release*/ true,
8637 /*weak*/ false, $res$$Register);
8638 %}
8639 ins_pipe(pipe_slow);
8640 %}
8641
8642 // This pattern is generated automatically from cas.m4.
8643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8644 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8645 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8646 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8647 ins_cost(VOLATILE_REF_COST);
8648 effect(TEMP_DEF res, KILL cr);
8649 format %{
8650 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8651 %}
8652 ins_encode %{
8653 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8654 Assembler::xword, /*acquire*/ true, /*release*/ true,
8655 /*weak*/ false, $res$$Register);
8656 %}
8657 ins_pipe(pipe_slow);
8658 %}
8659
8660 // This pattern is generated automatically from cas.m4.
8661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8662 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8663 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8664 ins_cost(2 * VOLATILE_REF_COST);
8665 effect(KILL cr);
8666 format %{
8667 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8668 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8669 %}
8670 ins_encode %{
8671 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8672 Assembler::byte, /*acquire*/ false, /*release*/ true,
8673 /*weak*/ true, noreg);
8674 __ csetw($res$$Register, Assembler::EQ);
8675 %}
8676 ins_pipe(pipe_slow);
8677 %}
8678
8679 // This pattern is generated automatically from cas.m4.
8680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8681 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8682 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8683 ins_cost(2 * VOLATILE_REF_COST);
8684 effect(KILL cr);
8685 format %{
8686 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8687 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8688 %}
8689 ins_encode %{
8690 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8691 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8692 /*weak*/ true, noreg);
8693 __ csetw($res$$Register, Assembler::EQ);
8694 %}
8695 ins_pipe(pipe_slow);
8696 %}
8697
8698 // This pattern is generated automatically from cas.m4.
8699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8700 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8701 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8702 ins_cost(2 * VOLATILE_REF_COST);
8703 effect(KILL cr);
8704 format %{
8705 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8706 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8707 %}
8708 ins_encode %{
8709 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8710 Assembler::word, /*acquire*/ false, /*release*/ true,
8711 /*weak*/ true, noreg);
8712 __ csetw($res$$Register, Assembler::EQ);
8713 %}
8714 ins_pipe(pipe_slow);
8715 %}
8716
8717 // This pattern is generated automatically from cas.m4.
8718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8719 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8720 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8721 ins_cost(2 * VOLATILE_REF_COST);
8722 effect(KILL cr);
8723 format %{
8724 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8725 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8726 %}
8727 ins_encode %{
8728 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8729 Assembler::xword, /*acquire*/ false, /*release*/ true,
8730 /*weak*/ true, noreg);
8731 __ csetw($res$$Register, Assembler::EQ);
8732 %}
8733 ins_pipe(pipe_slow);
8734 %}
8735
8736 // This pattern is generated automatically from cas.m4.
8737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8738 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8739 predicate(n->as_LoadStore()->barrier_data() == 0);
8740 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8741 ins_cost(2 * VOLATILE_REF_COST);
8742 effect(KILL cr);
8743 format %{
8744 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8745 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8746 %}
8747 ins_encode %{
8748 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8749 Assembler::word, /*acquire*/ false, /*release*/ true,
8750 /*weak*/ true, noreg);
8751 __ csetw($res$$Register, Assembler::EQ);
8752 %}
8753 ins_pipe(pipe_slow);
8754 %}
8755
8756 // This pattern is generated automatically from cas.m4.
8757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8758 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8759 predicate(n->as_LoadStore()->barrier_data() == 0);
8760 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8761 ins_cost(2 * VOLATILE_REF_COST);
8762 effect(KILL cr);
8763 format %{
8764 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8765 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8766 %}
8767 ins_encode %{
8768 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8769 Assembler::xword, /*acquire*/ false, /*release*/ true,
8770 /*weak*/ true, noreg);
8771 __ csetw($res$$Register, Assembler::EQ);
8772 %}
8773 ins_pipe(pipe_slow);
8774 %}
8775
8776 // This pattern is generated automatically from cas.m4.
8777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8778 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8779 predicate(needs_acquiring_load_exclusive(n));
8780 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8781 ins_cost(VOLATILE_REF_COST);
8782 effect(KILL cr);
8783 format %{
8784 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8785 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8786 %}
8787 ins_encode %{
8788 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8789 Assembler::byte, /*acquire*/ true, /*release*/ true,
8790 /*weak*/ true, noreg);
8791 __ csetw($res$$Register, Assembler::EQ);
8792 %}
8793 ins_pipe(pipe_slow);
8794 %}
8795
8796 // This pattern is generated automatically from cas.m4.
8797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8798 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8799 predicate(needs_acquiring_load_exclusive(n));
8800 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8801 ins_cost(VOLATILE_REF_COST);
8802 effect(KILL cr);
8803 format %{
8804 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8805 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8806 %}
8807 ins_encode %{
8808 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8809 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8810 /*weak*/ true, noreg);
8811 __ csetw($res$$Register, Assembler::EQ);
8812 %}
8813 ins_pipe(pipe_slow);
8814 %}
8815
8816 // This pattern is generated automatically from cas.m4.
8817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8818 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8819 predicate(needs_acquiring_load_exclusive(n));
8820 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8821 ins_cost(VOLATILE_REF_COST);
8822 effect(KILL cr);
8823 format %{
8824 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8825 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8826 %}
8827 ins_encode %{
8828 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8829 Assembler::word, /*acquire*/ true, /*release*/ true,
8830 /*weak*/ true, noreg);
8831 __ csetw($res$$Register, Assembler::EQ);
8832 %}
8833 ins_pipe(pipe_slow);
8834 %}
8835
8836 // This pattern is generated automatically from cas.m4.
8837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8838 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8839 predicate(needs_acquiring_load_exclusive(n));
8840 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8841 ins_cost(VOLATILE_REF_COST);
8842 effect(KILL cr);
8843 format %{
8844 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8845 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8846 %}
8847 ins_encode %{
8848 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8849 Assembler::xword, /*acquire*/ true, /*release*/ true,
8850 /*weak*/ true, noreg);
8851 __ csetw($res$$Register, Assembler::EQ);
8852 %}
8853 ins_pipe(pipe_slow);
8854 %}
8855
8856 // This pattern is generated automatically from cas.m4.
8857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8858 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8859 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8860 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8861 ins_cost(VOLATILE_REF_COST);
8862 effect(KILL cr);
8863 format %{
8864 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8865 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8866 %}
8867 ins_encode %{
8868 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8869 Assembler::word, /*acquire*/ true, /*release*/ true,
8870 /*weak*/ true, noreg);
8871 __ csetw($res$$Register, Assembler::EQ);
8872 %}
8873 ins_pipe(pipe_slow);
8874 %}
8875
8876 // This pattern is generated automatically from cas.m4.
8877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8878 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8879 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8880 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8881 ins_cost(VOLATILE_REF_COST);
8882 effect(KILL cr);
8883 format %{
8884 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8885 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8886 %}
8887 ins_encode %{
8888 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8889 Assembler::xword, /*acquire*/ true, /*release*/ true,
8890 /*weak*/ true, noreg);
8891 __ csetw($res$$Register, Assembler::EQ);
8892 %}
8893 ins_pipe(pipe_slow);
8894 %}
8895
8896 // END This section of the file is automatically generated. Do not edit --------------
8897 // ---------------------------------------------------------------------
8898
8899 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8900 match(Set prev (GetAndSetI mem newv));
8901 ins_cost(2 * VOLATILE_REF_COST);
8902 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8903 ins_encode %{
8904 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8905 %}
8906 ins_pipe(pipe_serial);
8907 %}
8908
8909 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8910 match(Set prev (GetAndSetL mem newv));
8911 ins_cost(2 * VOLATILE_REF_COST);
8912 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8913 ins_encode %{
8914 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8915 %}
8916 ins_pipe(pipe_serial);
8917 %}
8918
8919 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8920 predicate(n->as_LoadStore()->barrier_data() == 0);
8921 match(Set prev (GetAndSetN mem newv));
8922 ins_cost(2 * VOLATILE_REF_COST);
8923 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8924 ins_encode %{
8925 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8926 %}
8927 ins_pipe(pipe_serial);
8928 %}
8929
8930 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8931 predicate(n->as_LoadStore()->barrier_data() == 0);
8932 match(Set prev (GetAndSetP mem newv));
8933 ins_cost(2 * VOLATILE_REF_COST);
8934 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8935 ins_encode %{
8936 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8937 %}
8938 ins_pipe(pipe_serial);
8939 %}
8940
8941 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8942 predicate(needs_acquiring_load_exclusive(n));
8943 match(Set prev (GetAndSetI mem newv));
8944 ins_cost(VOLATILE_REF_COST);
8945 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8946 ins_encode %{
8947 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8948 %}
8949 ins_pipe(pipe_serial);
8950 %}
8951
8952 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8953 predicate(needs_acquiring_load_exclusive(n));
8954 match(Set prev (GetAndSetL mem newv));
8955 ins_cost(VOLATILE_REF_COST);
8956 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8957 ins_encode %{
8958 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8959 %}
8960 ins_pipe(pipe_serial);
8961 %}
8962
8963 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8964 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8965 match(Set prev (GetAndSetN mem newv));
8966 ins_cost(VOLATILE_REF_COST);
8967 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8968 ins_encode %{
8969 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8970 %}
8971 ins_pipe(pipe_serial);
8972 %}
8973
8974 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8975 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8976 match(Set prev (GetAndSetP mem newv));
8977 ins_cost(VOLATILE_REF_COST);
8978 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8979 ins_encode %{
8980 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8981 %}
8982 ins_pipe(pipe_serial);
8983 %}
8984
8985
8986 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8987 match(Set newval (GetAndAddL mem incr));
8988 ins_cost(2 * VOLATILE_REF_COST + 1);
8989 format %{ "get_and_addL $newval, [$mem], $incr" %}
8990 ins_encode %{
8991 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8992 %}
8993 ins_pipe(pipe_serial);
8994 %}
8995
8996 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8997 predicate(n->as_LoadStore()->result_not_used());
8998 match(Set dummy (GetAndAddL mem incr));
8999 ins_cost(2 * VOLATILE_REF_COST);
9000 format %{ "get_and_addL [$mem], $incr" %}
9001 ins_encode %{
9002 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9003 %}
9004 ins_pipe(pipe_serial);
9005 %}
9006
9007 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9008 match(Set newval (GetAndAddL mem incr));
9009 ins_cost(2 * VOLATILE_REF_COST + 1);
9010 format %{ "get_and_addL $newval, [$mem], $incr" %}
9011 ins_encode %{
9012 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9013 %}
9014 ins_pipe(pipe_serial);
9015 %}
9016
9017 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9018 predicate(n->as_LoadStore()->result_not_used());
9019 match(Set dummy (GetAndAddL mem incr));
9020 ins_cost(2 * VOLATILE_REF_COST);
9021 format %{ "get_and_addL [$mem], $incr" %}
9022 ins_encode %{
9023 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9024 %}
9025 ins_pipe(pipe_serial);
9026 %}
9027
9028 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9029 match(Set newval (GetAndAddI mem incr));
9030 ins_cost(2 * VOLATILE_REF_COST + 1);
9031 format %{ "get_and_addI $newval, [$mem], $incr" %}
9032 ins_encode %{
9033 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9034 %}
9035 ins_pipe(pipe_serial);
9036 %}
9037
9038 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9039 predicate(n->as_LoadStore()->result_not_used());
9040 match(Set dummy (GetAndAddI mem incr));
9041 ins_cost(2 * VOLATILE_REF_COST);
9042 format %{ "get_and_addI [$mem], $incr" %}
9043 ins_encode %{
9044 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9045 %}
9046 ins_pipe(pipe_serial);
9047 %}
9048
9049 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9050 match(Set newval (GetAndAddI mem incr));
9051 ins_cost(2 * VOLATILE_REF_COST + 1);
9052 format %{ "get_and_addI $newval, [$mem], $incr" %}
9053 ins_encode %{
9054 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9055 %}
9056 ins_pipe(pipe_serial);
9057 %}
9058
9059 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9060 predicate(n->as_LoadStore()->result_not_used());
9061 match(Set dummy (GetAndAddI mem incr));
9062 ins_cost(2 * VOLATILE_REF_COST);
9063 format %{ "get_and_addI [$mem], $incr" %}
9064 ins_encode %{
9065 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9066 %}
9067 ins_pipe(pipe_serial);
9068 %}
9069
9070 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9071 predicate(needs_acquiring_load_exclusive(n));
9072 match(Set newval (GetAndAddL mem incr));
9073 ins_cost(VOLATILE_REF_COST + 1);
9074 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9075 ins_encode %{
9076 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9077 %}
9078 ins_pipe(pipe_serial);
9079 %}
9080
9081 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9082 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9083 match(Set dummy (GetAndAddL mem incr));
9084 ins_cost(VOLATILE_REF_COST);
9085 format %{ "get_and_addL_acq [$mem], $incr" %}
9086 ins_encode %{
9087 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9088 %}
9089 ins_pipe(pipe_serial);
9090 %}
9091
9092 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9093 predicate(needs_acquiring_load_exclusive(n));
9094 match(Set newval (GetAndAddL mem incr));
9095 ins_cost(VOLATILE_REF_COST + 1);
9096 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9097 ins_encode %{
9098 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9099 %}
9100 ins_pipe(pipe_serial);
9101 %}
9102
9103 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9104 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9105 match(Set dummy (GetAndAddL mem incr));
9106 ins_cost(VOLATILE_REF_COST);
9107 format %{ "get_and_addL_acq [$mem], $incr" %}
9108 ins_encode %{
9109 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9110 %}
9111 ins_pipe(pipe_serial);
9112 %}
9113
9114 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9115 predicate(needs_acquiring_load_exclusive(n));
9116 match(Set newval (GetAndAddI mem incr));
9117 ins_cost(VOLATILE_REF_COST + 1);
9118 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9119 ins_encode %{
9120 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9121 %}
9122 ins_pipe(pipe_serial);
9123 %}
9124
9125 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9126 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9127 match(Set dummy (GetAndAddI mem incr));
9128 ins_cost(VOLATILE_REF_COST);
9129 format %{ "get_and_addI_acq [$mem], $incr" %}
9130 ins_encode %{
9131 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9132 %}
9133 ins_pipe(pipe_serial);
9134 %}
9135
9136 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9137 predicate(needs_acquiring_load_exclusive(n));
9138 match(Set newval (GetAndAddI mem incr));
9139 ins_cost(VOLATILE_REF_COST + 1);
9140 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9141 ins_encode %{
9142 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9143 %}
9144 ins_pipe(pipe_serial);
9145 %}
9146
9147 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9148 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9149 match(Set dummy (GetAndAddI mem incr));
9150 ins_cost(VOLATILE_REF_COST);
9151 format %{ "get_and_addI_acq [$mem], $incr" %}
9152 ins_encode %{
9153 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9154 %}
9155 ins_pipe(pipe_serial);
9156 %}
9157
9158 // Manifest a CmpU result in an integer register.
9159 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9160 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9161 %{
9162 match(Set dst (CmpU3 src1 src2));
9163 effect(KILL flags);
9164
9165 ins_cost(INSN_COST * 3);
9166 format %{
9167 "cmpw $src1, $src2\n\t"
9168 "csetw $dst, ne\n\t"
9169 "cnegw $dst, lo\t# CmpU3(reg)"
9170 %}
9171 ins_encode %{
9172 __ cmpw($src1$$Register, $src2$$Register);
9173 __ csetw($dst$$Register, Assembler::NE);
9174 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9175 %}
9176
9177 ins_pipe(pipe_class_default);
9178 %}
9179
9180 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9181 %{
9182 match(Set dst (CmpU3 src1 src2));
9183 effect(KILL flags);
9184
9185 ins_cost(INSN_COST * 3);
9186 format %{
9187 "subsw zr, $src1, $src2\n\t"
9188 "csetw $dst, ne\n\t"
9189 "cnegw $dst, lo\t# CmpU3(imm)"
9190 %}
9191 ins_encode %{
9192 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9193 __ csetw($dst$$Register, Assembler::NE);
9194 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9195 %}
9196
9197 ins_pipe(pipe_class_default);
9198 %}
9199
9200 // Manifest a CmpUL result in an integer register.
9201 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9202 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9203 %{
9204 match(Set dst (CmpUL3 src1 src2));
9205 effect(KILL flags);
9206
9207 ins_cost(INSN_COST * 3);
9208 format %{
9209 "cmp $src1, $src2\n\t"
9210 "csetw $dst, ne\n\t"
9211 "cnegw $dst, lo\t# CmpUL3(reg)"
9212 %}
9213 ins_encode %{
9214 __ cmp($src1$$Register, $src2$$Register);
9215 __ csetw($dst$$Register, Assembler::NE);
9216 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9217 %}
9218
9219 ins_pipe(pipe_class_default);
9220 %}
9221
9222 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9223 %{
9224 match(Set dst (CmpUL3 src1 src2));
9225 effect(KILL flags);
9226
9227 ins_cost(INSN_COST * 3);
9228 format %{
9229 "subs zr, $src1, $src2\n\t"
9230 "csetw $dst, ne\n\t"
9231 "cnegw $dst, lo\t# CmpUL3(imm)"
9232 %}
9233 ins_encode %{
9234 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9235 __ csetw($dst$$Register, Assembler::NE);
9236 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9237 %}
9238
9239 ins_pipe(pipe_class_default);
9240 %}
9241
9242 // Manifest a CmpL result in an integer register.
9243 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9244 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9245 %{
9246 match(Set dst (CmpL3 src1 src2));
9247 effect(KILL flags);
9248
9249 ins_cost(INSN_COST * 3);
9250 format %{
9251 "cmp $src1, $src2\n\t"
9252 "csetw $dst, ne\n\t"
9253 "cnegw $dst, lt\t# CmpL3(reg)"
9254 %}
9255 ins_encode %{
9256 __ cmp($src1$$Register, $src2$$Register);
9257 __ csetw($dst$$Register, Assembler::NE);
9258 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9259 %}
9260
9261 ins_pipe(pipe_class_default);
9262 %}
9263
9264 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9265 %{
9266 match(Set dst (CmpL3 src1 src2));
9267 effect(KILL flags);
9268
9269 ins_cost(INSN_COST * 3);
9270 format %{
9271 "subs zr, $src1, $src2\n\t"
9272 "csetw $dst, ne\n\t"
9273 "cnegw $dst, lt\t# CmpL3(imm)"
9274 %}
9275 ins_encode %{
9276 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9277 __ csetw($dst$$Register, Assembler::NE);
9278 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9279 %}
9280
9281 ins_pipe(pipe_class_default);
9282 %}
9283
9284 // ============================================================================
9285 // Conditional Move Instructions
9286
9287 // n.b. we have identical rules for both a signed compare op (cmpOp)
9288 // and an unsigned compare op (cmpOpU). it would be nice if we could
9289 // define an op class which merged both inputs and use it to type the
9290 // argument to a single rule. unfortunatelyt his fails because the
9291 // opclass does not live up to the COND_INTER interface of its
9292 // component operands. When the generic code tries to negate the
9293 // operand it ends up running the generci Machoper::negate method
9294 // which throws a ShouldNotHappen. So, we have to provide two flavours
9295 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9296
9297 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9298 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9299
9300 ins_cost(INSN_COST * 2);
9301 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9302
9303 ins_encode %{
9304 __ cselw(as_Register($dst$$reg),
9305 as_Register($src2$$reg),
9306 as_Register($src1$$reg),
9307 (Assembler::Condition)$cmp$$cmpcode);
9308 %}
9309
9310 ins_pipe(icond_reg_reg);
9311 %}
9312
9313 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9314 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9315
9316 ins_cost(INSN_COST * 2);
9317 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9318
9319 ins_encode %{
9320 __ cselw(as_Register($dst$$reg),
9321 as_Register($src2$$reg),
9322 as_Register($src1$$reg),
9323 (Assembler::Condition)$cmp$$cmpcode);
9324 %}
9325
9326 ins_pipe(icond_reg_reg);
9327 %}
9328
9329 // special cases where one arg is zero
9330
9331 // n.b. this is selected in preference to the rule above because it
9332 // avoids loading constant 0 into a source register
9333
9334 // TODO
9335 // we ought only to be able to cull one of these variants as the ideal
9336 // transforms ought always to order the zero consistently (to left/right?)
9337
9338 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9339 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9340
9341 ins_cost(INSN_COST * 2);
9342 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9343
9344 ins_encode %{
9345 __ cselw(as_Register($dst$$reg),
9346 as_Register($src$$reg),
9347 zr,
9348 (Assembler::Condition)$cmp$$cmpcode);
9349 %}
9350
9351 ins_pipe(icond_reg);
9352 %}
9353
9354 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9355 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9356
9357 ins_cost(INSN_COST * 2);
9358 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9359
9360 ins_encode %{
9361 __ cselw(as_Register($dst$$reg),
9362 as_Register($src$$reg),
9363 zr,
9364 (Assembler::Condition)$cmp$$cmpcode);
9365 %}
9366
9367 ins_pipe(icond_reg);
9368 %}
9369
9370 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9371 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9372
9373 ins_cost(INSN_COST * 2);
9374 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9375
9376 ins_encode %{
9377 __ cselw(as_Register($dst$$reg),
9378 zr,
9379 as_Register($src$$reg),
9380 (Assembler::Condition)$cmp$$cmpcode);
9381 %}
9382
9383 ins_pipe(icond_reg);
9384 %}
9385
9386 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9387 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9388
9389 ins_cost(INSN_COST * 2);
9390 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9391
9392 ins_encode %{
9393 __ cselw(as_Register($dst$$reg),
9394 zr,
9395 as_Register($src$$reg),
9396 (Assembler::Condition)$cmp$$cmpcode);
9397 %}
9398
9399 ins_pipe(icond_reg);
9400 %}
9401
9402 // special case for creating a boolean 0 or 1
9403
9404 // n.b. this is selected in preference to the rule above because it
9405 // avoids loading constants 0 and 1 into a source register
9406
9407 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9408 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9409
9410 ins_cost(INSN_COST * 2);
9411 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9412
9413 ins_encode %{
9414 // equivalently
9415 // cset(as_Register($dst$$reg),
9416 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9417 __ csincw(as_Register($dst$$reg),
9418 zr,
9419 zr,
9420 (Assembler::Condition)$cmp$$cmpcode);
9421 %}
9422
9423 ins_pipe(icond_none);
9424 %}
9425
9426 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9427 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9428
9429 ins_cost(INSN_COST * 2);
9430 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9431
9432 ins_encode %{
9433 // equivalently
9434 // cset(as_Register($dst$$reg),
9435 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9436 __ csincw(as_Register($dst$$reg),
9437 zr,
9438 zr,
9439 (Assembler::Condition)$cmp$$cmpcode);
9440 %}
9441
9442 ins_pipe(icond_none);
9443 %}
9444
9445 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9446 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9447
9448 ins_cost(INSN_COST * 2);
9449 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9450
9451 ins_encode %{
9452 __ csel(as_Register($dst$$reg),
9453 as_Register($src2$$reg),
9454 as_Register($src1$$reg),
9455 (Assembler::Condition)$cmp$$cmpcode);
9456 %}
9457
9458 ins_pipe(icond_reg_reg);
9459 %}
9460
9461 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9462 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9463
9464 ins_cost(INSN_COST * 2);
9465 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9466
9467 ins_encode %{
9468 __ csel(as_Register($dst$$reg),
9469 as_Register($src2$$reg),
9470 as_Register($src1$$reg),
9471 (Assembler::Condition)$cmp$$cmpcode);
9472 %}
9473
9474 ins_pipe(icond_reg_reg);
9475 %}
9476
9477 // special cases where one arg is zero
9478
9479 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9480 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9481
9482 ins_cost(INSN_COST * 2);
9483 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9484
9485 ins_encode %{
9486 __ csel(as_Register($dst$$reg),
9487 zr,
9488 as_Register($src$$reg),
9489 (Assembler::Condition)$cmp$$cmpcode);
9490 %}
9491
9492 ins_pipe(icond_reg);
9493 %}
9494
9495 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9496 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9497
9498 ins_cost(INSN_COST * 2);
9499 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9500
9501 ins_encode %{
9502 __ csel(as_Register($dst$$reg),
9503 zr,
9504 as_Register($src$$reg),
9505 (Assembler::Condition)$cmp$$cmpcode);
9506 %}
9507
9508 ins_pipe(icond_reg);
9509 %}
9510
9511 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9512 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9513
9514 ins_cost(INSN_COST * 2);
9515 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9516
9517 ins_encode %{
9518 __ csel(as_Register($dst$$reg),
9519 as_Register($src$$reg),
9520 zr,
9521 (Assembler::Condition)$cmp$$cmpcode);
9522 %}
9523
9524 ins_pipe(icond_reg);
9525 %}
9526
9527 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9528 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9529
9530 ins_cost(INSN_COST * 2);
9531 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9532
9533 ins_encode %{
9534 __ csel(as_Register($dst$$reg),
9535 as_Register($src$$reg),
9536 zr,
9537 (Assembler::Condition)$cmp$$cmpcode);
9538 %}
9539
9540 ins_pipe(icond_reg);
9541 %}
9542
9543 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9544 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9545
9546 ins_cost(INSN_COST * 2);
9547 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9548
9549 ins_encode %{
9550 __ csel(as_Register($dst$$reg),
9551 as_Register($src2$$reg),
9552 as_Register($src1$$reg),
9553 (Assembler::Condition)$cmp$$cmpcode);
9554 %}
9555
9556 ins_pipe(icond_reg_reg);
9557 %}
9558
9559 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9560 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9561
9562 ins_cost(INSN_COST * 2);
9563 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9564
9565 ins_encode %{
9566 __ csel(as_Register($dst$$reg),
9567 as_Register($src2$$reg),
9568 as_Register($src1$$reg),
9569 (Assembler::Condition)$cmp$$cmpcode);
9570 %}
9571
9572 ins_pipe(icond_reg_reg);
9573 %}
9574
9575 // special cases where one arg is zero
9576
9577 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9578 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9579
9580 ins_cost(INSN_COST * 2);
9581 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9582
9583 ins_encode %{
9584 __ csel(as_Register($dst$$reg),
9585 zr,
9586 as_Register($src$$reg),
9587 (Assembler::Condition)$cmp$$cmpcode);
9588 %}
9589
9590 ins_pipe(icond_reg);
9591 %}
9592
9593 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9594 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9595
9596 ins_cost(INSN_COST * 2);
9597 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9598
9599 ins_encode %{
9600 __ csel(as_Register($dst$$reg),
9601 zr,
9602 as_Register($src$$reg),
9603 (Assembler::Condition)$cmp$$cmpcode);
9604 %}
9605
9606 ins_pipe(icond_reg);
9607 %}
9608
9609 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9610 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9611
9612 ins_cost(INSN_COST * 2);
9613 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9614
9615 ins_encode %{
9616 __ csel(as_Register($dst$$reg),
9617 as_Register($src$$reg),
9618 zr,
9619 (Assembler::Condition)$cmp$$cmpcode);
9620 %}
9621
9622 ins_pipe(icond_reg);
9623 %}
9624
9625 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9626 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9627
9628 ins_cost(INSN_COST * 2);
9629 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9630
9631 ins_encode %{
9632 __ csel(as_Register($dst$$reg),
9633 as_Register($src$$reg),
9634 zr,
9635 (Assembler::Condition)$cmp$$cmpcode);
9636 %}
9637
9638 ins_pipe(icond_reg);
9639 %}
9640
9641 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9642 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9643
9644 ins_cost(INSN_COST * 2);
9645 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9646
9647 ins_encode %{
9648 __ cselw(as_Register($dst$$reg),
9649 as_Register($src2$$reg),
9650 as_Register($src1$$reg),
9651 (Assembler::Condition)$cmp$$cmpcode);
9652 %}
9653
9654 ins_pipe(icond_reg_reg);
9655 %}
9656
9657 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9658 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9659
9660 ins_cost(INSN_COST * 2);
9661 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9662
9663 ins_encode %{
9664 __ cselw(as_Register($dst$$reg),
9665 as_Register($src2$$reg),
9666 as_Register($src1$$reg),
9667 (Assembler::Condition)$cmp$$cmpcode);
9668 %}
9669
9670 ins_pipe(icond_reg_reg);
9671 %}
9672
9673 // special cases where one arg is zero
9674
9675 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9676 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9677
9678 ins_cost(INSN_COST * 2);
9679 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9680
9681 ins_encode %{
9682 __ cselw(as_Register($dst$$reg),
9683 zr,
9684 as_Register($src$$reg),
9685 (Assembler::Condition)$cmp$$cmpcode);
9686 %}
9687
9688 ins_pipe(icond_reg);
9689 %}
9690
9691 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9692 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9693
9694 ins_cost(INSN_COST * 2);
9695 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9696
9697 ins_encode %{
9698 __ cselw(as_Register($dst$$reg),
9699 zr,
9700 as_Register($src$$reg),
9701 (Assembler::Condition)$cmp$$cmpcode);
9702 %}
9703
9704 ins_pipe(icond_reg);
9705 %}
9706
9707 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9708 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9709
9710 ins_cost(INSN_COST * 2);
9711 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9712
9713 ins_encode %{
9714 __ cselw(as_Register($dst$$reg),
9715 as_Register($src$$reg),
9716 zr,
9717 (Assembler::Condition)$cmp$$cmpcode);
9718 %}
9719
9720 ins_pipe(icond_reg);
9721 %}
9722
9723 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9724 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9725
9726 ins_cost(INSN_COST * 2);
9727 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9728
9729 ins_encode %{
9730 __ cselw(as_Register($dst$$reg),
9731 as_Register($src$$reg),
9732 zr,
9733 (Assembler::Condition)$cmp$$cmpcode);
9734 %}
9735
9736 ins_pipe(icond_reg);
9737 %}
9738
9739 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9740 %{
9741 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9742
9743 ins_cost(INSN_COST * 3);
9744
9745 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9746 ins_encode %{
9747 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9748 __ fcsels(as_FloatRegister($dst$$reg),
9749 as_FloatRegister($src2$$reg),
9750 as_FloatRegister($src1$$reg),
9751 cond);
9752 %}
9753
9754 ins_pipe(fp_cond_reg_reg_s);
9755 %}
9756
9757 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9758 %{
9759 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9760
9761 ins_cost(INSN_COST * 3);
9762
9763 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9764 ins_encode %{
9765 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9766 __ fcsels(as_FloatRegister($dst$$reg),
9767 as_FloatRegister($src2$$reg),
9768 as_FloatRegister($src1$$reg),
9769 cond);
9770 %}
9771
9772 ins_pipe(fp_cond_reg_reg_s);
9773 %}
9774
9775 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9776 %{
9777 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9778
9779 ins_cost(INSN_COST * 3);
9780
9781 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9782 ins_encode %{
9783 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9784 __ fcseld(as_FloatRegister($dst$$reg),
9785 as_FloatRegister($src2$$reg),
9786 as_FloatRegister($src1$$reg),
9787 cond);
9788 %}
9789
9790 ins_pipe(fp_cond_reg_reg_d);
9791 %}
9792
9793 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9794 %{
9795 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9796
9797 ins_cost(INSN_COST * 3);
9798
9799 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9800 ins_encode %{
9801 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9802 __ fcseld(as_FloatRegister($dst$$reg),
9803 as_FloatRegister($src2$$reg),
9804 as_FloatRegister($src1$$reg),
9805 cond);
9806 %}
9807
9808 ins_pipe(fp_cond_reg_reg_d);
9809 %}
9810
9811 // ============================================================================
9812 // Arithmetic Instructions
9813 //
9814
9815 // Integer Addition
9816
9817 // TODO
9818 // these currently employ operations which do not set CR and hence are
9819 // not flagged as killing CR but we would like to isolate the cases
9820 // where we want to set flags from those where we don't. need to work
9821 // out how to do that.
9822
9823 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9824 match(Set dst (AddI src1 src2));
9825
9826 ins_cost(INSN_COST);
9827 format %{ "addw $dst, $src1, $src2" %}
9828
9829 ins_encode %{
9830 __ addw(as_Register($dst$$reg),
9831 as_Register($src1$$reg),
9832 as_Register($src2$$reg));
9833 %}
9834
9835 ins_pipe(ialu_reg_reg);
9836 %}
9837
9838 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9839 match(Set dst (AddI src1 src2));
9840
9841 ins_cost(INSN_COST);
9842 format %{ "addw $dst, $src1, $src2" %}
9843
9844 // use opcode to indicate that this is an add not a sub
9845 opcode(0x0);
9846
9847 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9848
9849 ins_pipe(ialu_reg_imm);
9850 %}
9851
9852 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9853 match(Set dst (AddI (ConvL2I src1) src2));
9854
9855 ins_cost(INSN_COST);
9856 format %{ "addw $dst, $src1, $src2" %}
9857
9858 // use opcode to indicate that this is an add not a sub
9859 opcode(0x0);
9860
9861 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9862
9863 ins_pipe(ialu_reg_imm);
9864 %}
9865
9866 // Pointer Addition
9867 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9868 match(Set dst (AddP src1 src2));
9869
9870 ins_cost(INSN_COST);
9871 format %{ "add $dst, $src1, $src2\t# ptr" %}
9872
9873 ins_encode %{
9874 __ add(as_Register($dst$$reg),
9875 as_Register($src1$$reg),
9876 as_Register($src2$$reg));
9877 %}
9878
9879 ins_pipe(ialu_reg_reg);
9880 %}
9881
9882 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9883 match(Set dst (AddP src1 (ConvI2L src2)));
9884
9885 ins_cost(1.9 * INSN_COST);
9886 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9887
9888 ins_encode %{
9889 __ add(as_Register($dst$$reg),
9890 as_Register($src1$$reg),
9891 as_Register($src2$$reg), ext::sxtw);
9892 %}
9893
9894 ins_pipe(ialu_reg_reg);
9895 %}
9896
9897 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9898 match(Set dst (AddP src1 (LShiftL src2 scale)));
9899
9900 ins_cost(1.9 * INSN_COST);
9901 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9902
9903 ins_encode %{
9904 __ lea(as_Register($dst$$reg),
9905 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9906 Address::lsl($scale$$constant)));
9907 %}
9908
9909 ins_pipe(ialu_reg_reg_shift);
9910 %}
9911
9912 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9913 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9914
9915 ins_cost(1.9 * INSN_COST);
9916 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9917
9918 ins_encode %{
9919 __ lea(as_Register($dst$$reg),
9920 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9921 Address::sxtw($scale$$constant)));
9922 %}
9923
9924 ins_pipe(ialu_reg_reg_shift);
9925 %}
9926
9927 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9928 match(Set dst (LShiftL (ConvI2L src) scale));
9929
9930 ins_cost(INSN_COST);
9931 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9932
9933 ins_encode %{
9934 __ sbfiz(as_Register($dst$$reg),
9935 as_Register($src$$reg),
9936 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9937 %}
9938
9939 ins_pipe(ialu_reg_shift);
9940 %}
9941
9942 // Pointer Immediate Addition
9943 // n.b. this needs to be more expensive than using an indirect memory
9944 // operand
9945 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9946 match(Set dst (AddP src1 src2));
9947
9948 ins_cost(INSN_COST);
9949 format %{ "add $dst, $src1, $src2\t# ptr" %}
9950
9951 // use opcode to indicate that this is an add not a sub
9952 opcode(0x0);
9953
9954 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9955
9956 ins_pipe(ialu_reg_imm);
9957 %}
9958
9959 // Long Addition
9960 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9961
9962 match(Set dst (AddL src1 src2));
9963
9964 ins_cost(INSN_COST);
9965 format %{ "add $dst, $src1, $src2" %}
9966
9967 ins_encode %{
9968 __ add(as_Register($dst$$reg),
9969 as_Register($src1$$reg),
9970 as_Register($src2$$reg));
9971 %}
9972
9973 ins_pipe(ialu_reg_reg);
9974 %}
9975
9976 // No constant pool entries requiredLong Immediate Addition.
9977 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9978 match(Set dst (AddL src1 src2));
9979
9980 ins_cost(INSN_COST);
9981 format %{ "add $dst, $src1, $src2" %}
9982
9983 // use opcode to indicate that this is an add not a sub
9984 opcode(0x0);
9985
9986 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9987
9988 ins_pipe(ialu_reg_imm);
9989 %}
9990
9991 // Integer Subtraction
9992 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9993 match(Set dst (SubI src1 src2));
9994
9995 ins_cost(INSN_COST);
9996 format %{ "subw $dst, $src1, $src2" %}
9997
9998 ins_encode %{
9999 __ subw(as_Register($dst$$reg),
10000 as_Register($src1$$reg),
10001 as_Register($src2$$reg));
10002 %}
10003
10004 ins_pipe(ialu_reg_reg);
10005 %}
10006
10007 // Immediate Subtraction
10008 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10009 match(Set dst (SubI src1 src2));
10010
10011 ins_cost(INSN_COST);
10012 format %{ "subw $dst, $src1, $src2" %}
10013
10014 // use opcode to indicate that this is a sub not an add
10015 opcode(0x1);
10016
10017 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10018
10019 ins_pipe(ialu_reg_imm);
10020 %}
10021
10022 // Long Subtraction
10023 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10024
10025 match(Set dst (SubL src1 src2));
10026
10027 ins_cost(INSN_COST);
10028 format %{ "sub $dst, $src1, $src2" %}
10029
10030 ins_encode %{
10031 __ sub(as_Register($dst$$reg),
10032 as_Register($src1$$reg),
10033 as_Register($src2$$reg));
10034 %}
10035
10036 ins_pipe(ialu_reg_reg);
10037 %}
10038
10039 // No constant pool entries requiredLong Immediate Subtraction.
10040 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10041 match(Set dst (SubL src1 src2));
10042
10043 ins_cost(INSN_COST);
10044 format %{ "sub$dst, $src1, $src2" %}
10045
10046 // use opcode to indicate that this is a sub not an add
10047 opcode(0x1);
10048
10049 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10050
10051 ins_pipe(ialu_reg_imm);
10052 %}
10053
10054 // Integer Negation (special case for sub)
10055
10056 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10057 match(Set dst (SubI zero src));
10058
10059 ins_cost(INSN_COST);
10060 format %{ "negw $dst, $src\t# int" %}
10061
10062 ins_encode %{
10063 __ negw(as_Register($dst$$reg),
10064 as_Register($src$$reg));
10065 %}
10066
10067 ins_pipe(ialu_reg);
10068 %}
10069
10070 // Long Negation
10071
10072 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10073 match(Set dst (SubL zero src));
10074
10075 ins_cost(INSN_COST);
10076 format %{ "neg $dst, $src\t# long" %}
10077
10078 ins_encode %{
10079 __ neg(as_Register($dst$$reg),
10080 as_Register($src$$reg));
10081 %}
10082
10083 ins_pipe(ialu_reg);
10084 %}
10085
10086 // Integer Multiply
10087
10088 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10089 match(Set dst (MulI src1 src2));
10090
10091 ins_cost(INSN_COST * 3);
10092 format %{ "mulw $dst, $src1, $src2" %}
10093
10094 ins_encode %{
10095 __ mulw(as_Register($dst$$reg),
10096 as_Register($src1$$reg),
10097 as_Register($src2$$reg));
10098 %}
10099
10100 ins_pipe(imul_reg_reg);
10101 %}
10102
10103 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10104 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10105
10106 ins_cost(INSN_COST * 3);
10107 format %{ "smull $dst, $src1, $src2" %}
10108
10109 ins_encode %{
10110 __ smull(as_Register($dst$$reg),
10111 as_Register($src1$$reg),
10112 as_Register($src2$$reg));
10113 %}
10114
10115 ins_pipe(imul_reg_reg);
10116 %}
10117
10118 // Long Multiply
10119
10120 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10121 match(Set dst (MulL src1 src2));
10122
10123 ins_cost(INSN_COST * 5);
10124 format %{ "mul $dst, $src1, $src2" %}
10125
10126 ins_encode %{
10127 __ mul(as_Register($dst$$reg),
10128 as_Register($src1$$reg),
10129 as_Register($src2$$reg));
10130 %}
10131
10132 ins_pipe(lmul_reg_reg);
10133 %}
10134
10135 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10136 %{
10137 match(Set dst (MulHiL src1 src2));
10138
10139 ins_cost(INSN_COST * 7);
10140 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10141
10142 ins_encode %{
10143 __ smulh(as_Register($dst$$reg),
10144 as_Register($src1$$reg),
10145 as_Register($src2$$reg));
10146 %}
10147
10148 ins_pipe(lmul_reg_reg);
10149 %}
10150
10151 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10152 %{
10153 match(Set dst (UMulHiL src1 src2));
10154
10155 ins_cost(INSN_COST * 7);
10156 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10157
10158 ins_encode %{
10159 __ umulh(as_Register($dst$$reg),
10160 as_Register($src1$$reg),
10161 as_Register($src2$$reg));
10162 %}
10163
10164 ins_pipe(lmul_reg_reg);
10165 %}
10166
10167 // Combined Integer Multiply & Add/Sub
10168
10169 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10170 match(Set dst (AddI src3 (MulI src1 src2)));
10171
10172 ins_cost(INSN_COST * 3);
10173 format %{ "madd $dst, $src1, $src2, $src3" %}
10174
10175 ins_encode %{
10176 __ maddw(as_Register($dst$$reg),
10177 as_Register($src1$$reg),
10178 as_Register($src2$$reg),
10179 as_Register($src3$$reg));
10180 %}
10181
10182 ins_pipe(imac_reg_reg);
10183 %}
10184
10185 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10186 match(Set dst (SubI src3 (MulI src1 src2)));
10187
10188 ins_cost(INSN_COST * 3);
10189 format %{ "msub $dst, $src1, $src2, $src3" %}
10190
10191 ins_encode %{
10192 __ msubw(as_Register($dst$$reg),
10193 as_Register($src1$$reg),
10194 as_Register($src2$$reg),
10195 as_Register($src3$$reg));
10196 %}
10197
10198 ins_pipe(imac_reg_reg);
10199 %}
10200
10201 // Combined Integer Multiply & Neg
10202
10203 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10204 match(Set dst (MulI (SubI zero src1) src2));
10205
10206 ins_cost(INSN_COST * 3);
10207 format %{ "mneg $dst, $src1, $src2" %}
10208
10209 ins_encode %{
10210 __ mnegw(as_Register($dst$$reg),
10211 as_Register($src1$$reg),
10212 as_Register($src2$$reg));
10213 %}
10214
10215 ins_pipe(imac_reg_reg);
10216 %}
10217
10218 // Combined Long Multiply & Add/Sub
10219
10220 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10221 match(Set dst (AddL src3 (MulL src1 src2)));
10222
10223 ins_cost(INSN_COST * 5);
10224 format %{ "madd $dst, $src1, $src2, $src3" %}
10225
10226 ins_encode %{
10227 __ madd(as_Register($dst$$reg),
10228 as_Register($src1$$reg),
10229 as_Register($src2$$reg),
10230 as_Register($src3$$reg));
10231 %}
10232
10233 ins_pipe(lmac_reg_reg);
10234 %}
10235
10236 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10237 match(Set dst (SubL src3 (MulL src1 src2)));
10238
10239 ins_cost(INSN_COST * 5);
10240 format %{ "msub $dst, $src1, $src2, $src3" %}
10241
10242 ins_encode %{
10243 __ msub(as_Register($dst$$reg),
10244 as_Register($src1$$reg),
10245 as_Register($src2$$reg),
10246 as_Register($src3$$reg));
10247 %}
10248
10249 ins_pipe(lmac_reg_reg);
10250 %}
10251
10252 // Combined Long Multiply & Neg
10253
10254 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10255 match(Set dst (MulL (SubL zero src1) src2));
10256
10257 ins_cost(INSN_COST * 5);
10258 format %{ "mneg $dst, $src1, $src2" %}
10259
10260 ins_encode %{
10261 __ mneg(as_Register($dst$$reg),
10262 as_Register($src1$$reg),
10263 as_Register($src2$$reg));
10264 %}
10265
10266 ins_pipe(lmac_reg_reg);
10267 %}
10268
10269 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10270
10271 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10272 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10273
10274 ins_cost(INSN_COST * 3);
10275 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10276
10277 ins_encode %{
10278 __ smaddl(as_Register($dst$$reg),
10279 as_Register($src1$$reg),
10280 as_Register($src2$$reg),
10281 as_Register($src3$$reg));
10282 %}
10283
10284 ins_pipe(imac_reg_reg);
10285 %}
10286
10287 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10288 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10289
10290 ins_cost(INSN_COST * 3);
10291 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10292
10293 ins_encode %{
10294 __ smsubl(as_Register($dst$$reg),
10295 as_Register($src1$$reg),
10296 as_Register($src2$$reg),
10297 as_Register($src3$$reg));
10298 %}
10299
10300 ins_pipe(imac_reg_reg);
10301 %}
10302
10303 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10304 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10305
10306 ins_cost(INSN_COST * 3);
10307 format %{ "smnegl $dst, $src1, $src2" %}
10308
10309 ins_encode %{
10310 __ smnegl(as_Register($dst$$reg),
10311 as_Register($src1$$reg),
10312 as_Register($src2$$reg));
10313 %}
10314
10315 ins_pipe(imac_reg_reg);
10316 %}
10317
10318 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10319
10320 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10321 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10322
10323 ins_cost(INSN_COST * 5);
10324 format %{ "mulw rscratch1, $src1, $src2\n\t"
10325 "maddw $dst, $src3, $src4, rscratch1" %}
10326
10327 ins_encode %{
10328 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10329 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10330
10331 ins_pipe(imac_reg_reg);
10332 %}
10333
10334 // Integer Divide
10335
10336 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10337 match(Set dst (DivI src1 src2));
10338
10339 ins_cost(INSN_COST * 19);
10340 format %{ "sdivw $dst, $src1, $src2" %}
10341
10342 ins_encode(aarch64_enc_divw(dst, src1, src2));
10343 ins_pipe(idiv_reg_reg);
10344 %}
10345
10346 // Long Divide
10347
10348 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10349 match(Set dst (DivL src1 src2));
10350
10351 ins_cost(INSN_COST * 35);
10352 format %{ "sdiv $dst, $src1, $src2" %}
10353
10354 ins_encode(aarch64_enc_div(dst, src1, src2));
10355 ins_pipe(ldiv_reg_reg);
10356 %}
10357
10358 // Integer Remainder
10359
10360 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10361 match(Set dst (ModI src1 src2));
10362
10363 ins_cost(INSN_COST * 22);
10364 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10365 "msubw $dst, rscratch1, $src2, $src1" %}
10366
10367 ins_encode(aarch64_enc_modw(dst, src1, src2));
10368 ins_pipe(idiv_reg_reg);
10369 %}
10370
10371 // Long Remainder
10372
10373 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10374 match(Set dst (ModL src1 src2));
10375
10376 ins_cost(INSN_COST * 38);
10377 format %{ "sdiv rscratch1, $src1, $src2\n"
10378 "msub $dst, rscratch1, $src2, $src1" %}
10379
10380 ins_encode(aarch64_enc_mod(dst, src1, src2));
10381 ins_pipe(ldiv_reg_reg);
10382 %}
10383
10384 // Unsigned Integer Divide
10385
10386 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10387 match(Set dst (UDivI src1 src2));
10388
10389 ins_cost(INSN_COST * 19);
10390 format %{ "udivw $dst, $src1, $src2" %}
10391
10392 ins_encode %{
10393 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10394 %}
10395
10396 ins_pipe(idiv_reg_reg);
10397 %}
10398
10399 // Unsigned Long Divide
10400
10401 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10402 match(Set dst (UDivL src1 src2));
10403
10404 ins_cost(INSN_COST * 35);
10405 format %{ "udiv $dst, $src1, $src2" %}
10406
10407 ins_encode %{
10408 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10409 %}
10410
10411 ins_pipe(ldiv_reg_reg);
10412 %}
10413
10414 // Unsigned Integer Remainder
10415
10416 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10417 match(Set dst (UModI src1 src2));
10418
10419 ins_cost(INSN_COST * 22);
10420 format %{ "udivw rscratch1, $src1, $src2\n\t"
10421 "msubw $dst, rscratch1, $src2, $src1" %}
10422
10423 ins_encode %{
10424 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10425 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10426 %}
10427
10428 ins_pipe(idiv_reg_reg);
10429 %}
10430
10431 // Unsigned Long Remainder
10432
10433 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10434 match(Set dst (UModL src1 src2));
10435
10436 ins_cost(INSN_COST * 38);
10437 format %{ "udiv rscratch1, $src1, $src2\n"
10438 "msub $dst, rscratch1, $src2, $src1" %}
10439
10440 ins_encode %{
10441 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10442 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10443 %}
10444
10445 ins_pipe(ldiv_reg_reg);
10446 %}
10447
10448 // Integer Shifts
10449
10450 // Shift Left Register
10451 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10452 match(Set dst (LShiftI src1 src2));
10453
10454 ins_cost(INSN_COST * 2);
10455 format %{ "lslvw $dst, $src1, $src2" %}
10456
10457 ins_encode %{
10458 __ lslvw(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg));
10461 %}
10462
10463 ins_pipe(ialu_reg_reg_vshift);
10464 %}
10465
10466 // Shift Left Immediate
10467 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10468 match(Set dst (LShiftI src1 src2));
10469
10470 ins_cost(INSN_COST);
10471 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10472
10473 ins_encode %{
10474 __ lslw(as_Register($dst$$reg),
10475 as_Register($src1$$reg),
10476 $src2$$constant & 0x1f);
10477 %}
10478
10479 ins_pipe(ialu_reg_shift);
10480 %}
10481
10482 // Shift Right Logical Register
10483 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10484 match(Set dst (URShiftI src1 src2));
10485
10486 ins_cost(INSN_COST * 2);
10487 format %{ "lsrvw $dst, $src1, $src2" %}
10488
10489 ins_encode %{
10490 __ lsrvw(as_Register($dst$$reg),
10491 as_Register($src1$$reg),
10492 as_Register($src2$$reg));
10493 %}
10494
10495 ins_pipe(ialu_reg_reg_vshift);
10496 %}
10497
10498 // Shift Right Logical Immediate
10499 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10500 match(Set dst (URShiftI src1 src2));
10501
10502 ins_cost(INSN_COST);
10503 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10504
10505 ins_encode %{
10506 __ lsrw(as_Register($dst$$reg),
10507 as_Register($src1$$reg),
10508 $src2$$constant & 0x1f);
10509 %}
10510
10511 ins_pipe(ialu_reg_shift);
10512 %}
10513
10514 // Shift Right Arithmetic Register
10515 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10516 match(Set dst (RShiftI src1 src2));
10517
10518 ins_cost(INSN_COST * 2);
10519 format %{ "asrvw $dst, $src1, $src2" %}
10520
10521 ins_encode %{
10522 __ asrvw(as_Register($dst$$reg),
10523 as_Register($src1$$reg),
10524 as_Register($src2$$reg));
10525 %}
10526
10527 ins_pipe(ialu_reg_reg_vshift);
10528 %}
10529
10530 // Shift Right Arithmetic Immediate
10531 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10532 match(Set dst (RShiftI src1 src2));
10533
10534 ins_cost(INSN_COST);
10535 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10536
10537 ins_encode %{
10538 __ asrw(as_Register($dst$$reg),
10539 as_Register($src1$$reg),
10540 $src2$$constant & 0x1f);
10541 %}
10542
10543 ins_pipe(ialu_reg_shift);
10544 %}
10545
10546 // Combined Int Mask and Right Shift (using UBFM)
10547 // TODO
10548
10549 // Long Shifts
10550
10551 // Shift Left Register
10552 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10553 match(Set dst (LShiftL src1 src2));
10554
10555 ins_cost(INSN_COST * 2);
10556 format %{ "lslv $dst, $src1, $src2" %}
10557
10558 ins_encode %{
10559 __ lslv(as_Register($dst$$reg),
10560 as_Register($src1$$reg),
10561 as_Register($src2$$reg));
10562 %}
10563
10564 ins_pipe(ialu_reg_reg_vshift);
10565 %}
10566
10567 // Shift Left Immediate
10568 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10569 match(Set dst (LShiftL src1 src2));
10570
10571 ins_cost(INSN_COST);
10572 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10573
10574 ins_encode %{
10575 __ lsl(as_Register($dst$$reg),
10576 as_Register($src1$$reg),
10577 $src2$$constant & 0x3f);
10578 %}
10579
10580 ins_pipe(ialu_reg_shift);
10581 %}
10582
10583 // Shift Right Logical Register
10584 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10585 match(Set dst (URShiftL src1 src2));
10586
10587 ins_cost(INSN_COST * 2);
10588 format %{ "lsrv $dst, $src1, $src2" %}
10589
10590 ins_encode %{
10591 __ lsrv(as_Register($dst$$reg),
10592 as_Register($src1$$reg),
10593 as_Register($src2$$reg));
10594 %}
10595
10596 ins_pipe(ialu_reg_reg_vshift);
10597 %}
10598
10599 // Shift Right Logical Immediate
10600 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10601 match(Set dst (URShiftL src1 src2));
10602
10603 ins_cost(INSN_COST);
10604 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10605
10606 ins_encode %{
10607 __ lsr(as_Register($dst$$reg),
10608 as_Register($src1$$reg),
10609 $src2$$constant & 0x3f);
10610 %}
10611
10612 ins_pipe(ialu_reg_shift);
10613 %}
10614
10615 // A special-case pattern for card table stores.
10616 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10617 match(Set dst (URShiftL (CastP2X src1) src2));
10618
10619 ins_cost(INSN_COST);
10620 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10621
10622 ins_encode %{
10623 __ lsr(as_Register($dst$$reg),
10624 as_Register($src1$$reg),
10625 $src2$$constant & 0x3f);
10626 %}
10627
10628 ins_pipe(ialu_reg_shift);
10629 %}
10630
10631 // Shift Right Arithmetic Register
10632 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10633 match(Set dst (RShiftL src1 src2));
10634
10635 ins_cost(INSN_COST * 2);
10636 format %{ "asrv $dst, $src1, $src2" %}
10637
10638 ins_encode %{
10639 __ asrv(as_Register($dst$$reg),
10640 as_Register($src1$$reg),
10641 as_Register($src2$$reg));
10642 %}
10643
10644 ins_pipe(ialu_reg_reg_vshift);
10645 %}
10646
10647 // Shift Right Arithmetic Immediate
10648 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10649 match(Set dst (RShiftL src1 src2));
10650
10651 ins_cost(INSN_COST);
10652 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10653
10654 ins_encode %{
10655 __ asr(as_Register($dst$$reg),
10656 as_Register($src1$$reg),
10657 $src2$$constant & 0x3f);
10658 %}
10659
10660 ins_pipe(ialu_reg_shift);
10661 %}
10662
10663 // BEGIN This section of the file is automatically generated. Do not edit --------------
10664 // This section is generated from aarch64_ad.m4
10665
10666 // This pattern is automatically generated from aarch64_ad.m4
10667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10668 instruct regL_not_reg(iRegLNoSp dst,
10669 iRegL src1, immL_M1 m1,
10670 rFlagsReg cr) %{
10671 match(Set dst (XorL src1 m1));
10672 ins_cost(INSN_COST);
10673 format %{ "eon $dst, $src1, zr" %}
10674
10675 ins_encode %{
10676 __ eon(as_Register($dst$$reg),
10677 as_Register($src1$$reg),
10678 zr,
10679 Assembler::LSL, 0);
10680 %}
10681
10682 ins_pipe(ialu_reg);
10683 %}
10684
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 instruct regI_not_reg(iRegINoSp dst,
10688 iRegIorL2I src1, immI_M1 m1,
10689 rFlagsReg cr) %{
10690 match(Set dst (XorI src1 m1));
10691 ins_cost(INSN_COST);
10692 format %{ "eonw $dst, $src1, zr" %}
10693
10694 ins_encode %{
10695 __ eonw(as_Register($dst$$reg),
10696 as_Register($src1$$reg),
10697 zr,
10698 Assembler::LSL, 0);
10699 %}
10700
10701 ins_pipe(ialu_reg);
10702 %}
10703
10704 // This pattern is automatically generated from aarch64_ad.m4
10705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10706 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10707 immI0 zero, iRegIorL2I src1, immI src2) %{
10708 match(Set dst (SubI zero (URShiftI src1 src2)));
10709
10710 ins_cost(1.9 * INSN_COST);
10711 format %{ "negw $dst, $src1, LSR $src2" %}
10712
10713 ins_encode %{
10714 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10715 Assembler::LSR, $src2$$constant & 0x1f);
10716 %}
10717
10718 ins_pipe(ialu_reg_shift);
10719 %}
10720
10721 // This pattern is automatically generated from aarch64_ad.m4
10722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10723 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10724 immI0 zero, iRegIorL2I src1, immI src2) %{
10725 match(Set dst (SubI zero (RShiftI src1 src2)));
10726
10727 ins_cost(1.9 * INSN_COST);
10728 format %{ "negw $dst, $src1, ASR $src2" %}
10729
10730 ins_encode %{
10731 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10732 Assembler::ASR, $src2$$constant & 0x1f);
10733 %}
10734
10735 ins_pipe(ialu_reg_shift);
10736 %}
10737
10738 // This pattern is automatically generated from aarch64_ad.m4
10739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10740 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10741 immI0 zero, iRegIorL2I src1, immI src2) %{
10742 match(Set dst (SubI zero (LShiftI src1 src2)));
10743
10744 ins_cost(1.9 * INSN_COST);
10745 format %{ "negw $dst, $src1, LSL $src2" %}
10746
10747 ins_encode %{
10748 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10749 Assembler::LSL, $src2$$constant & 0x1f);
10750 %}
10751
10752 ins_pipe(ialu_reg_shift);
10753 %}
10754
10755 // This pattern is automatically generated from aarch64_ad.m4
10756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10757 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10758 immL0 zero, iRegL src1, immI src2) %{
10759 match(Set dst (SubL zero (URShiftL src1 src2)));
10760
10761 ins_cost(1.9 * INSN_COST);
10762 format %{ "neg $dst, $src1, LSR $src2" %}
10763
10764 ins_encode %{
10765 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10766 Assembler::LSR, $src2$$constant & 0x3f);
10767 %}
10768
10769 ins_pipe(ialu_reg_shift);
10770 %}
10771
10772 // This pattern is automatically generated from aarch64_ad.m4
10773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10774 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10775 immL0 zero, iRegL src1, immI src2) %{
10776 match(Set dst (SubL zero (RShiftL src1 src2)));
10777
10778 ins_cost(1.9 * INSN_COST);
10779 format %{ "neg $dst, $src1, ASR $src2" %}
10780
10781 ins_encode %{
10782 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10783 Assembler::ASR, $src2$$constant & 0x3f);
10784 %}
10785
10786 ins_pipe(ialu_reg_shift);
10787 %}
10788
10789 // This pattern is automatically generated from aarch64_ad.m4
10790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10791 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10792 immL0 zero, iRegL src1, immI src2) %{
10793 match(Set dst (SubL zero (LShiftL src1 src2)));
10794
10795 ins_cost(1.9 * INSN_COST);
10796 format %{ "neg $dst, $src1, LSL $src2" %}
10797
10798 ins_encode %{
10799 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10800 Assembler::LSL, $src2$$constant & 0x3f);
10801 %}
10802
10803 ins_pipe(ialu_reg_shift);
10804 %}
10805
10806 // This pattern is automatically generated from aarch64_ad.m4
10807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10808 instruct AndI_reg_not_reg(iRegINoSp dst,
10809 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10810 match(Set dst (AndI src1 (XorI src2 m1)));
10811 ins_cost(INSN_COST);
10812 format %{ "bicw $dst, $src1, $src2" %}
10813
10814 ins_encode %{
10815 __ bicw(as_Register($dst$$reg),
10816 as_Register($src1$$reg),
10817 as_Register($src2$$reg),
10818 Assembler::LSL, 0);
10819 %}
10820
10821 ins_pipe(ialu_reg_reg);
10822 %}
10823
10824 // This pattern is automatically generated from aarch64_ad.m4
10825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10826 instruct AndL_reg_not_reg(iRegLNoSp dst,
10827 iRegL src1, iRegL src2, immL_M1 m1) %{
10828 match(Set dst (AndL src1 (XorL src2 m1)));
10829 ins_cost(INSN_COST);
10830 format %{ "bic $dst, $src1, $src2" %}
10831
10832 ins_encode %{
10833 __ bic(as_Register($dst$$reg),
10834 as_Register($src1$$reg),
10835 as_Register($src2$$reg),
10836 Assembler::LSL, 0);
10837 %}
10838
10839 ins_pipe(ialu_reg_reg);
10840 %}
10841
10842 // This pattern is automatically generated from aarch64_ad.m4
10843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10844 instruct OrI_reg_not_reg(iRegINoSp dst,
10845 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10846 match(Set dst (OrI src1 (XorI src2 m1)));
10847 ins_cost(INSN_COST);
10848 format %{ "ornw $dst, $src1, $src2" %}
10849
10850 ins_encode %{
10851 __ ornw(as_Register($dst$$reg),
10852 as_Register($src1$$reg),
10853 as_Register($src2$$reg),
10854 Assembler::LSL, 0);
10855 %}
10856
10857 ins_pipe(ialu_reg_reg);
10858 %}
10859
10860 // This pattern is automatically generated from aarch64_ad.m4
10861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10862 instruct OrL_reg_not_reg(iRegLNoSp dst,
10863 iRegL src1, iRegL src2, immL_M1 m1) %{
10864 match(Set dst (OrL src1 (XorL src2 m1)));
10865 ins_cost(INSN_COST);
10866 format %{ "orn $dst, $src1, $src2" %}
10867
10868 ins_encode %{
10869 __ orn(as_Register($dst$$reg),
10870 as_Register($src1$$reg),
10871 as_Register($src2$$reg),
10872 Assembler::LSL, 0);
10873 %}
10874
10875 ins_pipe(ialu_reg_reg);
10876 %}
10877
10878 // This pattern is automatically generated from aarch64_ad.m4
10879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10880 instruct XorI_reg_not_reg(iRegINoSp dst,
10881 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10882 match(Set dst (XorI m1 (XorI src2 src1)));
10883 ins_cost(INSN_COST);
10884 format %{ "eonw $dst, $src1, $src2" %}
10885
10886 ins_encode %{
10887 __ eonw(as_Register($dst$$reg),
10888 as_Register($src1$$reg),
10889 as_Register($src2$$reg),
10890 Assembler::LSL, 0);
10891 %}
10892
10893 ins_pipe(ialu_reg_reg);
10894 %}
10895
10896 // This pattern is automatically generated from aarch64_ad.m4
10897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10898 instruct XorL_reg_not_reg(iRegLNoSp dst,
10899 iRegL src1, iRegL src2, immL_M1 m1) %{
10900 match(Set dst (XorL m1 (XorL src2 src1)));
10901 ins_cost(INSN_COST);
10902 format %{ "eon $dst, $src1, $src2" %}
10903
10904 ins_encode %{
10905 __ eon(as_Register($dst$$reg),
10906 as_Register($src1$$reg),
10907 as_Register($src2$$reg),
10908 Assembler::LSL, 0);
10909 %}
10910
10911 ins_pipe(ialu_reg_reg);
10912 %}
10913
10914 // This pattern is automatically generated from aarch64_ad.m4
10915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10916 // val & (-1 ^ (val >>> shift)) ==> bicw
10917 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10918 iRegIorL2I src1, iRegIorL2I src2,
10919 immI src3, immI_M1 src4) %{
10920 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10921 ins_cost(1.9 * INSN_COST);
10922 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10923
10924 ins_encode %{
10925 __ bicw(as_Register($dst$$reg),
10926 as_Register($src1$$reg),
10927 as_Register($src2$$reg),
10928 Assembler::LSR,
10929 $src3$$constant & 0x1f);
10930 %}
10931
10932 ins_pipe(ialu_reg_reg_shift);
10933 %}
10934
10935 // This pattern is automatically generated from aarch64_ad.m4
10936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10937 // val & (-1 ^ (val >>> shift)) ==> bic
10938 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10939 iRegL src1, iRegL src2,
10940 immI src3, immL_M1 src4) %{
10941 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10942 ins_cost(1.9 * INSN_COST);
10943 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10944
10945 ins_encode %{
10946 __ bic(as_Register($dst$$reg),
10947 as_Register($src1$$reg),
10948 as_Register($src2$$reg),
10949 Assembler::LSR,
10950 $src3$$constant & 0x3f);
10951 %}
10952
10953 ins_pipe(ialu_reg_reg_shift);
10954 %}
10955
10956 // This pattern is automatically generated from aarch64_ad.m4
10957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10958 // val & (-1 ^ (val >> shift)) ==> bicw
10959 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10960 iRegIorL2I src1, iRegIorL2I src2,
10961 immI src3, immI_M1 src4) %{
10962 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10963 ins_cost(1.9 * INSN_COST);
10964 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10965
10966 ins_encode %{
10967 __ bicw(as_Register($dst$$reg),
10968 as_Register($src1$$reg),
10969 as_Register($src2$$reg),
10970 Assembler::ASR,
10971 $src3$$constant & 0x1f);
10972 %}
10973
10974 ins_pipe(ialu_reg_reg_shift);
10975 %}
10976
10977 // This pattern is automatically generated from aarch64_ad.m4
10978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10979 // val & (-1 ^ (val >> shift)) ==> bic
10980 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10981 iRegL src1, iRegL src2,
10982 immI src3, immL_M1 src4) %{
10983 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10984 ins_cost(1.9 * INSN_COST);
10985 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10986
10987 ins_encode %{
10988 __ bic(as_Register($dst$$reg),
10989 as_Register($src1$$reg),
10990 as_Register($src2$$reg),
10991 Assembler::ASR,
10992 $src3$$constant & 0x3f);
10993 %}
10994
10995 ins_pipe(ialu_reg_reg_shift);
10996 %}
10997
10998 // This pattern is automatically generated from aarch64_ad.m4
10999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11000 // val & (-1 ^ (val ror shift)) ==> bicw
11001 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11002 iRegIorL2I src1, iRegIorL2I src2,
11003 immI src3, immI_M1 src4) %{
11004 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11005 ins_cost(1.9 * INSN_COST);
11006 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11007
11008 ins_encode %{
11009 __ bicw(as_Register($dst$$reg),
11010 as_Register($src1$$reg),
11011 as_Register($src2$$reg),
11012 Assembler::ROR,
11013 $src3$$constant & 0x1f);
11014 %}
11015
11016 ins_pipe(ialu_reg_reg_shift);
11017 %}
11018
11019 // This pattern is automatically generated from aarch64_ad.m4
11020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11021 // val & (-1 ^ (val ror shift)) ==> bic
11022 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11023 iRegL src1, iRegL src2,
11024 immI src3, immL_M1 src4) %{
11025 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11026 ins_cost(1.9 * INSN_COST);
11027 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11028
11029 ins_encode %{
11030 __ bic(as_Register($dst$$reg),
11031 as_Register($src1$$reg),
11032 as_Register($src2$$reg),
11033 Assembler::ROR,
11034 $src3$$constant & 0x3f);
11035 %}
11036
11037 ins_pipe(ialu_reg_reg_shift);
11038 %}
11039
11040 // This pattern is automatically generated from aarch64_ad.m4
11041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11042 // val & (-1 ^ (val << shift)) ==> bicw
11043 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11044 iRegIorL2I src1, iRegIorL2I src2,
11045 immI src3, immI_M1 src4) %{
11046 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11047 ins_cost(1.9 * INSN_COST);
11048 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11049
11050 ins_encode %{
11051 __ bicw(as_Register($dst$$reg),
11052 as_Register($src1$$reg),
11053 as_Register($src2$$reg),
11054 Assembler::LSL,
11055 $src3$$constant & 0x1f);
11056 %}
11057
11058 ins_pipe(ialu_reg_reg_shift);
11059 %}
11060
11061 // This pattern is automatically generated from aarch64_ad.m4
11062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11063 // val & (-1 ^ (val << shift)) ==> bic
11064 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11065 iRegL src1, iRegL src2,
11066 immI src3, immL_M1 src4) %{
11067 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11068 ins_cost(1.9 * INSN_COST);
11069 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11070
11071 ins_encode %{
11072 __ bic(as_Register($dst$$reg),
11073 as_Register($src1$$reg),
11074 as_Register($src2$$reg),
11075 Assembler::LSL,
11076 $src3$$constant & 0x3f);
11077 %}
11078
11079 ins_pipe(ialu_reg_reg_shift);
11080 %}
11081
11082 // This pattern is automatically generated from aarch64_ad.m4
11083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11084 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11085 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11086 iRegIorL2I src1, iRegIorL2I src2,
11087 immI src3, immI_M1 src4) %{
11088 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11089 ins_cost(1.9 * INSN_COST);
11090 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11091
11092 ins_encode %{
11093 __ eonw(as_Register($dst$$reg),
11094 as_Register($src1$$reg),
11095 as_Register($src2$$reg),
11096 Assembler::LSR,
11097 $src3$$constant & 0x1f);
11098 %}
11099
11100 ins_pipe(ialu_reg_reg_shift);
11101 %}
11102
11103 // This pattern is automatically generated from aarch64_ad.m4
11104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11105 // val ^ (-1 ^ (val >>> shift)) ==> eon
11106 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11107 iRegL src1, iRegL src2,
11108 immI src3, immL_M1 src4) %{
11109 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11110 ins_cost(1.9 * INSN_COST);
11111 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11112
11113 ins_encode %{
11114 __ eon(as_Register($dst$$reg),
11115 as_Register($src1$$reg),
11116 as_Register($src2$$reg),
11117 Assembler::LSR,
11118 $src3$$constant & 0x3f);
11119 %}
11120
11121 ins_pipe(ialu_reg_reg_shift);
11122 %}
11123
11124 // This pattern is automatically generated from aarch64_ad.m4
11125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11126 // val ^ (-1 ^ (val >> shift)) ==> eonw
11127 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11128 iRegIorL2I src1, iRegIorL2I src2,
11129 immI src3, immI_M1 src4) %{
11130 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11131 ins_cost(1.9 * INSN_COST);
11132 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11133
11134 ins_encode %{
11135 __ eonw(as_Register($dst$$reg),
11136 as_Register($src1$$reg),
11137 as_Register($src2$$reg),
11138 Assembler::ASR,
11139 $src3$$constant & 0x1f);
11140 %}
11141
11142 ins_pipe(ialu_reg_reg_shift);
11143 %}
11144
11145 // This pattern is automatically generated from aarch64_ad.m4
11146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11147 // val ^ (-1 ^ (val >> shift)) ==> eon
11148 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11149 iRegL src1, iRegL src2,
11150 immI src3, immL_M1 src4) %{
11151 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11152 ins_cost(1.9 * INSN_COST);
11153 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11154
11155 ins_encode %{
11156 __ eon(as_Register($dst$$reg),
11157 as_Register($src1$$reg),
11158 as_Register($src2$$reg),
11159 Assembler::ASR,
11160 $src3$$constant & 0x3f);
11161 %}
11162
11163 ins_pipe(ialu_reg_reg_shift);
11164 %}
11165
11166 // This pattern is automatically generated from aarch64_ad.m4
11167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11168 // val ^ (-1 ^ (val ror shift)) ==> eonw
11169 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11170 iRegIorL2I src1, iRegIorL2I src2,
11171 immI src3, immI_M1 src4) %{
11172 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11173 ins_cost(1.9 * INSN_COST);
11174 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11175
11176 ins_encode %{
11177 __ eonw(as_Register($dst$$reg),
11178 as_Register($src1$$reg),
11179 as_Register($src2$$reg),
11180 Assembler::ROR,
11181 $src3$$constant & 0x1f);
11182 %}
11183
11184 ins_pipe(ialu_reg_reg_shift);
11185 %}
11186
11187 // This pattern is automatically generated from aarch64_ad.m4
11188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11189 // val ^ (-1 ^ (val ror shift)) ==> eon
11190 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11191 iRegL src1, iRegL src2,
11192 immI src3, immL_M1 src4) %{
11193 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11194 ins_cost(1.9 * INSN_COST);
11195 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11196
11197 ins_encode %{
11198 __ eon(as_Register($dst$$reg),
11199 as_Register($src1$$reg),
11200 as_Register($src2$$reg),
11201 Assembler::ROR,
11202 $src3$$constant & 0x3f);
11203 %}
11204
11205 ins_pipe(ialu_reg_reg_shift);
11206 %}
11207
11208 // This pattern is automatically generated from aarch64_ad.m4
11209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11210 // val ^ (-1 ^ (val << shift)) ==> eonw
11211 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11212 iRegIorL2I src1, iRegIorL2I src2,
11213 immI src3, immI_M1 src4) %{
11214 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11215 ins_cost(1.9 * INSN_COST);
11216 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11217
11218 ins_encode %{
11219 __ eonw(as_Register($dst$$reg),
11220 as_Register($src1$$reg),
11221 as_Register($src2$$reg),
11222 Assembler::LSL,
11223 $src3$$constant & 0x1f);
11224 %}
11225
11226 ins_pipe(ialu_reg_reg_shift);
11227 %}
11228
11229 // This pattern is automatically generated from aarch64_ad.m4
11230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11231 // val ^ (-1 ^ (val << shift)) ==> eon
11232 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11233 iRegL src1, iRegL src2,
11234 immI src3, immL_M1 src4) %{
11235 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11236 ins_cost(1.9 * INSN_COST);
11237 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11238
11239 ins_encode %{
11240 __ eon(as_Register($dst$$reg),
11241 as_Register($src1$$reg),
11242 as_Register($src2$$reg),
11243 Assembler::LSL,
11244 $src3$$constant & 0x3f);
11245 %}
11246
11247 ins_pipe(ialu_reg_reg_shift);
11248 %}
11249
11250 // This pattern is automatically generated from aarch64_ad.m4
11251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11252 // val | (-1 ^ (val >>> shift)) ==> ornw
11253 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11254 iRegIorL2I src1, iRegIorL2I src2,
11255 immI src3, immI_M1 src4) %{
11256 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11257 ins_cost(1.9 * INSN_COST);
11258 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11259
11260 ins_encode %{
11261 __ ornw(as_Register($dst$$reg),
11262 as_Register($src1$$reg),
11263 as_Register($src2$$reg),
11264 Assembler::LSR,
11265 $src3$$constant & 0x1f);
11266 %}
11267
11268 ins_pipe(ialu_reg_reg_shift);
11269 %}
11270
11271 // This pattern is automatically generated from aarch64_ad.m4
11272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11273 // val | (-1 ^ (val >>> shift)) ==> orn
11274 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11275 iRegL src1, iRegL src2,
11276 immI src3, immL_M1 src4) %{
11277 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11278 ins_cost(1.9 * INSN_COST);
11279 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11280
11281 ins_encode %{
11282 __ orn(as_Register($dst$$reg),
11283 as_Register($src1$$reg),
11284 as_Register($src2$$reg),
11285 Assembler::LSR,
11286 $src3$$constant & 0x3f);
11287 %}
11288
11289 ins_pipe(ialu_reg_reg_shift);
11290 %}
11291
11292 // This pattern is automatically generated from aarch64_ad.m4
11293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11294 // val | (-1 ^ (val >> shift)) ==> ornw
11295 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11296 iRegIorL2I src1, iRegIorL2I src2,
11297 immI src3, immI_M1 src4) %{
11298 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11299 ins_cost(1.9 * INSN_COST);
11300 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11301
11302 ins_encode %{
11303 __ ornw(as_Register($dst$$reg),
11304 as_Register($src1$$reg),
11305 as_Register($src2$$reg),
11306 Assembler::ASR,
11307 $src3$$constant & 0x1f);
11308 %}
11309
11310 ins_pipe(ialu_reg_reg_shift);
11311 %}
11312
11313 // This pattern is automatically generated from aarch64_ad.m4
11314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11315 // val | (-1 ^ (val >> shift)) ==> orn
11316 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11317 iRegL src1, iRegL src2,
11318 immI src3, immL_M1 src4) %{
11319 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11320 ins_cost(1.9 * INSN_COST);
11321 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11322
11323 ins_encode %{
11324 __ orn(as_Register($dst$$reg),
11325 as_Register($src1$$reg),
11326 as_Register($src2$$reg),
11327 Assembler::ASR,
11328 $src3$$constant & 0x3f);
11329 %}
11330
11331 ins_pipe(ialu_reg_reg_shift);
11332 %}
11333
11334 // This pattern is automatically generated from aarch64_ad.m4
11335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11336 // val | (-1 ^ (val ror shift)) ==> ornw
11337 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11338 iRegIorL2I src1, iRegIorL2I src2,
11339 immI src3, immI_M1 src4) %{
11340 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11341 ins_cost(1.9 * INSN_COST);
11342 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11343
11344 ins_encode %{
11345 __ ornw(as_Register($dst$$reg),
11346 as_Register($src1$$reg),
11347 as_Register($src2$$reg),
11348 Assembler::ROR,
11349 $src3$$constant & 0x1f);
11350 %}
11351
11352 ins_pipe(ialu_reg_reg_shift);
11353 %}
11354
11355 // This pattern is automatically generated from aarch64_ad.m4
11356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11357 // val | (-1 ^ (val ror shift)) ==> orn
11358 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11359 iRegL src1, iRegL src2,
11360 immI src3, immL_M1 src4) %{
11361 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11362 ins_cost(1.9 * INSN_COST);
11363 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11364
11365 ins_encode %{
11366 __ orn(as_Register($dst$$reg),
11367 as_Register($src1$$reg),
11368 as_Register($src2$$reg),
11369 Assembler::ROR,
11370 $src3$$constant & 0x3f);
11371 %}
11372
11373 ins_pipe(ialu_reg_reg_shift);
11374 %}
11375
11376 // This pattern is automatically generated from aarch64_ad.m4
11377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11378 // val | (-1 ^ (val << shift)) ==> ornw
11379 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11380 iRegIorL2I src1, iRegIorL2I src2,
11381 immI src3, immI_M1 src4) %{
11382 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11383 ins_cost(1.9 * INSN_COST);
11384 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11385
11386 ins_encode %{
11387 __ ornw(as_Register($dst$$reg),
11388 as_Register($src1$$reg),
11389 as_Register($src2$$reg),
11390 Assembler::LSL,
11391 $src3$$constant & 0x1f);
11392 %}
11393
11394 ins_pipe(ialu_reg_reg_shift);
11395 %}
11396
11397 // This pattern is automatically generated from aarch64_ad.m4
11398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11399 // val | (-1 ^ (val << shift)) ==> orn
11400 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11401 iRegL src1, iRegL src2,
11402 immI src3, immL_M1 src4) %{
11403 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11404 ins_cost(1.9 * INSN_COST);
11405 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11406
11407 ins_encode %{
11408 __ orn(as_Register($dst$$reg),
11409 as_Register($src1$$reg),
11410 as_Register($src2$$reg),
11411 Assembler::LSL,
11412 $src3$$constant & 0x3f);
11413 %}
11414
11415 ins_pipe(ialu_reg_reg_shift);
11416 %}
11417
11418 // This pattern is automatically generated from aarch64_ad.m4
11419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11420 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11421 iRegIorL2I src1, iRegIorL2I src2,
11422 immI src3) %{
11423 match(Set dst (AndI src1 (URShiftI src2 src3)));
11424
11425 ins_cost(1.9 * INSN_COST);
11426 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11427
11428 ins_encode %{
11429 __ andw(as_Register($dst$$reg),
11430 as_Register($src1$$reg),
11431 as_Register($src2$$reg),
11432 Assembler::LSR,
11433 $src3$$constant & 0x1f);
11434 %}
11435
11436 ins_pipe(ialu_reg_reg_shift);
11437 %}
11438
11439 // This pattern is automatically generated from aarch64_ad.m4
11440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11441 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11442 iRegL src1, iRegL src2,
11443 immI src3) %{
11444 match(Set dst (AndL src1 (URShiftL src2 src3)));
11445
11446 ins_cost(1.9 * INSN_COST);
11447 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11448
11449 ins_encode %{
11450 __ andr(as_Register($dst$$reg),
11451 as_Register($src1$$reg),
11452 as_Register($src2$$reg),
11453 Assembler::LSR,
11454 $src3$$constant & 0x3f);
11455 %}
11456
11457 ins_pipe(ialu_reg_reg_shift);
11458 %}
11459
11460 // This pattern is automatically generated from aarch64_ad.m4
11461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11462 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11463 iRegIorL2I src1, iRegIorL2I src2,
11464 immI src3) %{
11465 match(Set dst (AndI src1 (RShiftI src2 src3)));
11466
11467 ins_cost(1.9 * INSN_COST);
11468 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11469
11470 ins_encode %{
11471 __ andw(as_Register($dst$$reg),
11472 as_Register($src1$$reg),
11473 as_Register($src2$$reg),
11474 Assembler::ASR,
11475 $src3$$constant & 0x1f);
11476 %}
11477
11478 ins_pipe(ialu_reg_reg_shift);
11479 %}
11480
11481 // This pattern is automatically generated from aarch64_ad.m4
11482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11483 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11484 iRegL src1, iRegL src2,
11485 immI src3) %{
11486 match(Set dst (AndL src1 (RShiftL src2 src3)));
11487
11488 ins_cost(1.9 * INSN_COST);
11489 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11490
11491 ins_encode %{
11492 __ andr(as_Register($dst$$reg),
11493 as_Register($src1$$reg),
11494 as_Register($src2$$reg),
11495 Assembler::ASR,
11496 $src3$$constant & 0x3f);
11497 %}
11498
11499 ins_pipe(ialu_reg_reg_shift);
11500 %}
11501
11502 // This pattern is automatically generated from aarch64_ad.m4
11503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11504 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11505 iRegIorL2I src1, iRegIorL2I src2,
11506 immI src3) %{
11507 match(Set dst (AndI src1 (LShiftI src2 src3)));
11508
11509 ins_cost(1.9 * INSN_COST);
11510 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11511
11512 ins_encode %{
11513 __ andw(as_Register($dst$$reg),
11514 as_Register($src1$$reg),
11515 as_Register($src2$$reg),
11516 Assembler::LSL,
11517 $src3$$constant & 0x1f);
11518 %}
11519
11520 ins_pipe(ialu_reg_reg_shift);
11521 %}
11522
11523 // This pattern is automatically generated from aarch64_ad.m4
11524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11525 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11526 iRegL src1, iRegL src2,
11527 immI src3) %{
11528 match(Set dst (AndL src1 (LShiftL src2 src3)));
11529
11530 ins_cost(1.9 * INSN_COST);
11531 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11532
11533 ins_encode %{
11534 __ andr(as_Register($dst$$reg),
11535 as_Register($src1$$reg),
11536 as_Register($src2$$reg),
11537 Assembler::LSL,
11538 $src3$$constant & 0x3f);
11539 %}
11540
11541 ins_pipe(ialu_reg_reg_shift);
11542 %}
11543
11544 // This pattern is automatically generated from aarch64_ad.m4
11545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11546 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11547 iRegIorL2I src1, iRegIorL2I src2,
11548 immI src3) %{
11549 match(Set dst (AndI src1 (RotateRight src2 src3)));
11550
11551 ins_cost(1.9 * INSN_COST);
11552 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11553
11554 ins_encode %{
11555 __ andw(as_Register($dst$$reg),
11556 as_Register($src1$$reg),
11557 as_Register($src2$$reg),
11558 Assembler::ROR,
11559 $src3$$constant & 0x1f);
11560 %}
11561
11562 ins_pipe(ialu_reg_reg_shift);
11563 %}
11564
11565 // This pattern is automatically generated from aarch64_ad.m4
11566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11567 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11568 iRegL src1, iRegL src2,
11569 immI src3) %{
11570 match(Set dst (AndL src1 (RotateRight src2 src3)));
11571
11572 ins_cost(1.9 * INSN_COST);
11573 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11574
11575 ins_encode %{
11576 __ andr(as_Register($dst$$reg),
11577 as_Register($src1$$reg),
11578 as_Register($src2$$reg),
11579 Assembler::ROR,
11580 $src3$$constant & 0x3f);
11581 %}
11582
11583 ins_pipe(ialu_reg_reg_shift);
11584 %}
11585
11586 // This pattern is automatically generated from aarch64_ad.m4
11587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11588 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11589 iRegIorL2I src1, iRegIorL2I src2,
11590 immI src3) %{
11591 match(Set dst (XorI src1 (URShiftI src2 src3)));
11592
11593 ins_cost(1.9 * INSN_COST);
11594 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11595
11596 ins_encode %{
11597 __ eorw(as_Register($dst$$reg),
11598 as_Register($src1$$reg),
11599 as_Register($src2$$reg),
11600 Assembler::LSR,
11601 $src3$$constant & 0x1f);
11602 %}
11603
11604 ins_pipe(ialu_reg_reg_shift);
11605 %}
11606
11607 // This pattern is automatically generated from aarch64_ad.m4
11608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11609 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11610 iRegL src1, iRegL src2,
11611 immI src3) %{
11612 match(Set dst (XorL src1 (URShiftL src2 src3)));
11613
11614 ins_cost(1.9 * INSN_COST);
11615 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11616
11617 ins_encode %{
11618 __ eor(as_Register($dst$$reg),
11619 as_Register($src1$$reg),
11620 as_Register($src2$$reg),
11621 Assembler::LSR,
11622 $src3$$constant & 0x3f);
11623 %}
11624
11625 ins_pipe(ialu_reg_reg_shift);
11626 %}
11627
11628 // This pattern is automatically generated from aarch64_ad.m4
11629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11630 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11631 iRegIorL2I src1, iRegIorL2I src2,
11632 immI src3) %{
11633 match(Set dst (XorI src1 (RShiftI src2 src3)));
11634
11635 ins_cost(1.9 * INSN_COST);
11636 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11637
11638 ins_encode %{
11639 __ eorw(as_Register($dst$$reg),
11640 as_Register($src1$$reg),
11641 as_Register($src2$$reg),
11642 Assembler::ASR,
11643 $src3$$constant & 0x1f);
11644 %}
11645
11646 ins_pipe(ialu_reg_reg_shift);
11647 %}
11648
11649 // This pattern is automatically generated from aarch64_ad.m4
11650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11651 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11652 iRegL src1, iRegL src2,
11653 immI src3) %{
11654 match(Set dst (XorL src1 (RShiftL src2 src3)));
11655
11656 ins_cost(1.9 * INSN_COST);
11657 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11658
11659 ins_encode %{
11660 __ eor(as_Register($dst$$reg),
11661 as_Register($src1$$reg),
11662 as_Register($src2$$reg),
11663 Assembler::ASR,
11664 $src3$$constant & 0x3f);
11665 %}
11666
11667 ins_pipe(ialu_reg_reg_shift);
11668 %}
11669
11670 // This pattern is automatically generated from aarch64_ad.m4
11671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11672 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11673 iRegIorL2I src1, iRegIorL2I src2,
11674 immI src3) %{
11675 match(Set dst (XorI src1 (LShiftI src2 src3)));
11676
11677 ins_cost(1.9 * INSN_COST);
11678 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11679
11680 ins_encode %{
11681 __ eorw(as_Register($dst$$reg),
11682 as_Register($src1$$reg),
11683 as_Register($src2$$reg),
11684 Assembler::LSL,
11685 $src3$$constant & 0x1f);
11686 %}
11687
11688 ins_pipe(ialu_reg_reg_shift);
11689 %}
11690
11691 // This pattern is automatically generated from aarch64_ad.m4
11692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11693 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11694 iRegL src1, iRegL src2,
11695 immI src3) %{
11696 match(Set dst (XorL src1 (LShiftL src2 src3)));
11697
11698 ins_cost(1.9 * INSN_COST);
11699 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11700
11701 ins_encode %{
11702 __ eor(as_Register($dst$$reg),
11703 as_Register($src1$$reg),
11704 as_Register($src2$$reg),
11705 Assembler::LSL,
11706 $src3$$constant & 0x3f);
11707 %}
11708
11709 ins_pipe(ialu_reg_reg_shift);
11710 %}
11711
11712 // This pattern is automatically generated from aarch64_ad.m4
11713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11714 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11715 iRegIorL2I src1, iRegIorL2I src2,
11716 immI src3) %{
11717 match(Set dst (XorI src1 (RotateRight src2 src3)));
11718
11719 ins_cost(1.9 * INSN_COST);
11720 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11721
11722 ins_encode %{
11723 __ eorw(as_Register($dst$$reg),
11724 as_Register($src1$$reg),
11725 as_Register($src2$$reg),
11726 Assembler::ROR,
11727 $src3$$constant & 0x1f);
11728 %}
11729
11730 ins_pipe(ialu_reg_reg_shift);
11731 %}
11732
11733 // This pattern is automatically generated from aarch64_ad.m4
11734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11735 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11736 iRegL src1, iRegL src2,
11737 immI src3) %{
11738 match(Set dst (XorL src1 (RotateRight src2 src3)));
11739
11740 ins_cost(1.9 * INSN_COST);
11741 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11742
11743 ins_encode %{
11744 __ eor(as_Register($dst$$reg),
11745 as_Register($src1$$reg),
11746 as_Register($src2$$reg),
11747 Assembler::ROR,
11748 $src3$$constant & 0x3f);
11749 %}
11750
11751 ins_pipe(ialu_reg_reg_shift);
11752 %}
11753
11754 // This pattern is automatically generated from aarch64_ad.m4
11755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11756 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11757 iRegIorL2I src1, iRegIorL2I src2,
11758 immI src3) %{
11759 match(Set dst (OrI src1 (URShiftI src2 src3)));
11760
11761 ins_cost(1.9 * INSN_COST);
11762 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11763
11764 ins_encode %{
11765 __ orrw(as_Register($dst$$reg),
11766 as_Register($src1$$reg),
11767 as_Register($src2$$reg),
11768 Assembler::LSR,
11769 $src3$$constant & 0x1f);
11770 %}
11771
11772 ins_pipe(ialu_reg_reg_shift);
11773 %}
11774
11775 // This pattern is automatically generated from aarch64_ad.m4
11776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11777 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11778 iRegL src1, iRegL src2,
11779 immI src3) %{
11780 match(Set dst (OrL src1 (URShiftL src2 src3)));
11781
11782 ins_cost(1.9 * INSN_COST);
11783 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11784
11785 ins_encode %{
11786 __ orr(as_Register($dst$$reg),
11787 as_Register($src1$$reg),
11788 as_Register($src2$$reg),
11789 Assembler::LSR,
11790 $src3$$constant & 0x3f);
11791 %}
11792
11793 ins_pipe(ialu_reg_reg_shift);
11794 %}
11795
11796 // This pattern is automatically generated from aarch64_ad.m4
11797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11798 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11799 iRegIorL2I src1, iRegIorL2I src2,
11800 immI src3) %{
11801 match(Set dst (OrI src1 (RShiftI src2 src3)));
11802
11803 ins_cost(1.9 * INSN_COST);
11804 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11805
11806 ins_encode %{
11807 __ orrw(as_Register($dst$$reg),
11808 as_Register($src1$$reg),
11809 as_Register($src2$$reg),
11810 Assembler::ASR,
11811 $src3$$constant & 0x1f);
11812 %}
11813
11814 ins_pipe(ialu_reg_reg_shift);
11815 %}
11816
11817 // This pattern is automatically generated from aarch64_ad.m4
11818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11819 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11820 iRegL src1, iRegL src2,
11821 immI src3) %{
11822 match(Set dst (OrL src1 (RShiftL src2 src3)));
11823
11824 ins_cost(1.9 * INSN_COST);
11825 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11826
11827 ins_encode %{
11828 __ orr(as_Register($dst$$reg),
11829 as_Register($src1$$reg),
11830 as_Register($src2$$reg),
11831 Assembler::ASR,
11832 $src3$$constant & 0x3f);
11833 %}
11834
11835 ins_pipe(ialu_reg_reg_shift);
11836 %}
11837
11838 // This pattern is automatically generated from aarch64_ad.m4
11839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11840 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11841 iRegIorL2I src1, iRegIorL2I src2,
11842 immI src3) %{
11843 match(Set dst (OrI src1 (LShiftI src2 src3)));
11844
11845 ins_cost(1.9 * INSN_COST);
11846 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11847
11848 ins_encode %{
11849 __ orrw(as_Register($dst$$reg),
11850 as_Register($src1$$reg),
11851 as_Register($src2$$reg),
11852 Assembler::LSL,
11853 $src3$$constant & 0x1f);
11854 %}
11855
11856 ins_pipe(ialu_reg_reg_shift);
11857 %}
11858
11859 // This pattern is automatically generated from aarch64_ad.m4
11860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11861 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11862 iRegL src1, iRegL src2,
11863 immI src3) %{
11864 match(Set dst (OrL src1 (LShiftL src2 src3)));
11865
11866 ins_cost(1.9 * INSN_COST);
11867 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11868
11869 ins_encode %{
11870 __ orr(as_Register($dst$$reg),
11871 as_Register($src1$$reg),
11872 as_Register($src2$$reg),
11873 Assembler::LSL,
11874 $src3$$constant & 0x3f);
11875 %}
11876
11877 ins_pipe(ialu_reg_reg_shift);
11878 %}
11879
11880 // This pattern is automatically generated from aarch64_ad.m4
11881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11882 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11883 iRegIorL2I src1, iRegIorL2I src2,
11884 immI src3) %{
11885 match(Set dst (OrI src1 (RotateRight src2 src3)));
11886
11887 ins_cost(1.9 * INSN_COST);
11888 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11889
11890 ins_encode %{
11891 __ orrw(as_Register($dst$$reg),
11892 as_Register($src1$$reg),
11893 as_Register($src2$$reg),
11894 Assembler::ROR,
11895 $src3$$constant & 0x1f);
11896 %}
11897
11898 ins_pipe(ialu_reg_reg_shift);
11899 %}
11900
11901 // This pattern is automatically generated from aarch64_ad.m4
11902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11903 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11904 iRegL src1, iRegL src2,
11905 immI src3) %{
11906 match(Set dst (OrL src1 (RotateRight src2 src3)));
11907
11908 ins_cost(1.9 * INSN_COST);
11909 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11910
11911 ins_encode %{
11912 __ orr(as_Register($dst$$reg),
11913 as_Register($src1$$reg),
11914 as_Register($src2$$reg),
11915 Assembler::ROR,
11916 $src3$$constant & 0x3f);
11917 %}
11918
11919 ins_pipe(ialu_reg_reg_shift);
11920 %}
11921
11922 // This pattern is automatically generated from aarch64_ad.m4
11923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11924 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11925 iRegIorL2I src1, iRegIorL2I src2,
11926 immI src3) %{
11927 match(Set dst (AddI src1 (URShiftI src2 src3)));
11928
11929 ins_cost(1.9 * INSN_COST);
11930 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11931
11932 ins_encode %{
11933 __ addw(as_Register($dst$$reg),
11934 as_Register($src1$$reg),
11935 as_Register($src2$$reg),
11936 Assembler::LSR,
11937 $src3$$constant & 0x1f);
11938 %}
11939
11940 ins_pipe(ialu_reg_reg_shift);
11941 %}
11942
11943 // This pattern is automatically generated from aarch64_ad.m4
11944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11945 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11946 iRegL src1, iRegL src2,
11947 immI src3) %{
11948 match(Set dst (AddL src1 (URShiftL src2 src3)));
11949
11950 ins_cost(1.9 * INSN_COST);
11951 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11952
11953 ins_encode %{
11954 __ add(as_Register($dst$$reg),
11955 as_Register($src1$$reg),
11956 as_Register($src2$$reg),
11957 Assembler::LSR,
11958 $src3$$constant & 0x3f);
11959 %}
11960
11961 ins_pipe(ialu_reg_reg_shift);
11962 %}
11963
11964 // This pattern is automatically generated from aarch64_ad.m4
11965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11966 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11967 iRegIorL2I src1, iRegIorL2I src2,
11968 immI src3) %{
11969 match(Set dst (AddI src1 (RShiftI src2 src3)));
11970
11971 ins_cost(1.9 * INSN_COST);
11972 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11973
11974 ins_encode %{
11975 __ addw(as_Register($dst$$reg),
11976 as_Register($src1$$reg),
11977 as_Register($src2$$reg),
11978 Assembler::ASR,
11979 $src3$$constant & 0x1f);
11980 %}
11981
11982 ins_pipe(ialu_reg_reg_shift);
11983 %}
11984
11985 // This pattern is automatically generated from aarch64_ad.m4
11986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11987 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11988 iRegL src1, iRegL src2,
11989 immI src3) %{
11990 match(Set dst (AddL src1 (RShiftL src2 src3)));
11991
11992 ins_cost(1.9 * INSN_COST);
11993 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11994
11995 ins_encode %{
11996 __ add(as_Register($dst$$reg),
11997 as_Register($src1$$reg),
11998 as_Register($src2$$reg),
11999 Assembler::ASR,
12000 $src3$$constant & 0x3f);
12001 %}
12002
12003 ins_pipe(ialu_reg_reg_shift);
12004 %}
12005
12006 // This pattern is automatically generated from aarch64_ad.m4
12007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12008 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12009 iRegIorL2I src1, iRegIorL2I src2,
12010 immI src3) %{
12011 match(Set dst (AddI src1 (LShiftI src2 src3)));
12012
12013 ins_cost(1.9 * INSN_COST);
12014 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12015
12016 ins_encode %{
12017 __ addw(as_Register($dst$$reg),
12018 as_Register($src1$$reg),
12019 as_Register($src2$$reg),
12020 Assembler::LSL,
12021 $src3$$constant & 0x1f);
12022 %}
12023
12024 ins_pipe(ialu_reg_reg_shift);
12025 %}
12026
12027 // This pattern is automatically generated from aarch64_ad.m4
12028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12029 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12030 iRegL src1, iRegL src2,
12031 immI src3) %{
12032 match(Set dst (AddL src1 (LShiftL src2 src3)));
12033
12034 ins_cost(1.9 * INSN_COST);
12035 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12036
12037 ins_encode %{
12038 __ add(as_Register($dst$$reg),
12039 as_Register($src1$$reg),
12040 as_Register($src2$$reg),
12041 Assembler::LSL,
12042 $src3$$constant & 0x3f);
12043 %}
12044
12045 ins_pipe(ialu_reg_reg_shift);
12046 %}
12047
12048 // This pattern is automatically generated from aarch64_ad.m4
12049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12050 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12051 iRegIorL2I src1, iRegIorL2I src2,
12052 immI src3) %{
12053 match(Set dst (SubI src1 (URShiftI src2 src3)));
12054
12055 ins_cost(1.9 * INSN_COST);
12056 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12057
12058 ins_encode %{
12059 __ subw(as_Register($dst$$reg),
12060 as_Register($src1$$reg),
12061 as_Register($src2$$reg),
12062 Assembler::LSR,
12063 $src3$$constant & 0x1f);
12064 %}
12065
12066 ins_pipe(ialu_reg_reg_shift);
12067 %}
12068
12069 // This pattern is automatically generated from aarch64_ad.m4
12070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12071 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12072 iRegL src1, iRegL src2,
12073 immI src3) %{
12074 match(Set dst (SubL src1 (URShiftL src2 src3)));
12075
12076 ins_cost(1.9 * INSN_COST);
12077 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12078
12079 ins_encode %{
12080 __ sub(as_Register($dst$$reg),
12081 as_Register($src1$$reg),
12082 as_Register($src2$$reg),
12083 Assembler::LSR,
12084 $src3$$constant & 0x3f);
12085 %}
12086
12087 ins_pipe(ialu_reg_reg_shift);
12088 %}
12089
12090 // This pattern is automatically generated from aarch64_ad.m4
12091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12092 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12093 iRegIorL2I src1, iRegIorL2I src2,
12094 immI src3) %{
12095 match(Set dst (SubI src1 (RShiftI src2 src3)));
12096
12097 ins_cost(1.9 * INSN_COST);
12098 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12099
12100 ins_encode %{
12101 __ subw(as_Register($dst$$reg),
12102 as_Register($src1$$reg),
12103 as_Register($src2$$reg),
12104 Assembler::ASR,
12105 $src3$$constant & 0x1f);
12106 %}
12107
12108 ins_pipe(ialu_reg_reg_shift);
12109 %}
12110
12111 // This pattern is automatically generated from aarch64_ad.m4
12112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12113 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12114 iRegL src1, iRegL src2,
12115 immI src3) %{
12116 match(Set dst (SubL src1 (RShiftL src2 src3)));
12117
12118 ins_cost(1.9 * INSN_COST);
12119 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12120
12121 ins_encode %{
12122 __ sub(as_Register($dst$$reg),
12123 as_Register($src1$$reg),
12124 as_Register($src2$$reg),
12125 Assembler::ASR,
12126 $src3$$constant & 0x3f);
12127 %}
12128
12129 ins_pipe(ialu_reg_reg_shift);
12130 %}
12131
12132 // This pattern is automatically generated from aarch64_ad.m4
12133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12134 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12135 iRegIorL2I src1, iRegIorL2I src2,
12136 immI src3) %{
12137 match(Set dst (SubI src1 (LShiftI src2 src3)));
12138
12139 ins_cost(1.9 * INSN_COST);
12140 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12141
12142 ins_encode %{
12143 __ subw(as_Register($dst$$reg),
12144 as_Register($src1$$reg),
12145 as_Register($src2$$reg),
12146 Assembler::LSL,
12147 $src3$$constant & 0x1f);
12148 %}
12149
12150 ins_pipe(ialu_reg_reg_shift);
12151 %}
12152
12153 // This pattern is automatically generated from aarch64_ad.m4
12154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12155 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12156 iRegL src1, iRegL src2,
12157 immI src3) %{
12158 match(Set dst (SubL src1 (LShiftL src2 src3)));
12159
12160 ins_cost(1.9 * INSN_COST);
12161 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12162
12163 ins_encode %{
12164 __ sub(as_Register($dst$$reg),
12165 as_Register($src1$$reg),
12166 as_Register($src2$$reg),
12167 Assembler::LSL,
12168 $src3$$constant & 0x3f);
12169 %}
12170
12171 ins_pipe(ialu_reg_reg_shift);
12172 %}
12173
12174 // This pattern is automatically generated from aarch64_ad.m4
12175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12176
12177 // Shift Left followed by Shift Right.
12178 // This idiom is used by the compiler for the i2b bytecode etc.
12179 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12180 %{
12181 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12182 ins_cost(INSN_COST * 2);
12183 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12184 ins_encode %{
12185 int lshift = $lshift_count$$constant & 63;
12186 int rshift = $rshift_count$$constant & 63;
12187 int s = 63 - lshift;
12188 int r = (rshift - lshift) & 63;
12189 __ sbfm(as_Register($dst$$reg),
12190 as_Register($src$$reg),
12191 r, s);
12192 %}
12193
12194 ins_pipe(ialu_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199
12200 // Shift Left followed by Shift Right.
12201 // This idiom is used by the compiler for the i2b bytecode etc.
12202 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12203 %{
12204 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12205 ins_cost(INSN_COST * 2);
12206 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12207 ins_encode %{
12208 int lshift = $lshift_count$$constant & 31;
12209 int rshift = $rshift_count$$constant & 31;
12210 int s = 31 - lshift;
12211 int r = (rshift - lshift) & 31;
12212 __ sbfmw(as_Register($dst$$reg),
12213 as_Register($src$$reg),
12214 r, s);
12215 %}
12216
12217 ins_pipe(ialu_reg_shift);
12218 %}
12219
12220 // This pattern is automatically generated from aarch64_ad.m4
12221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12222
12223 // Shift Left followed by Shift Right.
12224 // This idiom is used by the compiler for the i2b bytecode etc.
12225 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12226 %{
12227 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12228 ins_cost(INSN_COST * 2);
12229 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12230 ins_encode %{
12231 int lshift = $lshift_count$$constant & 63;
12232 int rshift = $rshift_count$$constant & 63;
12233 int s = 63 - lshift;
12234 int r = (rshift - lshift) & 63;
12235 __ ubfm(as_Register($dst$$reg),
12236 as_Register($src$$reg),
12237 r, s);
12238 %}
12239
12240 ins_pipe(ialu_reg_shift);
12241 %}
12242
12243 // This pattern is automatically generated from aarch64_ad.m4
12244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12245
12246 // Shift Left followed by Shift Right.
12247 // This idiom is used by the compiler for the i2b bytecode etc.
12248 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12249 %{
12250 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12251 ins_cost(INSN_COST * 2);
12252 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12253 ins_encode %{
12254 int lshift = $lshift_count$$constant & 31;
12255 int rshift = $rshift_count$$constant & 31;
12256 int s = 31 - lshift;
12257 int r = (rshift - lshift) & 31;
12258 __ ubfmw(as_Register($dst$$reg),
12259 as_Register($src$$reg),
12260 r, s);
12261 %}
12262
12263 ins_pipe(ialu_reg_shift);
12264 %}
12265
12266 // Bitfield extract with shift & mask
12267
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12271 %{
12272 match(Set dst (AndI (URShiftI src rshift) mask));
12273 // Make sure we are not going to exceed what ubfxw can do.
12274 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12275
12276 ins_cost(INSN_COST);
12277 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12278 ins_encode %{
12279 int rshift = $rshift$$constant & 31;
12280 intptr_t mask = $mask$$constant;
12281 int width = exact_log2(mask+1);
12282 __ ubfxw(as_Register($dst$$reg),
12283 as_Register($src$$reg), rshift, width);
12284 %}
12285 ins_pipe(ialu_reg_shift);
12286 %}
12287
12288 // This pattern is automatically generated from aarch64_ad.m4
12289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12290 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12291 %{
12292 match(Set dst (AndL (URShiftL src rshift) mask));
12293 // Make sure we are not going to exceed what ubfx can do.
12294 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12295
12296 ins_cost(INSN_COST);
12297 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12298 ins_encode %{
12299 int rshift = $rshift$$constant & 63;
12300 intptr_t mask = $mask$$constant;
12301 int width = exact_log2_long(mask+1);
12302 __ ubfx(as_Register($dst$$reg),
12303 as_Register($src$$reg), rshift, width);
12304 %}
12305 ins_pipe(ialu_reg_shift);
12306 %}
12307
12308
12309 // This pattern is automatically generated from aarch64_ad.m4
12310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12311
12312 // We can use ubfx when extending an And with a mask when we know mask
12313 // is positive. We know that because immI_bitmask guarantees it.
12314 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12315 %{
12316 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12317 // Make sure we are not going to exceed what ubfxw can do.
12318 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12319
12320 ins_cost(INSN_COST * 2);
12321 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12322 ins_encode %{
12323 int rshift = $rshift$$constant & 31;
12324 intptr_t mask = $mask$$constant;
12325 int width = exact_log2(mask+1);
12326 __ ubfx(as_Register($dst$$reg),
12327 as_Register($src$$reg), rshift, width);
12328 %}
12329 ins_pipe(ialu_reg_shift);
12330 %}
12331
12332
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335
12336 // We can use ubfiz when masking by a positive number and then left shifting the result.
12337 // We know that the mask is positive because immI_bitmask guarantees it.
12338 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12339 %{
12340 match(Set dst (LShiftI (AndI src mask) lshift));
12341 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12342
12343 ins_cost(INSN_COST);
12344 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12345 ins_encode %{
12346 int lshift = $lshift$$constant & 31;
12347 intptr_t mask = $mask$$constant;
12348 int width = exact_log2(mask+1);
12349 __ ubfizw(as_Register($dst$$reg),
12350 as_Register($src$$reg), lshift, width);
12351 %}
12352 ins_pipe(ialu_reg_shift);
12353 %}
12354
12355 // This pattern is automatically generated from aarch64_ad.m4
12356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12357
12358 // We can use ubfiz when masking by a positive number and then left shifting the result.
12359 // We know that the mask is positive because immL_bitmask guarantees it.
12360 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12361 %{
12362 match(Set dst (LShiftL (AndL src mask) lshift));
12363 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12364
12365 ins_cost(INSN_COST);
12366 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12367 ins_encode %{
12368 int lshift = $lshift$$constant & 63;
12369 intptr_t mask = $mask$$constant;
12370 int width = exact_log2_long(mask+1);
12371 __ ubfiz(as_Register($dst$$reg),
12372 as_Register($src$$reg), lshift, width);
12373 %}
12374 ins_pipe(ialu_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379
12380 // We can use ubfiz when masking by a positive number and then left shifting the result.
12381 // We know that the mask is positive because immI_bitmask guarantees it.
12382 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12383 %{
12384 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12385 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12386
12387 ins_cost(INSN_COST);
12388 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12389 ins_encode %{
12390 int lshift = $lshift$$constant & 31;
12391 intptr_t mask = $mask$$constant;
12392 int width = exact_log2(mask+1);
12393 __ ubfizw(as_Register($dst$$reg),
12394 as_Register($src$$reg), lshift, width);
12395 %}
12396 ins_pipe(ialu_reg_shift);
12397 %}
12398
12399 // This pattern is automatically generated from aarch64_ad.m4
12400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12401
12402 // We can use ubfiz when masking by a positive number and then left shifting the result.
12403 // We know that the mask is positive because immL_bitmask guarantees it.
12404 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12405 %{
12406 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12407 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12408
12409 ins_cost(INSN_COST);
12410 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12411 ins_encode %{
12412 int lshift = $lshift$$constant & 63;
12413 intptr_t mask = $mask$$constant;
12414 int width = exact_log2_long(mask+1);
12415 __ ubfiz(as_Register($dst$$reg),
12416 as_Register($src$$reg), lshift, width);
12417 %}
12418 ins_pipe(ialu_reg_shift);
12419 %}
12420
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424
12425 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12426 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12427 %{
12428 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12429 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12430
12431 ins_cost(INSN_COST);
12432 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12433 ins_encode %{
12434 int lshift = $lshift$$constant & 63;
12435 intptr_t mask = $mask$$constant;
12436 int width = exact_log2(mask+1);
12437 __ ubfiz(as_Register($dst$$reg),
12438 as_Register($src$$reg), lshift, width);
12439 %}
12440 ins_pipe(ialu_reg_shift);
12441 %}
12442
12443 // This pattern is automatically generated from aarch64_ad.m4
12444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12445
12446 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12447 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12448 %{
12449 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12450 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12451
12452 ins_cost(INSN_COST);
12453 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12454 ins_encode %{
12455 int lshift = $lshift$$constant & 31;
12456 intptr_t mask = $mask$$constant;
12457 int width = exact_log2(mask+1);
12458 __ ubfiz(as_Register($dst$$reg),
12459 as_Register($src$$reg), lshift, width);
12460 %}
12461 ins_pipe(ialu_reg_shift);
12462 %}
12463
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466
12467 // Can skip int2long conversions after AND with small bitmask
12468 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12469 %{
12470 match(Set dst (ConvI2L (AndI src msk)));
12471 ins_cost(INSN_COST);
12472 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12473 ins_encode %{
12474 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12475 %}
12476 ins_pipe(ialu_reg_shift);
12477 %}
12478
12479
12480 // Rotations
12481
12482 // This pattern is automatically generated from aarch64_ad.m4
12483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12484 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12485 %{
12486 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12487 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12488
12489 ins_cost(INSN_COST);
12490 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12491
12492 ins_encode %{
12493 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12494 $rshift$$constant & 63);
12495 %}
12496 ins_pipe(ialu_reg_reg_extr);
12497 %}
12498
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12503 %{
12504 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12505 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12506
12507 ins_cost(INSN_COST);
12508 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12509
12510 ins_encode %{
12511 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12512 $rshift$$constant & 31);
12513 %}
12514 ins_pipe(ialu_reg_reg_extr);
12515 %}
12516
12517
12518 // This pattern is automatically generated from aarch64_ad.m4
12519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12520 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12521 %{
12522 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12523 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12524
12525 ins_cost(INSN_COST);
12526 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12527
12528 ins_encode %{
12529 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12530 $rshift$$constant & 63);
12531 %}
12532 ins_pipe(ialu_reg_reg_extr);
12533 %}
12534
12535
12536 // This pattern is automatically generated from aarch64_ad.m4
12537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12538 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12539 %{
12540 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12541 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12542
12543 ins_cost(INSN_COST);
12544 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12545
12546 ins_encode %{
12547 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12548 $rshift$$constant & 31);
12549 %}
12550 ins_pipe(ialu_reg_reg_extr);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12556 %{
12557 match(Set dst (RotateRight src shift));
12558
12559 ins_cost(INSN_COST);
12560 format %{ "ror $dst, $src, $shift" %}
12561
12562 ins_encode %{
12563 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12564 $shift$$constant & 0x1f);
12565 %}
12566 ins_pipe(ialu_reg_reg_vshift);
12567 %}
12568
12569 // This pattern is automatically generated from aarch64_ad.m4
12570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12571 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12572 %{
12573 match(Set dst (RotateRight src shift));
12574
12575 ins_cost(INSN_COST);
12576 format %{ "ror $dst, $src, $shift" %}
12577
12578 ins_encode %{
12579 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12580 $shift$$constant & 0x3f);
12581 %}
12582 ins_pipe(ialu_reg_reg_vshift);
12583 %}
12584
12585 // This pattern is automatically generated from aarch64_ad.m4
12586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12587 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12588 %{
12589 match(Set dst (RotateRight src shift));
12590
12591 ins_cost(INSN_COST);
12592 format %{ "ror $dst, $src, $shift" %}
12593
12594 ins_encode %{
12595 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12596 %}
12597 ins_pipe(ialu_reg_reg_vshift);
12598 %}
12599
12600 // This pattern is automatically generated from aarch64_ad.m4
12601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12602 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12603 %{
12604 match(Set dst (RotateRight src shift));
12605
12606 ins_cost(INSN_COST);
12607 format %{ "ror $dst, $src, $shift" %}
12608
12609 ins_encode %{
12610 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12611 %}
12612 ins_pipe(ialu_reg_reg_vshift);
12613 %}
12614
12615 // This pattern is automatically generated from aarch64_ad.m4
12616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12617 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12618 %{
12619 match(Set dst (RotateLeft src shift));
12620
12621 ins_cost(INSN_COST);
12622 format %{ "rol $dst, $src, $shift" %}
12623
12624 ins_encode %{
12625 __ subw(rscratch1, zr, as_Register($shift$$reg));
12626 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12627 %}
12628 ins_pipe(ialu_reg_reg_vshift);
12629 %}
12630
12631 // This pattern is automatically generated from aarch64_ad.m4
12632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12633 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12634 %{
12635 match(Set dst (RotateLeft src shift));
12636
12637 ins_cost(INSN_COST);
12638 format %{ "rol $dst, $src, $shift" %}
12639
12640 ins_encode %{
12641 __ subw(rscratch1, zr, as_Register($shift$$reg));
12642 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12643 %}
12644 ins_pipe(ialu_reg_reg_vshift);
12645 %}
12646
12647
12648 // Add/subtract (extended)
12649
12650 // This pattern is automatically generated from aarch64_ad.m4
12651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12652 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12653 %{
12654 match(Set dst (AddL src1 (ConvI2L src2)));
12655 ins_cost(INSN_COST);
12656 format %{ "add $dst, $src1, $src2, sxtw" %}
12657
12658 ins_encode %{
12659 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12660 as_Register($src2$$reg), ext::sxtw);
12661 %}
12662 ins_pipe(ialu_reg_reg);
12663 %}
12664
12665 // This pattern is automatically generated from aarch64_ad.m4
12666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12667 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12668 %{
12669 match(Set dst (SubL src1 (ConvI2L src2)));
12670 ins_cost(INSN_COST);
12671 format %{ "sub $dst, $src1, $src2, sxtw" %}
12672
12673 ins_encode %{
12674 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12675 as_Register($src2$$reg), ext::sxtw);
12676 %}
12677 ins_pipe(ialu_reg_reg);
12678 %}
12679
12680 // This pattern is automatically generated from aarch64_ad.m4
12681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12682 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12683 %{
12684 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12685 ins_cost(INSN_COST);
12686 format %{ "add $dst, $src1, $src2, sxth" %}
12687
12688 ins_encode %{
12689 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12690 as_Register($src2$$reg), ext::sxth);
12691 %}
12692 ins_pipe(ialu_reg_reg);
12693 %}
12694
12695 // This pattern is automatically generated from aarch64_ad.m4
12696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12697 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12698 %{
12699 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12700 ins_cost(INSN_COST);
12701 format %{ "add $dst, $src1, $src2, sxtb" %}
12702
12703 ins_encode %{
12704 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12705 as_Register($src2$$reg), ext::sxtb);
12706 %}
12707 ins_pipe(ialu_reg_reg);
12708 %}
12709
12710 // This pattern is automatically generated from aarch64_ad.m4
12711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12712 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12713 %{
12714 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12715 ins_cost(INSN_COST);
12716 format %{ "add $dst, $src1, $src2, uxtb" %}
12717
12718 ins_encode %{
12719 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12720 as_Register($src2$$reg), ext::uxtb);
12721 %}
12722 ins_pipe(ialu_reg_reg);
12723 %}
12724
12725 // This pattern is automatically generated from aarch64_ad.m4
12726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12727 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12728 %{
12729 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12730 ins_cost(INSN_COST);
12731 format %{ "add $dst, $src1, $src2, sxth" %}
12732
12733 ins_encode %{
12734 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12735 as_Register($src2$$reg), ext::sxth);
12736 %}
12737 ins_pipe(ialu_reg_reg);
12738 %}
12739
12740 // This pattern is automatically generated from aarch64_ad.m4
12741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12742 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12743 %{
12744 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12745 ins_cost(INSN_COST);
12746 format %{ "add $dst, $src1, $src2, sxtw" %}
12747
12748 ins_encode %{
12749 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12750 as_Register($src2$$reg), ext::sxtw);
12751 %}
12752 ins_pipe(ialu_reg_reg);
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12758 %{
12759 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12760 ins_cost(INSN_COST);
12761 format %{ "add $dst, $src1, $src2, sxtb" %}
12762
12763 ins_encode %{
12764 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12765 as_Register($src2$$reg), ext::sxtb);
12766 %}
12767 ins_pipe(ialu_reg_reg);
12768 %}
12769
12770 // This pattern is automatically generated from aarch64_ad.m4
12771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12772 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12773 %{
12774 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12775 ins_cost(INSN_COST);
12776 format %{ "add $dst, $src1, $src2, uxtb" %}
12777
12778 ins_encode %{
12779 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12780 as_Register($src2$$reg), ext::uxtb);
12781 %}
12782 ins_pipe(ialu_reg_reg);
12783 %}
12784
12785 // This pattern is automatically generated from aarch64_ad.m4
12786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12787 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12788 %{
12789 match(Set dst (AddI src1 (AndI src2 mask)));
12790 ins_cost(INSN_COST);
12791 format %{ "addw $dst, $src1, $src2, uxtb" %}
12792
12793 ins_encode %{
12794 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12795 as_Register($src2$$reg), ext::uxtb);
12796 %}
12797 ins_pipe(ialu_reg_reg);
12798 %}
12799
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12803 %{
12804 match(Set dst (AddI src1 (AndI src2 mask)));
12805 ins_cost(INSN_COST);
12806 format %{ "addw $dst, $src1, $src2, uxth" %}
12807
12808 ins_encode %{
12809 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12810 as_Register($src2$$reg), ext::uxth);
12811 %}
12812 ins_pipe(ialu_reg_reg);
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12818 %{
12819 match(Set dst (AddL src1 (AndL src2 mask)));
12820 ins_cost(INSN_COST);
12821 format %{ "add $dst, $src1, $src2, uxtb" %}
12822
12823 ins_encode %{
12824 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12825 as_Register($src2$$reg), ext::uxtb);
12826 %}
12827 ins_pipe(ialu_reg_reg);
12828 %}
12829
12830 // This pattern is automatically generated from aarch64_ad.m4
12831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12832 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12833 %{
12834 match(Set dst (AddL src1 (AndL src2 mask)));
12835 ins_cost(INSN_COST);
12836 format %{ "add $dst, $src1, $src2, uxth" %}
12837
12838 ins_encode %{
12839 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12840 as_Register($src2$$reg), ext::uxth);
12841 %}
12842 ins_pipe(ialu_reg_reg);
12843 %}
12844
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12848 %{
12849 match(Set dst (AddL src1 (AndL src2 mask)));
12850 ins_cost(INSN_COST);
12851 format %{ "add $dst, $src1, $src2, uxtw" %}
12852
12853 ins_encode %{
12854 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12855 as_Register($src2$$reg), ext::uxtw);
12856 %}
12857 ins_pipe(ialu_reg_reg);
12858 %}
12859
12860 // This pattern is automatically generated from aarch64_ad.m4
12861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12862 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12863 %{
12864 match(Set dst (SubI src1 (AndI src2 mask)));
12865 ins_cost(INSN_COST);
12866 format %{ "subw $dst, $src1, $src2, uxtb" %}
12867
12868 ins_encode %{
12869 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12870 as_Register($src2$$reg), ext::uxtb);
12871 %}
12872 ins_pipe(ialu_reg_reg);
12873 %}
12874
12875 // This pattern is automatically generated from aarch64_ad.m4
12876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12877 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12878 %{
12879 match(Set dst (SubI src1 (AndI src2 mask)));
12880 ins_cost(INSN_COST);
12881 format %{ "subw $dst, $src1, $src2, uxth" %}
12882
12883 ins_encode %{
12884 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12885 as_Register($src2$$reg), ext::uxth);
12886 %}
12887 ins_pipe(ialu_reg_reg);
12888 %}
12889
12890 // This pattern is automatically generated from aarch64_ad.m4
12891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12892 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12893 %{
12894 match(Set dst (SubL src1 (AndL src2 mask)));
12895 ins_cost(INSN_COST);
12896 format %{ "sub $dst, $src1, $src2, uxtb" %}
12897
12898 ins_encode %{
12899 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12900 as_Register($src2$$reg), ext::uxtb);
12901 %}
12902 ins_pipe(ialu_reg_reg);
12903 %}
12904
12905 // This pattern is automatically generated from aarch64_ad.m4
12906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12908 %{
12909 match(Set dst (SubL src1 (AndL src2 mask)));
12910 ins_cost(INSN_COST);
12911 format %{ "sub $dst, $src1, $src2, uxth" %}
12912
12913 ins_encode %{
12914 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12915 as_Register($src2$$reg), ext::uxth);
12916 %}
12917 ins_pipe(ialu_reg_reg);
12918 %}
12919
12920 // This pattern is automatically generated from aarch64_ad.m4
12921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12922 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12923 %{
12924 match(Set dst (SubL src1 (AndL src2 mask)));
12925 ins_cost(INSN_COST);
12926 format %{ "sub $dst, $src1, $src2, uxtw" %}
12927
12928 ins_encode %{
12929 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12930 as_Register($src2$$reg), ext::uxtw);
12931 %}
12932 ins_pipe(ialu_reg_reg);
12933 %}
12934
12935
12936 // This pattern is automatically generated from aarch64_ad.m4
12937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12938 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12939 %{
12940 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12941 ins_cost(1.9 * INSN_COST);
12942 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12943
12944 ins_encode %{
12945 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12946 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12947 %}
12948 ins_pipe(ialu_reg_reg_shift);
12949 %}
12950
12951 // This pattern is automatically generated from aarch64_ad.m4
12952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12953 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12954 %{
12955 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12956 ins_cost(1.9 * INSN_COST);
12957 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12958
12959 ins_encode %{
12960 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12961 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12962 %}
12963 ins_pipe(ialu_reg_reg_shift);
12964 %}
12965
12966 // This pattern is automatically generated from aarch64_ad.m4
12967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12968 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12969 %{
12970 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12971 ins_cost(1.9 * INSN_COST);
12972 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12973
12974 ins_encode %{
12975 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12976 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12977 %}
12978 ins_pipe(ialu_reg_reg_shift);
12979 %}
12980
12981 // This pattern is automatically generated from aarch64_ad.m4
12982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12983 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12984 %{
12985 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12986 ins_cost(1.9 * INSN_COST);
12987 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12988
12989 ins_encode %{
12990 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12991 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12992 %}
12993 ins_pipe(ialu_reg_reg_shift);
12994 %}
12995
12996 // This pattern is automatically generated from aarch64_ad.m4
12997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12998 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12999 %{
13000 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13001 ins_cost(1.9 * INSN_COST);
13002 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13003
13004 ins_encode %{
13005 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13006 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13007 %}
13008 ins_pipe(ialu_reg_reg_shift);
13009 %}
13010
13011 // This pattern is automatically generated from aarch64_ad.m4
13012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13013 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13014 %{
13015 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13016 ins_cost(1.9 * INSN_COST);
13017 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13018
13019 ins_encode %{
13020 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13021 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13022 %}
13023 ins_pipe(ialu_reg_reg_shift);
13024 %}
13025
13026 // This pattern is automatically generated from aarch64_ad.m4
13027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13028 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13029 %{
13030 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13031 ins_cost(1.9 * INSN_COST);
13032 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13033
13034 ins_encode %{
13035 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13036 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13037 %}
13038 ins_pipe(ialu_reg_reg_shift);
13039 %}
13040
13041 // This pattern is automatically generated from aarch64_ad.m4
13042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13043 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13044 %{
13045 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13046 ins_cost(1.9 * INSN_COST);
13047 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13048
13049 ins_encode %{
13050 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13051 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13052 %}
13053 ins_pipe(ialu_reg_reg_shift);
13054 %}
13055
13056 // This pattern is automatically generated from aarch64_ad.m4
13057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13058 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13059 %{
13060 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13061 ins_cost(1.9 * INSN_COST);
13062 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13063
13064 ins_encode %{
13065 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13066 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13067 %}
13068 ins_pipe(ialu_reg_reg_shift);
13069 %}
13070
13071 // This pattern is automatically generated from aarch64_ad.m4
13072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13073 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13074 %{
13075 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13076 ins_cost(1.9 * INSN_COST);
13077 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13078
13079 ins_encode %{
13080 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13081 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13082 %}
13083 ins_pipe(ialu_reg_reg_shift);
13084 %}
13085
13086 // This pattern is automatically generated from aarch64_ad.m4
13087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13088 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13089 %{
13090 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13091 ins_cost(1.9 * INSN_COST);
13092 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13093
13094 ins_encode %{
13095 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13096 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13097 %}
13098 ins_pipe(ialu_reg_reg_shift);
13099 %}
13100
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13104 %{
13105 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13106 ins_cost(1.9 * INSN_COST);
13107 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13108
13109 ins_encode %{
13110 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13111 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13112 %}
13113 ins_pipe(ialu_reg_reg_shift);
13114 %}
13115
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13119 %{
13120 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13121 ins_cost(1.9 * INSN_COST);
13122 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13123
13124 ins_encode %{
13125 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13126 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13127 %}
13128 ins_pipe(ialu_reg_reg_shift);
13129 %}
13130
13131 // This pattern is automatically generated from aarch64_ad.m4
13132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13133 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13134 %{
13135 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13136 ins_cost(1.9 * INSN_COST);
13137 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13138
13139 ins_encode %{
13140 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13141 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13142 %}
13143 ins_pipe(ialu_reg_reg_shift);
13144 %}
13145
13146 // This pattern is automatically generated from aarch64_ad.m4
13147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13148 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13149 %{
13150 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13151 ins_cost(1.9 * INSN_COST);
13152 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13153
13154 ins_encode %{
13155 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13156 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13157 %}
13158 ins_pipe(ialu_reg_reg_shift);
13159 %}
13160
13161 // This pattern is automatically generated from aarch64_ad.m4
13162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13163 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13164 %{
13165 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13166 ins_cost(1.9 * INSN_COST);
13167 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13168
13169 ins_encode %{
13170 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13171 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13172 %}
13173 ins_pipe(ialu_reg_reg_shift);
13174 %}
13175
13176 // This pattern is automatically generated from aarch64_ad.m4
13177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13178 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13179 %{
13180 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13181 ins_cost(1.9 * INSN_COST);
13182 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13183
13184 ins_encode %{
13185 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13186 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13187 %}
13188 ins_pipe(ialu_reg_reg_shift);
13189 %}
13190
13191 // This pattern is automatically generated from aarch64_ad.m4
13192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13193 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13194 %{
13195 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13196 ins_cost(1.9 * INSN_COST);
13197 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13198
13199 ins_encode %{
13200 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13201 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13202 %}
13203 ins_pipe(ialu_reg_reg_shift);
13204 %}
13205
13206 // This pattern is automatically generated from aarch64_ad.m4
13207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13208 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13209 %{
13210 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13211 ins_cost(1.9 * INSN_COST);
13212 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13213
13214 ins_encode %{
13215 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13216 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13217 %}
13218 ins_pipe(ialu_reg_reg_shift);
13219 %}
13220
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13224 %{
13225 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13226 ins_cost(1.9 * INSN_COST);
13227 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13228
13229 ins_encode %{
13230 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13231 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13232 %}
13233 ins_pipe(ialu_reg_reg_shift);
13234 %}
13235
13236 // This pattern is automatically generated from aarch64_ad.m4
13237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13238 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13239 %{
13240 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13241 ins_cost(1.9 * INSN_COST);
13242 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13243
13244 ins_encode %{
13245 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13246 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13247 %}
13248 ins_pipe(ialu_reg_reg_shift);
13249 %}
13250
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13254 %{
13255 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13256 ins_cost(1.9 * INSN_COST);
13257 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13258
13259 ins_encode %{
13260 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13261 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13262 %}
13263 ins_pipe(ialu_reg_reg_shift);
13264 %}
13265
13266 // This pattern is automatically generated from aarch64_ad.m4
13267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13268 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13269 %{
13270 effect(DEF dst, USE src1, USE src2, USE cr);
13271 ins_cost(INSN_COST * 2);
13272 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13273
13274 ins_encode %{
13275 __ cselw($dst$$Register,
13276 $src1$$Register,
13277 $src2$$Register,
13278 Assembler::LT);
13279 %}
13280 ins_pipe(icond_reg_reg);
13281 %}
13282
13283 // This pattern is automatically generated from aarch64_ad.m4
13284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13285 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13286 %{
13287 effect(DEF dst, USE src1, USE src2, USE cr);
13288 ins_cost(INSN_COST * 2);
13289 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13290
13291 ins_encode %{
13292 __ cselw($dst$$Register,
13293 $src1$$Register,
13294 $src2$$Register,
13295 Assembler::GT);
13296 %}
13297 ins_pipe(icond_reg_reg);
13298 %}
13299
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13303 %{
13304 effect(DEF dst, USE src1, USE cr);
13305 ins_cost(INSN_COST * 2);
13306 format %{ "cselw $dst, $src1, zr lt\t" %}
13307
13308 ins_encode %{
13309 __ cselw($dst$$Register,
13310 $src1$$Register,
13311 zr,
13312 Assembler::LT);
13313 %}
13314 ins_pipe(icond_reg);
13315 %}
13316
13317 // This pattern is automatically generated from aarch64_ad.m4
13318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13319 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13320 %{
13321 effect(DEF dst, USE src1, USE cr);
13322 ins_cost(INSN_COST * 2);
13323 format %{ "cselw $dst, $src1, zr gt\t" %}
13324
13325 ins_encode %{
13326 __ cselw($dst$$Register,
13327 $src1$$Register,
13328 zr,
13329 Assembler::GT);
13330 %}
13331 ins_pipe(icond_reg);
13332 %}
13333
13334 // This pattern is automatically generated from aarch64_ad.m4
13335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13336 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13337 %{
13338 effect(DEF dst, USE src1, USE cr);
13339 ins_cost(INSN_COST * 2);
13340 format %{ "csincw $dst, $src1, zr le\t" %}
13341
13342 ins_encode %{
13343 __ csincw($dst$$Register,
13344 $src1$$Register,
13345 zr,
13346 Assembler::LE);
13347 %}
13348 ins_pipe(icond_reg);
13349 %}
13350
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13354 %{
13355 effect(DEF dst, USE src1, USE cr);
13356 ins_cost(INSN_COST * 2);
13357 format %{ "csincw $dst, $src1, zr gt\t" %}
13358
13359 ins_encode %{
13360 __ csincw($dst$$Register,
13361 $src1$$Register,
13362 zr,
13363 Assembler::GT);
13364 %}
13365 ins_pipe(icond_reg);
13366 %}
13367
13368 // This pattern is automatically generated from aarch64_ad.m4
13369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13370 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13371 %{
13372 effect(DEF dst, USE src1, USE cr);
13373 ins_cost(INSN_COST * 2);
13374 format %{ "csinvw $dst, $src1, zr lt\t" %}
13375
13376 ins_encode %{
13377 __ csinvw($dst$$Register,
13378 $src1$$Register,
13379 zr,
13380 Assembler::LT);
13381 %}
13382 ins_pipe(icond_reg);
13383 %}
13384
13385 // This pattern is automatically generated from aarch64_ad.m4
13386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13387 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13388 %{
13389 effect(DEF dst, USE src1, USE cr);
13390 ins_cost(INSN_COST * 2);
13391 format %{ "csinvw $dst, $src1, zr ge\t" %}
13392
13393 ins_encode %{
13394 __ csinvw($dst$$Register,
13395 $src1$$Register,
13396 zr,
13397 Assembler::GE);
13398 %}
13399 ins_pipe(icond_reg);
13400 %}
13401
13402 // This pattern is automatically generated from aarch64_ad.m4
13403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13404 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13405 %{
13406 match(Set dst (MinI src imm));
13407 ins_cost(INSN_COST * 3);
13408 expand %{
13409 rFlagsReg cr;
13410 compI_reg_imm0(cr, src);
13411 cmovI_reg_imm0_lt(dst, src, cr);
13412 %}
13413 %}
13414
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13418 %{
13419 match(Set dst (MinI imm src));
13420 ins_cost(INSN_COST * 3);
13421 expand %{
13422 rFlagsReg cr;
13423 compI_reg_imm0(cr, src);
13424 cmovI_reg_imm0_lt(dst, src, cr);
13425 %}
13426 %}
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13431 %{
13432 match(Set dst (MinI src imm));
13433 ins_cost(INSN_COST * 3);
13434 expand %{
13435 rFlagsReg cr;
13436 compI_reg_imm0(cr, src);
13437 cmovI_reg_imm1_le(dst, src, cr);
13438 %}
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13444 %{
13445 match(Set dst (MinI imm src));
13446 ins_cost(INSN_COST * 3);
13447 expand %{
13448 rFlagsReg cr;
13449 compI_reg_imm0(cr, src);
13450 cmovI_reg_imm1_le(dst, src, cr);
13451 %}
13452 %}
13453
13454 // This pattern is automatically generated from aarch64_ad.m4
13455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13456 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13457 %{
13458 match(Set dst (MinI src imm));
13459 ins_cost(INSN_COST * 3);
13460 expand %{
13461 rFlagsReg cr;
13462 compI_reg_imm0(cr, src);
13463 cmovI_reg_immM1_lt(dst, src, cr);
13464 %}
13465 %}
13466
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13470 %{
13471 match(Set dst (MinI imm src));
13472 ins_cost(INSN_COST * 3);
13473 expand %{
13474 rFlagsReg cr;
13475 compI_reg_imm0(cr, src);
13476 cmovI_reg_immM1_lt(dst, src, cr);
13477 %}
13478 %}
13479
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13483 %{
13484 match(Set dst (MaxI src imm));
13485 ins_cost(INSN_COST * 3);
13486 expand %{
13487 rFlagsReg cr;
13488 compI_reg_imm0(cr, src);
13489 cmovI_reg_imm0_gt(dst, src, cr);
13490 %}
13491 %}
13492
13493 // This pattern is automatically generated from aarch64_ad.m4
13494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13495 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13496 %{
13497 match(Set dst (MaxI imm src));
13498 ins_cost(INSN_COST * 3);
13499 expand %{
13500 rFlagsReg cr;
13501 compI_reg_imm0(cr, src);
13502 cmovI_reg_imm0_gt(dst, src, cr);
13503 %}
13504 %}
13505
13506 // This pattern is automatically generated from aarch64_ad.m4
13507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13508 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13509 %{
13510 match(Set dst (MaxI src imm));
13511 ins_cost(INSN_COST * 3);
13512 expand %{
13513 rFlagsReg cr;
13514 compI_reg_imm0(cr, src);
13515 cmovI_reg_imm1_gt(dst, src, cr);
13516 %}
13517 %}
13518
13519 // This pattern is automatically generated from aarch64_ad.m4
13520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13521 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13522 %{
13523 match(Set dst (MaxI imm src));
13524 ins_cost(INSN_COST * 3);
13525 expand %{
13526 rFlagsReg cr;
13527 compI_reg_imm0(cr, src);
13528 cmovI_reg_imm1_gt(dst, src, cr);
13529 %}
13530 %}
13531
13532 // This pattern is automatically generated from aarch64_ad.m4
13533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13534 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13535 %{
13536 match(Set dst (MaxI src imm));
13537 ins_cost(INSN_COST * 3);
13538 expand %{
13539 rFlagsReg cr;
13540 compI_reg_imm0(cr, src);
13541 cmovI_reg_immM1_ge(dst, src, cr);
13542 %}
13543 %}
13544
13545 // This pattern is automatically generated from aarch64_ad.m4
13546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13547 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13548 %{
13549 match(Set dst (MaxI imm src));
13550 ins_cost(INSN_COST * 3);
13551 expand %{
13552 rFlagsReg cr;
13553 compI_reg_imm0(cr, src);
13554 cmovI_reg_immM1_ge(dst, src, cr);
13555 %}
13556 %}
13557
13558 // This pattern is automatically generated from aarch64_ad.m4
13559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13560 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13561 %{
13562 match(Set dst (ReverseI src));
13563 ins_cost(INSN_COST);
13564 format %{ "rbitw $dst, $src" %}
13565 ins_encode %{
13566 __ rbitw($dst$$Register, $src$$Register);
13567 %}
13568 ins_pipe(ialu_reg);
13569 %}
13570
13571 // This pattern is automatically generated from aarch64_ad.m4
13572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13573 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13574 %{
13575 match(Set dst (ReverseL src));
13576 ins_cost(INSN_COST);
13577 format %{ "rbit $dst, $src" %}
13578 ins_encode %{
13579 __ rbit($dst$$Register, $src$$Register);
13580 %}
13581 ins_pipe(ialu_reg);
13582 %}
13583
13584
13585 // END This section of the file is automatically generated. Do not edit --------------
13586
13587
13588 // ============================================================================
13589 // Floating Point Arithmetic Instructions
13590
13591 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13592 match(Set dst (AddF src1 src2));
13593
13594 ins_cost(INSN_COST * 5);
13595 format %{ "fadds $dst, $src1, $src2" %}
13596
13597 ins_encode %{
13598 __ fadds(as_FloatRegister($dst$$reg),
13599 as_FloatRegister($src1$$reg),
13600 as_FloatRegister($src2$$reg));
13601 %}
13602
13603 ins_pipe(fp_dop_reg_reg_s);
13604 %}
13605
13606 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13607 match(Set dst (AddD src1 src2));
13608
13609 ins_cost(INSN_COST * 5);
13610 format %{ "faddd $dst, $src1, $src2" %}
13611
13612 ins_encode %{
13613 __ faddd(as_FloatRegister($dst$$reg),
13614 as_FloatRegister($src1$$reg),
13615 as_FloatRegister($src2$$reg));
13616 %}
13617
13618 ins_pipe(fp_dop_reg_reg_d);
13619 %}
13620
13621 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13622 match(Set dst (SubF src1 src2));
13623
13624 ins_cost(INSN_COST * 5);
13625 format %{ "fsubs $dst, $src1, $src2" %}
13626
13627 ins_encode %{
13628 __ fsubs(as_FloatRegister($dst$$reg),
13629 as_FloatRegister($src1$$reg),
13630 as_FloatRegister($src2$$reg));
13631 %}
13632
13633 ins_pipe(fp_dop_reg_reg_s);
13634 %}
13635
13636 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13637 match(Set dst (SubD src1 src2));
13638
13639 ins_cost(INSN_COST * 5);
13640 format %{ "fsubd $dst, $src1, $src2" %}
13641
13642 ins_encode %{
13643 __ fsubd(as_FloatRegister($dst$$reg),
13644 as_FloatRegister($src1$$reg),
13645 as_FloatRegister($src2$$reg));
13646 %}
13647
13648 ins_pipe(fp_dop_reg_reg_d);
13649 %}
13650
13651 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13652 match(Set dst (MulF src1 src2));
13653
13654 ins_cost(INSN_COST * 6);
13655 format %{ "fmuls $dst, $src1, $src2" %}
13656
13657 ins_encode %{
13658 __ fmuls(as_FloatRegister($dst$$reg),
13659 as_FloatRegister($src1$$reg),
13660 as_FloatRegister($src2$$reg));
13661 %}
13662
13663 ins_pipe(fp_dop_reg_reg_s);
13664 %}
13665
13666 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13667 match(Set dst (MulD src1 src2));
13668
13669 ins_cost(INSN_COST * 6);
13670 format %{ "fmuld $dst, $src1, $src2" %}
13671
13672 ins_encode %{
13673 __ fmuld(as_FloatRegister($dst$$reg),
13674 as_FloatRegister($src1$$reg),
13675 as_FloatRegister($src2$$reg));
13676 %}
13677
13678 ins_pipe(fp_dop_reg_reg_d);
13679 %}
13680
13681 // src1 * src2 + src3
13682 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13683 match(Set dst (FmaF src3 (Binary src1 src2)));
13684
13685 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13686
13687 ins_encode %{
13688 assert(UseFMA, "Needs FMA instructions support.");
13689 __ fmadds(as_FloatRegister($dst$$reg),
13690 as_FloatRegister($src1$$reg),
13691 as_FloatRegister($src2$$reg),
13692 as_FloatRegister($src3$$reg));
13693 %}
13694
13695 ins_pipe(pipe_class_default);
13696 %}
13697
13698 // src1 * src2 + src3
13699 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13700 match(Set dst (FmaD src3 (Binary src1 src2)));
13701
13702 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13703
13704 ins_encode %{
13705 assert(UseFMA, "Needs FMA instructions support.");
13706 __ fmaddd(as_FloatRegister($dst$$reg),
13707 as_FloatRegister($src1$$reg),
13708 as_FloatRegister($src2$$reg),
13709 as_FloatRegister($src3$$reg));
13710 %}
13711
13712 ins_pipe(pipe_class_default);
13713 %}
13714
13715 // src1 * (-src2) + src3
13716 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13717 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13718 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13719
13720 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13721
13722 ins_encode %{
13723 assert(UseFMA, "Needs FMA instructions support.");
13724 __ fmsubs(as_FloatRegister($dst$$reg),
13725 as_FloatRegister($src1$$reg),
13726 as_FloatRegister($src2$$reg),
13727 as_FloatRegister($src3$$reg));
13728 %}
13729
13730 ins_pipe(pipe_class_default);
13731 %}
13732
13733 // src1 * (-src2) + src3
13734 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13735 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13736 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13737
13738 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13739
13740 ins_encode %{
13741 assert(UseFMA, "Needs FMA instructions support.");
13742 __ fmsubd(as_FloatRegister($dst$$reg),
13743 as_FloatRegister($src1$$reg),
13744 as_FloatRegister($src2$$reg),
13745 as_FloatRegister($src3$$reg));
13746 %}
13747
13748 ins_pipe(pipe_class_default);
13749 %}
13750
13751 // src1 * (-src2) - src3
13752 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13753 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13754 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13755
13756 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13757
13758 ins_encode %{
13759 assert(UseFMA, "Needs FMA instructions support.");
13760 __ fnmadds(as_FloatRegister($dst$$reg),
13761 as_FloatRegister($src1$$reg),
13762 as_FloatRegister($src2$$reg),
13763 as_FloatRegister($src3$$reg));
13764 %}
13765
13766 ins_pipe(pipe_class_default);
13767 %}
13768
13769 // src1 * (-src2) - src3
13770 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13771 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13772 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13773
13774 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13775
13776 ins_encode %{
13777 assert(UseFMA, "Needs FMA instructions support.");
13778 __ fnmaddd(as_FloatRegister($dst$$reg),
13779 as_FloatRegister($src1$$reg),
13780 as_FloatRegister($src2$$reg),
13781 as_FloatRegister($src3$$reg));
13782 %}
13783
13784 ins_pipe(pipe_class_default);
13785 %}
13786
13787 // src1 * src2 - src3
13788 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13789 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13790
13791 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13792
13793 ins_encode %{
13794 assert(UseFMA, "Needs FMA instructions support.");
13795 __ fnmsubs(as_FloatRegister($dst$$reg),
13796 as_FloatRegister($src1$$reg),
13797 as_FloatRegister($src2$$reg),
13798 as_FloatRegister($src3$$reg));
13799 %}
13800
13801 ins_pipe(pipe_class_default);
13802 %}
13803
13804 // src1 * src2 - src3
13805 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13806 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13807
13808 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13809
13810 ins_encode %{
13811 assert(UseFMA, "Needs FMA instructions support.");
13812 // n.b. insn name should be fnmsubd
13813 __ fnmsub(as_FloatRegister($dst$$reg),
13814 as_FloatRegister($src1$$reg),
13815 as_FloatRegister($src2$$reg),
13816 as_FloatRegister($src3$$reg));
13817 %}
13818
13819 ins_pipe(pipe_class_default);
13820 %}
13821
13822
13823 // Math.max(FF)F
13824 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13825 match(Set dst (MaxF src1 src2));
13826
13827 format %{ "fmaxs $dst, $src1, $src2" %}
13828 ins_encode %{
13829 __ fmaxs(as_FloatRegister($dst$$reg),
13830 as_FloatRegister($src1$$reg),
13831 as_FloatRegister($src2$$reg));
13832 %}
13833
13834 ins_pipe(fp_dop_reg_reg_s);
13835 %}
13836
13837 // Math.min(FF)F
13838 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13839 match(Set dst (MinF src1 src2));
13840
13841 format %{ "fmins $dst, $src1, $src2" %}
13842 ins_encode %{
13843 __ fmins(as_FloatRegister($dst$$reg),
13844 as_FloatRegister($src1$$reg),
13845 as_FloatRegister($src2$$reg));
13846 %}
13847
13848 ins_pipe(fp_dop_reg_reg_s);
13849 %}
13850
13851 // Math.max(DD)D
13852 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13853 match(Set dst (MaxD src1 src2));
13854
13855 format %{ "fmaxd $dst, $src1, $src2" %}
13856 ins_encode %{
13857 __ fmaxd(as_FloatRegister($dst$$reg),
13858 as_FloatRegister($src1$$reg),
13859 as_FloatRegister($src2$$reg));
13860 %}
13861
13862 ins_pipe(fp_dop_reg_reg_d);
13863 %}
13864
13865 // Math.min(DD)D
13866 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13867 match(Set dst (MinD src1 src2));
13868
13869 format %{ "fmind $dst, $src1, $src2" %}
13870 ins_encode %{
13871 __ fmind(as_FloatRegister($dst$$reg),
13872 as_FloatRegister($src1$$reg),
13873 as_FloatRegister($src2$$reg));
13874 %}
13875
13876 ins_pipe(fp_dop_reg_reg_d);
13877 %}
13878
13879
13880 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13881 match(Set dst (DivF src1 src2));
13882
13883 ins_cost(INSN_COST * 18);
13884 format %{ "fdivs $dst, $src1, $src2" %}
13885
13886 ins_encode %{
13887 __ fdivs(as_FloatRegister($dst$$reg),
13888 as_FloatRegister($src1$$reg),
13889 as_FloatRegister($src2$$reg));
13890 %}
13891
13892 ins_pipe(fp_div_s);
13893 %}
13894
13895 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13896 match(Set dst (DivD src1 src2));
13897
13898 ins_cost(INSN_COST * 32);
13899 format %{ "fdivd $dst, $src1, $src2" %}
13900
13901 ins_encode %{
13902 __ fdivd(as_FloatRegister($dst$$reg),
13903 as_FloatRegister($src1$$reg),
13904 as_FloatRegister($src2$$reg));
13905 %}
13906
13907 ins_pipe(fp_div_d);
13908 %}
13909
13910 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13911 match(Set dst (NegF src));
13912
13913 ins_cost(INSN_COST * 3);
13914 format %{ "fneg $dst, $src" %}
13915
13916 ins_encode %{
13917 __ fnegs(as_FloatRegister($dst$$reg),
13918 as_FloatRegister($src$$reg));
13919 %}
13920
13921 ins_pipe(fp_uop_s);
13922 %}
13923
13924 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13925 match(Set dst (NegD src));
13926
13927 ins_cost(INSN_COST * 3);
13928 format %{ "fnegd $dst, $src" %}
13929
13930 ins_encode %{
13931 __ fnegd(as_FloatRegister($dst$$reg),
13932 as_FloatRegister($src$$reg));
13933 %}
13934
13935 ins_pipe(fp_uop_d);
13936 %}
13937
13938 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13939 %{
13940 match(Set dst (AbsI src));
13941
13942 effect(KILL cr);
13943 ins_cost(INSN_COST * 2);
13944 format %{ "cmpw $src, zr\n\t"
13945 "cnegw $dst, $src, Assembler::LT\t# int abs"
13946 %}
13947
13948 ins_encode %{
13949 __ cmpw(as_Register($src$$reg), zr);
13950 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13951 %}
13952 ins_pipe(pipe_class_default);
13953 %}
13954
13955 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13956 %{
13957 match(Set dst (AbsL src));
13958
13959 effect(KILL cr);
13960 ins_cost(INSN_COST * 2);
13961 format %{ "cmp $src, zr\n\t"
13962 "cneg $dst, $src, Assembler::LT\t# long abs"
13963 %}
13964
13965 ins_encode %{
13966 __ cmp(as_Register($src$$reg), zr);
13967 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13968 %}
13969 ins_pipe(pipe_class_default);
13970 %}
13971
13972 instruct absF_reg(vRegF dst, vRegF src) %{
13973 match(Set dst (AbsF src));
13974
13975 ins_cost(INSN_COST * 3);
13976 format %{ "fabss $dst, $src" %}
13977 ins_encode %{
13978 __ fabss(as_FloatRegister($dst$$reg),
13979 as_FloatRegister($src$$reg));
13980 %}
13981
13982 ins_pipe(fp_uop_s);
13983 %}
13984
13985 instruct absD_reg(vRegD dst, vRegD src) %{
13986 match(Set dst (AbsD src));
13987
13988 ins_cost(INSN_COST * 3);
13989 format %{ "fabsd $dst, $src" %}
13990 ins_encode %{
13991 __ fabsd(as_FloatRegister($dst$$reg),
13992 as_FloatRegister($src$$reg));
13993 %}
13994
13995 ins_pipe(fp_uop_d);
13996 %}
13997
13998 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13999 match(Set dst (AbsF (SubF src1 src2)));
14000
14001 ins_cost(INSN_COST * 3);
14002 format %{ "fabds $dst, $src1, $src2" %}
14003 ins_encode %{
14004 __ fabds(as_FloatRegister($dst$$reg),
14005 as_FloatRegister($src1$$reg),
14006 as_FloatRegister($src2$$reg));
14007 %}
14008
14009 ins_pipe(fp_uop_s);
14010 %}
14011
14012 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14013 match(Set dst (AbsD (SubD src1 src2)));
14014
14015 ins_cost(INSN_COST * 3);
14016 format %{ "fabdd $dst, $src1, $src2" %}
14017 ins_encode %{
14018 __ fabdd(as_FloatRegister($dst$$reg),
14019 as_FloatRegister($src1$$reg),
14020 as_FloatRegister($src2$$reg));
14021 %}
14022
14023 ins_pipe(fp_uop_d);
14024 %}
14025
14026 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14027 match(Set dst (SqrtD src));
14028
14029 ins_cost(INSN_COST * 50);
14030 format %{ "fsqrtd $dst, $src" %}
14031 ins_encode %{
14032 __ fsqrtd(as_FloatRegister($dst$$reg),
14033 as_FloatRegister($src$$reg));
14034 %}
14035
14036 ins_pipe(fp_div_s);
14037 %}
14038
14039 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14040 match(Set dst (SqrtF src));
14041
14042 ins_cost(INSN_COST * 50);
14043 format %{ "fsqrts $dst, $src" %}
14044 ins_encode %{
14045 __ fsqrts(as_FloatRegister($dst$$reg),
14046 as_FloatRegister($src$$reg));
14047 %}
14048
14049 ins_pipe(fp_div_d);
14050 %}
14051
14052 // Math.rint, floor, ceil
14053 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14054 match(Set dst (RoundDoubleMode src rmode));
14055 format %{ "frint $dst, $src, $rmode" %}
14056 ins_encode %{
14057 switch ($rmode$$constant) {
14058 case RoundDoubleModeNode::rmode_rint:
14059 __ frintnd(as_FloatRegister($dst$$reg),
14060 as_FloatRegister($src$$reg));
14061 break;
14062 case RoundDoubleModeNode::rmode_floor:
14063 __ frintmd(as_FloatRegister($dst$$reg),
14064 as_FloatRegister($src$$reg));
14065 break;
14066 case RoundDoubleModeNode::rmode_ceil:
14067 __ frintpd(as_FloatRegister($dst$$reg),
14068 as_FloatRegister($src$$reg));
14069 break;
14070 }
14071 %}
14072 ins_pipe(fp_uop_d);
14073 %}
14074
14075 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14076 match(Set dst (CopySignD src1 (Binary src2 zero)));
14077 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14078 format %{ "CopySignD $dst $src1 $src2" %}
14079 ins_encode %{
14080 FloatRegister dst = as_FloatRegister($dst$$reg),
14081 src1 = as_FloatRegister($src1$$reg),
14082 src2 = as_FloatRegister($src2$$reg),
14083 zero = as_FloatRegister($zero$$reg);
14084 __ fnegd(dst, zero);
14085 __ bsl(dst, __ T8B, src2, src1);
14086 %}
14087 ins_pipe(fp_uop_d);
14088 %}
14089
14090 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14091 match(Set dst (CopySignF src1 src2));
14092 effect(TEMP_DEF dst, USE src1, USE src2);
14093 format %{ "CopySignF $dst $src1 $src2" %}
14094 ins_encode %{
14095 FloatRegister dst = as_FloatRegister($dst$$reg),
14096 src1 = as_FloatRegister($src1$$reg),
14097 src2 = as_FloatRegister($src2$$reg);
14098 __ movi(dst, __ T2S, 0x80, 24);
14099 __ bsl(dst, __ T8B, src2, src1);
14100 %}
14101 ins_pipe(fp_uop_d);
14102 %}
14103
14104 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14105 match(Set dst (SignumD src (Binary zero one)));
14106 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14107 format %{ "signumD $dst, $src" %}
14108 ins_encode %{
14109 FloatRegister src = as_FloatRegister($src$$reg),
14110 dst = as_FloatRegister($dst$$reg),
14111 zero = as_FloatRegister($zero$$reg),
14112 one = as_FloatRegister($one$$reg);
14113 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14114 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14115 // Bit selection instruction gets bit from "one" for each enabled bit in
14116 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14117 // NaN the whole "src" will be copied because "dst" is zero. For all other
14118 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14119 // from "src", and all other bits are copied from 1.0.
14120 __ bsl(dst, __ T8B, one, src);
14121 %}
14122 ins_pipe(fp_uop_d);
14123 %}
14124
14125 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14126 match(Set dst (SignumF src (Binary zero one)));
14127 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14128 format %{ "signumF $dst, $src" %}
14129 ins_encode %{
14130 FloatRegister src = as_FloatRegister($src$$reg),
14131 dst = as_FloatRegister($dst$$reg),
14132 zero = as_FloatRegister($zero$$reg),
14133 one = as_FloatRegister($one$$reg);
14134 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14135 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14136 // Bit selection instruction gets bit from "one" for each enabled bit in
14137 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14138 // NaN the whole "src" will be copied because "dst" is zero. For all other
14139 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14140 // from "src", and all other bits are copied from 1.0.
14141 __ bsl(dst, __ T8B, one, src);
14142 %}
14143 ins_pipe(fp_uop_d);
14144 %}
14145
14146 instruct onspinwait() %{
14147 match(OnSpinWait);
14148 ins_cost(INSN_COST);
14149
14150 format %{ "onspinwait" %}
14151
14152 ins_encode %{
14153 __ spin_wait();
14154 %}
14155 ins_pipe(pipe_class_empty);
14156 %}
14157
14158 // ============================================================================
14159 // Logical Instructions
14160
14161 // Integer Logical Instructions
14162
14163 // And Instructions
14164
14165
14166 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14167 match(Set dst (AndI src1 src2));
14168
14169 format %{ "andw $dst, $src1, $src2\t# int" %}
14170
14171 ins_cost(INSN_COST);
14172 ins_encode %{
14173 __ andw(as_Register($dst$$reg),
14174 as_Register($src1$$reg),
14175 as_Register($src2$$reg));
14176 %}
14177
14178 ins_pipe(ialu_reg_reg);
14179 %}
14180
14181 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14182 match(Set dst (AndI src1 src2));
14183
14184 format %{ "andsw $dst, $src1, $src2\t# int" %}
14185
14186 ins_cost(INSN_COST);
14187 ins_encode %{
14188 __ andw(as_Register($dst$$reg),
14189 as_Register($src1$$reg),
14190 (uint64_t)($src2$$constant));
14191 %}
14192
14193 ins_pipe(ialu_reg_imm);
14194 %}
14195
14196 // Or Instructions
14197
14198 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14199 match(Set dst (OrI src1 src2));
14200
14201 format %{ "orrw $dst, $src1, $src2\t# int" %}
14202
14203 ins_cost(INSN_COST);
14204 ins_encode %{
14205 __ orrw(as_Register($dst$$reg),
14206 as_Register($src1$$reg),
14207 as_Register($src2$$reg));
14208 %}
14209
14210 ins_pipe(ialu_reg_reg);
14211 %}
14212
14213 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14214 match(Set dst (OrI src1 src2));
14215
14216 format %{ "orrw $dst, $src1, $src2\t# int" %}
14217
14218 ins_cost(INSN_COST);
14219 ins_encode %{
14220 __ orrw(as_Register($dst$$reg),
14221 as_Register($src1$$reg),
14222 (uint64_t)($src2$$constant));
14223 %}
14224
14225 ins_pipe(ialu_reg_imm);
14226 %}
14227
14228 // Xor Instructions
14229
14230 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14231 match(Set dst (XorI src1 src2));
14232
14233 format %{ "eorw $dst, $src1, $src2\t# int" %}
14234
14235 ins_cost(INSN_COST);
14236 ins_encode %{
14237 __ eorw(as_Register($dst$$reg),
14238 as_Register($src1$$reg),
14239 as_Register($src2$$reg));
14240 %}
14241
14242 ins_pipe(ialu_reg_reg);
14243 %}
14244
14245 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14246 match(Set dst (XorI src1 src2));
14247
14248 format %{ "eorw $dst, $src1, $src2\t# int" %}
14249
14250 ins_cost(INSN_COST);
14251 ins_encode %{
14252 __ eorw(as_Register($dst$$reg),
14253 as_Register($src1$$reg),
14254 (uint64_t)($src2$$constant));
14255 %}
14256
14257 ins_pipe(ialu_reg_imm);
14258 %}
14259
14260 // Long Logical Instructions
14261 // TODO
14262
14263 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14264 match(Set dst (AndL src1 src2));
14265
14266 format %{ "and $dst, $src1, $src2\t# int" %}
14267
14268 ins_cost(INSN_COST);
14269 ins_encode %{
14270 __ andr(as_Register($dst$$reg),
14271 as_Register($src1$$reg),
14272 as_Register($src2$$reg));
14273 %}
14274
14275 ins_pipe(ialu_reg_reg);
14276 %}
14277
14278 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14279 match(Set dst (AndL src1 src2));
14280
14281 format %{ "and $dst, $src1, $src2\t# int" %}
14282
14283 ins_cost(INSN_COST);
14284 ins_encode %{
14285 __ andr(as_Register($dst$$reg),
14286 as_Register($src1$$reg),
14287 (uint64_t)($src2$$constant));
14288 %}
14289
14290 ins_pipe(ialu_reg_imm);
14291 %}
14292
14293 // Or Instructions
14294
14295 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14296 match(Set dst (OrL src1 src2));
14297
14298 format %{ "orr $dst, $src1, $src2\t# int" %}
14299
14300 ins_cost(INSN_COST);
14301 ins_encode %{
14302 __ orr(as_Register($dst$$reg),
14303 as_Register($src1$$reg),
14304 as_Register($src2$$reg));
14305 %}
14306
14307 ins_pipe(ialu_reg_reg);
14308 %}
14309
14310 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14311 match(Set dst (OrL src1 src2));
14312
14313 format %{ "orr $dst, $src1, $src2\t# int" %}
14314
14315 ins_cost(INSN_COST);
14316 ins_encode %{
14317 __ orr(as_Register($dst$$reg),
14318 as_Register($src1$$reg),
14319 (uint64_t)($src2$$constant));
14320 %}
14321
14322 ins_pipe(ialu_reg_imm);
14323 %}
14324
14325 // Xor Instructions
14326
14327 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14328 match(Set dst (XorL src1 src2));
14329
14330 format %{ "eor $dst, $src1, $src2\t# int" %}
14331
14332 ins_cost(INSN_COST);
14333 ins_encode %{
14334 __ eor(as_Register($dst$$reg),
14335 as_Register($src1$$reg),
14336 as_Register($src2$$reg));
14337 %}
14338
14339 ins_pipe(ialu_reg_reg);
14340 %}
14341
14342 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14343 match(Set dst (XorL src1 src2));
14344
14345 ins_cost(INSN_COST);
14346 format %{ "eor $dst, $src1, $src2\t# int" %}
14347
14348 ins_encode %{
14349 __ eor(as_Register($dst$$reg),
14350 as_Register($src1$$reg),
14351 (uint64_t)($src2$$constant));
14352 %}
14353
14354 ins_pipe(ialu_reg_imm);
14355 %}
14356
14357 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14358 %{
14359 match(Set dst (ConvI2L src));
14360
14361 ins_cost(INSN_COST);
14362 format %{ "sxtw $dst, $src\t# i2l" %}
14363 ins_encode %{
14364 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14365 %}
14366 ins_pipe(ialu_reg_shift);
14367 %}
14368
14369 // this pattern occurs in bigmath arithmetic
14370 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14371 %{
14372 match(Set dst (AndL (ConvI2L src) mask));
14373
14374 ins_cost(INSN_COST);
14375 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14376 ins_encode %{
14377 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14378 %}
14379
14380 ins_pipe(ialu_reg_shift);
14381 %}
14382
14383 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14384 match(Set dst (ConvL2I src));
14385
14386 ins_cost(INSN_COST);
14387 format %{ "movw $dst, $src \t// l2i" %}
14388
14389 ins_encode %{
14390 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14391 %}
14392
14393 ins_pipe(ialu_reg);
14394 %}
14395
14396 instruct convD2F_reg(vRegF dst, vRegD src) %{
14397 match(Set dst (ConvD2F src));
14398
14399 ins_cost(INSN_COST * 5);
14400 format %{ "fcvtd $dst, $src \t// d2f" %}
14401
14402 ins_encode %{
14403 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14404 %}
14405
14406 ins_pipe(fp_d2f);
14407 %}
14408
14409 instruct convF2D_reg(vRegD dst, vRegF src) %{
14410 match(Set dst (ConvF2D src));
14411
14412 ins_cost(INSN_COST * 5);
14413 format %{ "fcvts $dst, $src \t// f2d" %}
14414
14415 ins_encode %{
14416 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14417 %}
14418
14419 ins_pipe(fp_f2d);
14420 %}
14421
14422 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14423 match(Set dst (ConvF2I src));
14424
14425 ins_cost(INSN_COST * 5);
14426 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14427
14428 ins_encode %{
14429 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14430 %}
14431
14432 ins_pipe(fp_f2i);
14433 %}
14434
14435 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14436 match(Set dst (ConvF2L src));
14437
14438 ins_cost(INSN_COST * 5);
14439 format %{ "fcvtzs $dst, $src \t// f2l" %}
14440
14441 ins_encode %{
14442 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14443 %}
14444
14445 ins_pipe(fp_f2l);
14446 %}
14447
14448 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14449 match(Set dst (ConvF2HF src));
14450 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14451 "smov $dst, $tmp\t# move result from $tmp to $dst"
14452 %}
14453 effect(TEMP tmp);
14454 ins_encode %{
14455 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14456 %}
14457 ins_pipe(pipe_slow);
14458 %}
14459
14460 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14461 match(Set dst (ConvHF2F src));
14462 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14463 "fcvt $dst, $tmp\t# convert half to single precision"
14464 %}
14465 effect(TEMP tmp);
14466 ins_encode %{
14467 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14468 %}
14469 ins_pipe(pipe_slow);
14470 %}
14471
14472 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14473 match(Set dst (ConvI2F src));
14474
14475 ins_cost(INSN_COST * 5);
14476 format %{ "scvtfws $dst, $src \t// i2f" %}
14477
14478 ins_encode %{
14479 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14480 %}
14481
14482 ins_pipe(fp_i2f);
14483 %}
14484
14485 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14486 match(Set dst (ConvL2F src));
14487
14488 ins_cost(INSN_COST * 5);
14489 format %{ "scvtfs $dst, $src \t// l2f" %}
14490
14491 ins_encode %{
14492 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14493 %}
14494
14495 ins_pipe(fp_l2f);
14496 %}
14497
14498 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14499 match(Set dst (ConvD2I src));
14500
14501 ins_cost(INSN_COST * 5);
14502 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14503
14504 ins_encode %{
14505 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14506 %}
14507
14508 ins_pipe(fp_d2i);
14509 %}
14510
14511 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14512 match(Set dst (ConvD2L src));
14513
14514 ins_cost(INSN_COST * 5);
14515 format %{ "fcvtzd $dst, $src \t// d2l" %}
14516
14517 ins_encode %{
14518 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14519 %}
14520
14521 ins_pipe(fp_d2l);
14522 %}
14523
14524 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14525 match(Set dst (ConvI2D src));
14526
14527 ins_cost(INSN_COST * 5);
14528 format %{ "scvtfwd $dst, $src \t// i2d" %}
14529
14530 ins_encode %{
14531 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14532 %}
14533
14534 ins_pipe(fp_i2d);
14535 %}
14536
14537 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14538 match(Set dst (ConvL2D src));
14539
14540 ins_cost(INSN_COST * 5);
14541 format %{ "scvtfd $dst, $src \t// l2d" %}
14542
14543 ins_encode %{
14544 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14545 %}
14546
14547 ins_pipe(fp_l2d);
14548 %}
14549
14550 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14551 %{
14552 match(Set dst (RoundD src));
14553 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14554 format %{ "java_round_double $dst,$src"%}
14555 ins_encode %{
14556 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14557 as_FloatRegister($ftmp$$reg));
14558 %}
14559 ins_pipe(pipe_slow);
14560 %}
14561
14562 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14563 %{
14564 match(Set dst (RoundF src));
14565 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14566 format %{ "java_round_float $dst,$src"%}
14567 ins_encode %{
14568 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14569 as_FloatRegister($ftmp$$reg));
14570 %}
14571 ins_pipe(pipe_slow);
14572 %}
14573
14574 // stack <-> reg and reg <-> reg shuffles with no conversion
14575
14576 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14577
14578 match(Set dst (MoveF2I src));
14579
14580 effect(DEF dst, USE src);
14581
14582 ins_cost(4 * INSN_COST);
14583
14584 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14585
14586 ins_encode %{
14587 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14588 %}
14589
14590 ins_pipe(iload_reg_reg);
14591
14592 %}
14593
14594 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14595
14596 match(Set dst (MoveI2F src));
14597
14598 effect(DEF dst, USE src);
14599
14600 ins_cost(4 * INSN_COST);
14601
14602 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14603
14604 ins_encode %{
14605 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14606 %}
14607
14608 ins_pipe(pipe_class_memory);
14609
14610 %}
14611
14612 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14613
14614 match(Set dst (MoveD2L src));
14615
14616 effect(DEF dst, USE src);
14617
14618 ins_cost(4 * INSN_COST);
14619
14620 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14621
14622 ins_encode %{
14623 __ ldr($dst$$Register, Address(sp, $src$$disp));
14624 %}
14625
14626 ins_pipe(iload_reg_reg);
14627
14628 %}
14629
14630 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14631
14632 match(Set dst (MoveL2D src));
14633
14634 effect(DEF dst, USE src);
14635
14636 ins_cost(4 * INSN_COST);
14637
14638 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14639
14640 ins_encode %{
14641 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14642 %}
14643
14644 ins_pipe(pipe_class_memory);
14645
14646 %}
14647
14648 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14649
14650 match(Set dst (MoveF2I src));
14651
14652 effect(DEF dst, USE src);
14653
14654 ins_cost(INSN_COST);
14655
14656 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14657
14658 ins_encode %{
14659 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14660 %}
14661
14662 ins_pipe(pipe_class_memory);
14663
14664 %}
14665
14666 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14667
14668 match(Set dst (MoveI2F src));
14669
14670 effect(DEF dst, USE src);
14671
14672 ins_cost(INSN_COST);
14673
14674 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14675
14676 ins_encode %{
14677 __ strw($src$$Register, Address(sp, $dst$$disp));
14678 %}
14679
14680 ins_pipe(istore_reg_reg);
14681
14682 %}
14683
14684 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14685
14686 match(Set dst (MoveD2L src));
14687
14688 effect(DEF dst, USE src);
14689
14690 ins_cost(INSN_COST);
14691
14692 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14693
14694 ins_encode %{
14695 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14696 %}
14697
14698 ins_pipe(pipe_class_memory);
14699
14700 %}
14701
14702 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14703
14704 match(Set dst (MoveL2D src));
14705
14706 effect(DEF dst, USE src);
14707
14708 ins_cost(INSN_COST);
14709
14710 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14711
14712 ins_encode %{
14713 __ str($src$$Register, Address(sp, $dst$$disp));
14714 %}
14715
14716 ins_pipe(istore_reg_reg);
14717
14718 %}
14719
14720 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14721
14722 match(Set dst (MoveF2I src));
14723
14724 effect(DEF dst, USE src);
14725
14726 ins_cost(INSN_COST);
14727
14728 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14729
14730 ins_encode %{
14731 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14732 %}
14733
14734 ins_pipe(fp_f2i);
14735
14736 %}
14737
14738 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14739
14740 match(Set dst (MoveI2F src));
14741
14742 effect(DEF dst, USE src);
14743
14744 ins_cost(INSN_COST);
14745
14746 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14747
14748 ins_encode %{
14749 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14750 %}
14751
14752 ins_pipe(fp_i2f);
14753
14754 %}
14755
14756 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14757
14758 match(Set dst (MoveD2L src));
14759
14760 effect(DEF dst, USE src);
14761
14762 ins_cost(INSN_COST);
14763
14764 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14765
14766 ins_encode %{
14767 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14768 %}
14769
14770 ins_pipe(fp_d2l);
14771
14772 %}
14773
14774 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14775
14776 match(Set dst (MoveL2D src));
14777
14778 effect(DEF dst, USE src);
14779
14780 ins_cost(INSN_COST);
14781
14782 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14783
14784 ins_encode %{
14785 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14786 %}
14787
14788 ins_pipe(fp_l2d);
14789
14790 %}
14791
14792 // ============================================================================
14793 // clearing of an array
14794
14795 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14796 %{
14797 match(Set dummy (ClearArray (Binary cnt base) zero));
14798 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14799
14800 ins_cost(4 * INSN_COST);
14801 format %{ "ClearArray $cnt, $base" %}
14802
14803 ins_encode %{
14804 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14805 if (tpc == nullptr) {
14806 ciEnv::current()->record_failure("CodeCache is full");
14807 return;
14808 }
14809 %}
14810
14811 ins_pipe(pipe_class_memory);
14812 %}
14813
14814 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14815 %{
14816 predicate(((ClearArrayNode*)n)->word_copy_only());
14817 match(Set dummy (ClearArray (Binary cnt base) val));
14818 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14819
14820 ins_cost(4 * INSN_COST);
14821 format %{ "ClearArray $cnt, $base, $val" %}
14822
14823 ins_encode %{
14824 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14825 %}
14826
14827 ins_pipe(pipe_class_memory);
14828 %}
14829
14830 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14831 %{
14832 predicate((uint64_t)n->in(2)->get_long()
14833 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14834 && !((ClearArrayNode*)n)->word_copy_only());
14835 match(Set dummy (ClearArray cnt base));
14836 effect(TEMP temp, USE_KILL base, KILL cr);
14837
14838 ins_cost(4 * INSN_COST);
14839 format %{ "ClearArray $cnt, $base" %}
14840
14841 ins_encode %{
14842 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14843 if (tpc == nullptr) {
14844 ciEnv::current()->record_failure("CodeCache is full");
14845 return;
14846 }
14847 %}
14848
14849 ins_pipe(pipe_class_memory);
14850 %}
14851
14852 // ============================================================================
14853 // Overflow Math Instructions
14854
14855 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14856 %{
14857 match(Set cr (OverflowAddI op1 op2));
14858
14859 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14860 ins_cost(INSN_COST);
14861 ins_encode %{
14862 __ cmnw($op1$$Register, $op2$$Register);
14863 %}
14864
14865 ins_pipe(icmp_reg_reg);
14866 %}
14867
14868 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14869 %{
14870 match(Set cr (OverflowAddI op1 op2));
14871
14872 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14873 ins_cost(INSN_COST);
14874 ins_encode %{
14875 __ cmnw($op1$$Register, $op2$$constant);
14876 %}
14877
14878 ins_pipe(icmp_reg_imm);
14879 %}
14880
14881 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14882 %{
14883 match(Set cr (OverflowAddL op1 op2));
14884
14885 format %{ "cmn $op1, $op2\t# overflow check long" %}
14886 ins_cost(INSN_COST);
14887 ins_encode %{
14888 __ cmn($op1$$Register, $op2$$Register);
14889 %}
14890
14891 ins_pipe(icmp_reg_reg);
14892 %}
14893
14894 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14895 %{
14896 match(Set cr (OverflowAddL op1 op2));
14897
14898 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14899 ins_cost(INSN_COST);
14900 ins_encode %{
14901 __ adds(zr, $op1$$Register, $op2$$constant);
14902 %}
14903
14904 ins_pipe(icmp_reg_imm);
14905 %}
14906
14907 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14908 %{
14909 match(Set cr (OverflowSubI op1 op2));
14910
14911 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14912 ins_cost(INSN_COST);
14913 ins_encode %{
14914 __ cmpw($op1$$Register, $op2$$Register);
14915 %}
14916
14917 ins_pipe(icmp_reg_reg);
14918 %}
14919
14920 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14921 %{
14922 match(Set cr (OverflowSubI op1 op2));
14923
14924 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14925 ins_cost(INSN_COST);
14926 ins_encode %{
14927 __ cmpw($op1$$Register, $op2$$constant);
14928 %}
14929
14930 ins_pipe(icmp_reg_imm);
14931 %}
14932
14933 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14934 %{
14935 match(Set cr (OverflowSubL op1 op2));
14936
14937 format %{ "cmp $op1, $op2\t# overflow check long" %}
14938 ins_cost(INSN_COST);
14939 ins_encode %{
14940 __ cmp($op1$$Register, $op2$$Register);
14941 %}
14942
14943 ins_pipe(icmp_reg_reg);
14944 %}
14945
14946 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14947 %{
14948 match(Set cr (OverflowSubL op1 op2));
14949
14950 format %{ "cmp $op1, $op2\t# overflow check long" %}
14951 ins_cost(INSN_COST);
14952 ins_encode %{
14953 __ subs(zr, $op1$$Register, $op2$$constant);
14954 %}
14955
14956 ins_pipe(icmp_reg_imm);
14957 %}
14958
14959 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14960 %{
14961 match(Set cr (OverflowSubI zero op1));
14962
14963 format %{ "cmpw zr, $op1\t# overflow check int" %}
14964 ins_cost(INSN_COST);
14965 ins_encode %{
14966 __ cmpw(zr, $op1$$Register);
14967 %}
14968
14969 ins_pipe(icmp_reg_imm);
14970 %}
14971
14972 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14973 %{
14974 match(Set cr (OverflowSubL zero op1));
14975
14976 format %{ "cmp zr, $op1\t# overflow check long" %}
14977 ins_cost(INSN_COST);
14978 ins_encode %{
14979 __ cmp(zr, $op1$$Register);
14980 %}
14981
14982 ins_pipe(icmp_reg_imm);
14983 %}
14984
14985 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14986 %{
14987 match(Set cr (OverflowMulI op1 op2));
14988
14989 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14990 "cmp rscratch1, rscratch1, sxtw\n\t"
14991 "movw rscratch1, #0x80000000\n\t"
14992 "cselw rscratch1, rscratch1, zr, NE\n\t"
14993 "cmpw rscratch1, #1" %}
14994 ins_cost(5 * INSN_COST);
14995 ins_encode %{
14996 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14997 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14998 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14999 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15000 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15001 %}
15002
15003 ins_pipe(pipe_slow);
15004 %}
15005
15006 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15007 %{
15008 match(If cmp (OverflowMulI op1 op2));
15009 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15010 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15011 effect(USE labl, KILL cr);
15012
15013 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15014 "cmp rscratch1, rscratch1, sxtw\n\t"
15015 "b$cmp $labl" %}
15016 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15017 ins_encode %{
15018 Label* L = $labl$$label;
15019 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15020 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15021 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15022 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15023 %}
15024
15025 ins_pipe(pipe_serial);
15026 %}
15027
15028 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15029 %{
15030 match(Set cr (OverflowMulL op1 op2));
15031
15032 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15033 "smulh rscratch2, $op1, $op2\n\t"
15034 "cmp rscratch2, rscratch1, ASR #63\n\t"
15035 "movw rscratch1, #0x80000000\n\t"
15036 "cselw rscratch1, rscratch1, zr, NE\n\t"
15037 "cmpw rscratch1, #1" %}
15038 ins_cost(6 * INSN_COST);
15039 ins_encode %{
15040 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15041 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15042 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15043 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15044 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15045 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15046 %}
15047
15048 ins_pipe(pipe_slow);
15049 %}
15050
15051 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15052 %{
15053 match(If cmp (OverflowMulL op1 op2));
15054 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15055 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15056 effect(USE labl, KILL cr);
15057
15058 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15059 "smulh rscratch2, $op1, $op2\n\t"
15060 "cmp rscratch2, rscratch1, ASR #63\n\t"
15061 "b$cmp $labl" %}
15062 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15063 ins_encode %{
15064 Label* L = $labl$$label;
15065 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15066 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15067 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15068 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15069 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15070 %}
15071
15072 ins_pipe(pipe_serial);
15073 %}
15074
15075 // ============================================================================
15076 // Compare Instructions
15077
15078 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15079 %{
15080 match(Set cr (CmpI op1 op2));
15081
15082 effect(DEF cr, USE op1, USE op2);
15083
15084 ins_cost(INSN_COST);
15085 format %{ "cmpw $op1, $op2" %}
15086
15087 ins_encode(aarch64_enc_cmpw(op1, op2));
15088
15089 ins_pipe(icmp_reg_reg);
15090 %}
15091
15092 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15093 %{
15094 match(Set cr (CmpI op1 zero));
15095
15096 effect(DEF cr, USE op1);
15097
15098 ins_cost(INSN_COST);
15099 format %{ "cmpw $op1, 0" %}
15100
15101 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15102
15103 ins_pipe(icmp_reg_imm);
15104 %}
15105
15106 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15107 %{
15108 match(Set cr (CmpI op1 op2));
15109
15110 effect(DEF cr, USE op1);
15111
15112 ins_cost(INSN_COST);
15113 format %{ "cmpw $op1, $op2" %}
15114
15115 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15116
15117 ins_pipe(icmp_reg_imm);
15118 %}
15119
15120 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15121 %{
15122 match(Set cr (CmpI op1 op2));
15123
15124 effect(DEF cr, USE op1);
15125
15126 ins_cost(INSN_COST * 2);
15127 format %{ "cmpw $op1, $op2" %}
15128
15129 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15130
15131 ins_pipe(icmp_reg_imm);
15132 %}
15133
15134 // Unsigned compare Instructions; really, same as signed compare
15135 // except it should only be used to feed an If or a CMovI which takes a
15136 // cmpOpU.
15137
15138 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15139 %{
15140 match(Set cr (CmpU op1 op2));
15141
15142 effect(DEF cr, USE op1, USE op2);
15143
15144 ins_cost(INSN_COST);
15145 format %{ "cmpw $op1, $op2\t# unsigned" %}
15146
15147 ins_encode(aarch64_enc_cmpw(op1, op2));
15148
15149 ins_pipe(icmp_reg_reg);
15150 %}
15151
15152 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15153 %{
15154 match(Set cr (CmpU op1 zero));
15155
15156 effect(DEF cr, USE op1);
15157
15158 ins_cost(INSN_COST);
15159 format %{ "cmpw $op1, #0\t# unsigned" %}
15160
15161 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15162
15163 ins_pipe(icmp_reg_imm);
15164 %}
15165
15166 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15167 %{
15168 match(Set cr (CmpU op1 op2));
15169
15170 effect(DEF cr, USE op1);
15171
15172 ins_cost(INSN_COST);
15173 format %{ "cmpw $op1, $op2\t# unsigned" %}
15174
15175 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15176
15177 ins_pipe(icmp_reg_imm);
15178 %}
15179
15180 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15181 %{
15182 match(Set cr (CmpU op1 op2));
15183
15184 effect(DEF cr, USE op1);
15185
15186 ins_cost(INSN_COST * 2);
15187 format %{ "cmpw $op1, $op2\t# unsigned" %}
15188
15189 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15190
15191 ins_pipe(icmp_reg_imm);
15192 %}
15193
15194 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15195 %{
15196 match(Set cr (CmpL op1 op2));
15197
15198 effect(DEF cr, USE op1, USE op2);
15199
15200 ins_cost(INSN_COST);
15201 format %{ "cmp $op1, $op2" %}
15202
15203 ins_encode(aarch64_enc_cmp(op1, op2));
15204
15205 ins_pipe(icmp_reg_reg);
15206 %}
15207
15208 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15209 %{
15210 match(Set cr (CmpL op1 zero));
15211
15212 effect(DEF cr, USE op1);
15213
15214 ins_cost(INSN_COST);
15215 format %{ "tst $op1" %}
15216
15217 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15218
15219 ins_pipe(icmp_reg_imm);
15220 %}
15221
15222 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15223 %{
15224 match(Set cr (CmpL op1 op2));
15225
15226 effect(DEF cr, USE op1);
15227
15228 ins_cost(INSN_COST);
15229 format %{ "cmp $op1, $op2" %}
15230
15231 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15232
15233 ins_pipe(icmp_reg_imm);
15234 %}
15235
15236 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15237 %{
15238 match(Set cr (CmpL op1 op2));
15239
15240 effect(DEF cr, USE op1);
15241
15242 ins_cost(INSN_COST * 2);
15243 format %{ "cmp $op1, $op2" %}
15244
15245 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15246
15247 ins_pipe(icmp_reg_imm);
15248 %}
15249
15250 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15251 %{
15252 match(Set cr (CmpUL op1 op2));
15253
15254 effect(DEF cr, USE op1, USE op2);
15255
15256 ins_cost(INSN_COST);
15257 format %{ "cmp $op1, $op2" %}
15258
15259 ins_encode(aarch64_enc_cmp(op1, op2));
15260
15261 ins_pipe(icmp_reg_reg);
15262 %}
15263
15264 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15265 %{
15266 match(Set cr (CmpUL op1 zero));
15267
15268 effect(DEF cr, USE op1);
15269
15270 ins_cost(INSN_COST);
15271 format %{ "tst $op1" %}
15272
15273 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15274
15275 ins_pipe(icmp_reg_imm);
15276 %}
15277
15278 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15279 %{
15280 match(Set cr (CmpUL op1 op2));
15281
15282 effect(DEF cr, USE op1);
15283
15284 ins_cost(INSN_COST);
15285 format %{ "cmp $op1, $op2" %}
15286
15287 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15288
15289 ins_pipe(icmp_reg_imm);
15290 %}
15291
15292 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15293 %{
15294 match(Set cr (CmpUL op1 op2));
15295
15296 effect(DEF cr, USE op1);
15297
15298 ins_cost(INSN_COST * 2);
15299 format %{ "cmp $op1, $op2" %}
15300
15301 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15302
15303 ins_pipe(icmp_reg_imm);
15304 %}
15305
15306 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15307 %{
15308 match(Set cr (CmpP op1 op2));
15309
15310 effect(DEF cr, USE op1, USE op2);
15311
15312 ins_cost(INSN_COST);
15313 format %{ "cmp $op1, $op2\t // ptr" %}
15314
15315 ins_encode(aarch64_enc_cmpp(op1, op2));
15316
15317 ins_pipe(icmp_reg_reg);
15318 %}
15319
15320 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15321 %{
15322 match(Set cr (CmpN op1 op2));
15323
15324 effect(DEF cr, USE op1, USE op2);
15325
15326 ins_cost(INSN_COST);
15327 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15328
15329 ins_encode(aarch64_enc_cmpn(op1, op2));
15330
15331 ins_pipe(icmp_reg_reg);
15332 %}
15333
15334 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15335 %{
15336 match(Set cr (CmpP op1 zero));
15337
15338 effect(DEF cr, USE op1, USE zero);
15339
15340 ins_cost(INSN_COST);
15341 format %{ "cmp $op1, 0\t // ptr" %}
15342
15343 ins_encode(aarch64_enc_testp(op1));
15344
15345 ins_pipe(icmp_reg_imm);
15346 %}
15347
15348 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15349 %{
15350 match(Set cr (CmpN op1 zero));
15351
15352 effect(DEF cr, USE op1, USE zero);
15353
15354 ins_cost(INSN_COST);
15355 format %{ "cmp $op1, 0\t // compressed ptr" %}
15356
15357 ins_encode(aarch64_enc_testn(op1));
15358
15359 ins_pipe(icmp_reg_imm);
15360 %}
15361
15362 // FP comparisons
15363 //
15364 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15365 // using normal cmpOp. See declaration of rFlagsReg for details.
15366
15367 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15368 %{
15369 match(Set cr (CmpF src1 src2));
15370
15371 ins_cost(3 * INSN_COST);
15372 format %{ "fcmps $src1, $src2" %}
15373
15374 ins_encode %{
15375 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15376 %}
15377
15378 ins_pipe(pipe_class_compare);
15379 %}
15380
15381 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15382 %{
15383 match(Set cr (CmpF src1 src2));
15384
15385 ins_cost(3 * INSN_COST);
15386 format %{ "fcmps $src1, 0.0" %}
15387
15388 ins_encode %{
15389 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15390 %}
15391
15392 ins_pipe(pipe_class_compare);
15393 %}
15394 // FROM HERE
15395
15396 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15397 %{
15398 match(Set cr (CmpD src1 src2));
15399
15400 ins_cost(3 * INSN_COST);
15401 format %{ "fcmpd $src1, $src2" %}
15402
15403 ins_encode %{
15404 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15405 %}
15406
15407 ins_pipe(pipe_class_compare);
15408 %}
15409
15410 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15411 %{
15412 match(Set cr (CmpD src1 src2));
15413
15414 ins_cost(3 * INSN_COST);
15415 format %{ "fcmpd $src1, 0.0" %}
15416
15417 ins_encode %{
15418 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15419 %}
15420
15421 ins_pipe(pipe_class_compare);
15422 %}
15423
15424 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15425 %{
15426 match(Set dst (CmpF3 src1 src2));
15427 effect(KILL cr);
15428
15429 ins_cost(5 * INSN_COST);
15430 format %{ "fcmps $src1, $src2\n\t"
15431 "csinvw($dst, zr, zr, eq\n\t"
15432 "csnegw($dst, $dst, $dst, lt)"
15433 %}
15434
15435 ins_encode %{
15436 Label done;
15437 FloatRegister s1 = as_FloatRegister($src1$$reg);
15438 FloatRegister s2 = as_FloatRegister($src2$$reg);
15439 Register d = as_Register($dst$$reg);
15440 __ fcmps(s1, s2);
15441 // installs 0 if EQ else -1
15442 __ csinvw(d, zr, zr, Assembler::EQ);
15443 // keeps -1 if less or unordered else installs 1
15444 __ csnegw(d, d, d, Assembler::LT);
15445 __ bind(done);
15446 %}
15447
15448 ins_pipe(pipe_class_default);
15449
15450 %}
15451
15452 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15453 %{
15454 match(Set dst (CmpD3 src1 src2));
15455 effect(KILL cr);
15456
15457 ins_cost(5 * INSN_COST);
15458 format %{ "fcmpd $src1, $src2\n\t"
15459 "csinvw($dst, zr, zr, eq\n\t"
15460 "csnegw($dst, $dst, $dst, lt)"
15461 %}
15462
15463 ins_encode %{
15464 Label done;
15465 FloatRegister s1 = as_FloatRegister($src1$$reg);
15466 FloatRegister s2 = as_FloatRegister($src2$$reg);
15467 Register d = as_Register($dst$$reg);
15468 __ fcmpd(s1, s2);
15469 // installs 0 if EQ else -1
15470 __ csinvw(d, zr, zr, Assembler::EQ);
15471 // keeps -1 if less or unordered else installs 1
15472 __ csnegw(d, d, d, Assembler::LT);
15473 __ bind(done);
15474 %}
15475 ins_pipe(pipe_class_default);
15476
15477 %}
15478
15479 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15480 %{
15481 match(Set dst (CmpF3 src1 zero));
15482 effect(KILL cr);
15483
15484 ins_cost(5 * INSN_COST);
15485 format %{ "fcmps $src1, 0.0\n\t"
15486 "csinvw($dst, zr, zr, eq\n\t"
15487 "csnegw($dst, $dst, $dst, lt)"
15488 %}
15489
15490 ins_encode %{
15491 Label done;
15492 FloatRegister s1 = as_FloatRegister($src1$$reg);
15493 Register d = as_Register($dst$$reg);
15494 __ fcmps(s1, 0.0);
15495 // installs 0 if EQ else -1
15496 __ csinvw(d, zr, zr, Assembler::EQ);
15497 // keeps -1 if less or unordered else installs 1
15498 __ csnegw(d, d, d, Assembler::LT);
15499 __ bind(done);
15500 %}
15501
15502 ins_pipe(pipe_class_default);
15503
15504 %}
15505
15506 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15507 %{
15508 match(Set dst (CmpD3 src1 zero));
15509 effect(KILL cr);
15510
15511 ins_cost(5 * INSN_COST);
15512 format %{ "fcmpd $src1, 0.0\n\t"
15513 "csinvw($dst, zr, zr, eq\n\t"
15514 "csnegw($dst, $dst, $dst, lt)"
15515 %}
15516
15517 ins_encode %{
15518 Label done;
15519 FloatRegister s1 = as_FloatRegister($src1$$reg);
15520 Register d = as_Register($dst$$reg);
15521 __ fcmpd(s1, 0.0);
15522 // installs 0 if EQ else -1
15523 __ csinvw(d, zr, zr, Assembler::EQ);
15524 // keeps -1 if less or unordered else installs 1
15525 __ csnegw(d, d, d, Assembler::LT);
15526 __ bind(done);
15527 %}
15528 ins_pipe(pipe_class_default);
15529
15530 %}
15531
15532 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15533 %{
15534 match(Set dst (CmpLTMask p q));
15535 effect(KILL cr);
15536
15537 ins_cost(3 * INSN_COST);
15538
15539 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15540 "csetw $dst, lt\n\t"
15541 "subw $dst, zr, $dst"
15542 %}
15543
15544 ins_encode %{
15545 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15546 __ csetw(as_Register($dst$$reg), Assembler::LT);
15547 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15548 %}
15549
15550 ins_pipe(ialu_reg_reg);
15551 %}
15552
15553 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15554 %{
15555 match(Set dst (CmpLTMask src zero));
15556 effect(KILL cr);
15557
15558 ins_cost(INSN_COST);
15559
15560 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15561
15562 ins_encode %{
15563 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15564 %}
15565
15566 ins_pipe(ialu_reg_shift);
15567 %}
15568
15569 // ============================================================================
15570 // Max and Min
15571
15572 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15573
15574 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15575 %{
15576 effect(DEF cr, USE src);
15577 ins_cost(INSN_COST);
15578 format %{ "cmpw $src, 0" %}
15579
15580 ins_encode %{
15581 __ cmpw($src$$Register, 0);
15582 %}
15583 ins_pipe(icmp_reg_imm);
15584 %}
15585
15586 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15587 %{
15588 match(Set dst (MinI src1 src2));
15589 ins_cost(INSN_COST * 3);
15590
15591 expand %{
15592 rFlagsReg cr;
15593 compI_reg_reg(cr, src1, src2);
15594 cmovI_reg_reg_lt(dst, src1, src2, cr);
15595 %}
15596 %}
15597
15598 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15599 %{
15600 match(Set dst (MaxI src1 src2));
15601 ins_cost(INSN_COST * 3);
15602
15603 expand %{
15604 rFlagsReg cr;
15605 compI_reg_reg(cr, src1, src2);
15606 cmovI_reg_reg_gt(dst, src1, src2, cr);
15607 %}
15608 %}
15609
15610
15611 // ============================================================================
15612 // Branch Instructions
15613
15614 // Direct Branch.
15615 instruct branch(label lbl)
15616 %{
15617 match(Goto);
15618
15619 effect(USE lbl);
15620
15621 ins_cost(BRANCH_COST);
15622 format %{ "b $lbl" %}
15623
15624 ins_encode(aarch64_enc_b(lbl));
15625
15626 ins_pipe(pipe_branch);
15627 %}
15628
15629 // Conditional Near Branch
15630 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15631 %{
15632 // Same match rule as `branchConFar'.
15633 match(If cmp cr);
15634
15635 effect(USE lbl);
15636
15637 ins_cost(BRANCH_COST);
15638 // If set to 1 this indicates that the current instruction is a
15639 // short variant of a long branch. This avoids using this
15640 // instruction in first-pass matching. It will then only be used in
15641 // the `Shorten_branches' pass.
15642 // ins_short_branch(1);
15643 format %{ "b$cmp $lbl" %}
15644
15645 ins_encode(aarch64_enc_br_con(cmp, lbl));
15646
15647 ins_pipe(pipe_branch_cond);
15648 %}
15649
15650 // Conditional Near Branch Unsigned
15651 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15652 %{
15653 // Same match rule as `branchConFar'.
15654 match(If cmp cr);
15655
15656 effect(USE lbl);
15657
15658 ins_cost(BRANCH_COST);
15659 // If set to 1 this indicates that the current instruction is a
15660 // short variant of a long branch. This avoids using this
15661 // instruction in first-pass matching. It will then only be used in
15662 // the `Shorten_branches' pass.
15663 // ins_short_branch(1);
15664 format %{ "b$cmp $lbl\t# unsigned" %}
15665
15666 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15667
15668 ins_pipe(pipe_branch_cond);
15669 %}
15670
15671 // Make use of CBZ and CBNZ. These instructions, as well as being
15672 // shorter than (cmp; branch), have the additional benefit of not
15673 // killing the flags.
15674
15675 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15676 match(If cmp (CmpI op1 op2));
15677 effect(USE labl);
15678
15679 ins_cost(BRANCH_COST);
15680 format %{ "cbw$cmp $op1, $labl" %}
15681 ins_encode %{
15682 Label* L = $labl$$label;
15683 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15684 if (cond == Assembler::EQ)
15685 __ cbzw($op1$$Register, *L);
15686 else
15687 __ cbnzw($op1$$Register, *L);
15688 %}
15689 ins_pipe(pipe_cmp_branch);
15690 %}
15691
15692 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15693 match(If cmp (CmpL op1 op2));
15694 effect(USE labl);
15695
15696 ins_cost(BRANCH_COST);
15697 format %{ "cb$cmp $op1, $labl" %}
15698 ins_encode %{
15699 Label* L = $labl$$label;
15700 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15701 if (cond == Assembler::EQ)
15702 __ cbz($op1$$Register, *L);
15703 else
15704 __ cbnz($op1$$Register, *L);
15705 %}
15706 ins_pipe(pipe_cmp_branch);
15707 %}
15708
15709 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15710 match(If cmp (CmpP op1 op2));
15711 effect(USE labl);
15712
15713 ins_cost(BRANCH_COST);
15714 format %{ "cb$cmp $op1, $labl" %}
15715 ins_encode %{
15716 Label* L = $labl$$label;
15717 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15718 if (cond == Assembler::EQ)
15719 __ cbz($op1$$Register, *L);
15720 else
15721 __ cbnz($op1$$Register, *L);
15722 %}
15723 ins_pipe(pipe_cmp_branch);
15724 %}
15725
15726 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15727 match(If cmp (CmpN op1 op2));
15728 effect(USE labl);
15729
15730 ins_cost(BRANCH_COST);
15731 format %{ "cbw$cmp $op1, $labl" %}
15732 ins_encode %{
15733 Label* L = $labl$$label;
15734 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15735 if (cond == Assembler::EQ)
15736 __ cbzw($op1$$Register, *L);
15737 else
15738 __ cbnzw($op1$$Register, *L);
15739 %}
15740 ins_pipe(pipe_cmp_branch);
15741 %}
15742
15743 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15744 match(If cmp (CmpP (DecodeN oop) zero));
15745 effect(USE labl);
15746
15747 ins_cost(BRANCH_COST);
15748 format %{ "cb$cmp $oop, $labl" %}
15749 ins_encode %{
15750 Label* L = $labl$$label;
15751 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15752 if (cond == Assembler::EQ)
15753 __ cbzw($oop$$Register, *L);
15754 else
15755 __ cbnzw($oop$$Register, *L);
15756 %}
15757 ins_pipe(pipe_cmp_branch);
15758 %}
15759
15760 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15761 match(If cmp (CmpU op1 op2));
15762 effect(USE labl);
15763
15764 ins_cost(BRANCH_COST);
15765 format %{ "cbw$cmp $op1, $labl" %}
15766 ins_encode %{
15767 Label* L = $labl$$label;
15768 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15769 if (cond == Assembler::EQ || cond == Assembler::LS) {
15770 __ cbzw($op1$$Register, *L);
15771 } else {
15772 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15773 __ cbnzw($op1$$Register, *L);
15774 }
15775 %}
15776 ins_pipe(pipe_cmp_branch);
15777 %}
15778
15779 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15780 match(If cmp (CmpUL op1 op2));
15781 effect(USE labl);
15782
15783 ins_cost(BRANCH_COST);
15784 format %{ "cb$cmp $op1, $labl" %}
15785 ins_encode %{
15786 Label* L = $labl$$label;
15787 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15788 if (cond == Assembler::EQ || cond == Assembler::LS) {
15789 __ cbz($op1$$Register, *L);
15790 } else {
15791 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15792 __ cbnz($op1$$Register, *L);
15793 }
15794 %}
15795 ins_pipe(pipe_cmp_branch);
15796 %}
15797
15798 // Test bit and Branch
15799
15800 // Patterns for short (< 32KiB) variants
15801 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15802 match(If cmp (CmpL op1 op2));
15803 effect(USE labl);
15804
15805 ins_cost(BRANCH_COST);
15806 format %{ "cb$cmp $op1, $labl # long" %}
15807 ins_encode %{
15808 Label* L = $labl$$label;
15809 Assembler::Condition cond =
15810 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15811 __ tbr(cond, $op1$$Register, 63, *L);
15812 %}
15813 ins_pipe(pipe_cmp_branch);
15814 ins_short_branch(1);
15815 %}
15816
15817 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15818 match(If cmp (CmpI op1 op2));
15819 effect(USE labl);
15820
15821 ins_cost(BRANCH_COST);
15822 format %{ "cb$cmp $op1, $labl # int" %}
15823 ins_encode %{
15824 Label* L = $labl$$label;
15825 Assembler::Condition cond =
15826 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15827 __ tbr(cond, $op1$$Register, 31, *L);
15828 %}
15829 ins_pipe(pipe_cmp_branch);
15830 ins_short_branch(1);
15831 %}
15832
15833 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15834 match(If cmp (CmpL (AndL op1 op2) op3));
15835 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15836 effect(USE labl);
15837
15838 ins_cost(BRANCH_COST);
15839 format %{ "tb$cmp $op1, $op2, $labl" %}
15840 ins_encode %{
15841 Label* L = $labl$$label;
15842 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15843 int bit = exact_log2_long($op2$$constant);
15844 __ tbr(cond, $op1$$Register, bit, *L);
15845 %}
15846 ins_pipe(pipe_cmp_branch);
15847 ins_short_branch(1);
15848 %}
15849
15850 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15851 match(If cmp (CmpI (AndI op1 op2) op3));
15852 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15853 effect(USE labl);
15854
15855 ins_cost(BRANCH_COST);
15856 format %{ "tb$cmp $op1, $op2, $labl" %}
15857 ins_encode %{
15858 Label* L = $labl$$label;
15859 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15860 int bit = exact_log2((juint)$op2$$constant);
15861 __ tbr(cond, $op1$$Register, bit, *L);
15862 %}
15863 ins_pipe(pipe_cmp_branch);
15864 ins_short_branch(1);
15865 %}
15866
15867 // And far variants
15868 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15869 match(If cmp (CmpL op1 op2));
15870 effect(USE labl);
15871
15872 ins_cost(BRANCH_COST);
15873 format %{ "cb$cmp $op1, $labl # long" %}
15874 ins_encode %{
15875 Label* L = $labl$$label;
15876 Assembler::Condition cond =
15877 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15878 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15879 %}
15880 ins_pipe(pipe_cmp_branch);
15881 %}
15882
15883 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15884 match(If cmp (CmpI op1 op2));
15885 effect(USE labl);
15886
15887 ins_cost(BRANCH_COST);
15888 format %{ "cb$cmp $op1, $labl # int" %}
15889 ins_encode %{
15890 Label* L = $labl$$label;
15891 Assembler::Condition cond =
15892 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15893 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15894 %}
15895 ins_pipe(pipe_cmp_branch);
15896 %}
15897
15898 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15899 match(If cmp (CmpL (AndL op1 op2) op3));
15900 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15901 effect(USE labl);
15902
15903 ins_cost(BRANCH_COST);
15904 format %{ "tb$cmp $op1, $op2, $labl" %}
15905 ins_encode %{
15906 Label* L = $labl$$label;
15907 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15908 int bit = exact_log2_long($op2$$constant);
15909 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15910 %}
15911 ins_pipe(pipe_cmp_branch);
15912 %}
15913
15914 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15915 match(If cmp (CmpI (AndI op1 op2) op3));
15916 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15917 effect(USE labl);
15918
15919 ins_cost(BRANCH_COST);
15920 format %{ "tb$cmp $op1, $op2, $labl" %}
15921 ins_encode %{
15922 Label* L = $labl$$label;
15923 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15924 int bit = exact_log2((juint)$op2$$constant);
15925 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15926 %}
15927 ins_pipe(pipe_cmp_branch);
15928 %}
15929
15930 // Test bits
15931
15932 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15933 match(Set cr (CmpL (AndL op1 op2) op3));
15934 predicate(Assembler::operand_valid_for_logical_immediate
15935 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15936
15937 ins_cost(INSN_COST);
15938 format %{ "tst $op1, $op2 # long" %}
15939 ins_encode %{
15940 __ tst($op1$$Register, $op2$$constant);
15941 %}
15942 ins_pipe(ialu_reg_reg);
15943 %}
15944
15945 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15946 match(Set cr (CmpI (AndI op1 op2) op3));
15947 predicate(Assembler::operand_valid_for_logical_immediate
15948 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15949
15950 ins_cost(INSN_COST);
15951 format %{ "tst $op1, $op2 # int" %}
15952 ins_encode %{
15953 __ tstw($op1$$Register, $op2$$constant);
15954 %}
15955 ins_pipe(ialu_reg_reg);
15956 %}
15957
15958 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15959 match(Set cr (CmpL (AndL op1 op2) op3));
15960
15961 ins_cost(INSN_COST);
15962 format %{ "tst $op1, $op2 # long" %}
15963 ins_encode %{
15964 __ tst($op1$$Register, $op2$$Register);
15965 %}
15966 ins_pipe(ialu_reg_reg);
15967 %}
15968
15969 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15970 match(Set cr (CmpI (AndI op1 op2) op3));
15971
15972 ins_cost(INSN_COST);
15973 format %{ "tstw $op1, $op2 # int" %}
15974 ins_encode %{
15975 __ tstw($op1$$Register, $op2$$Register);
15976 %}
15977 ins_pipe(ialu_reg_reg);
15978 %}
15979
15980
15981 // Conditional Far Branch
15982 // Conditional Far Branch Unsigned
15983 // TODO: fixme
15984
15985 // counted loop end branch near
15986 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15987 %{
15988 match(CountedLoopEnd cmp cr);
15989
15990 effect(USE lbl);
15991
15992 ins_cost(BRANCH_COST);
15993 // short variant.
15994 // ins_short_branch(1);
15995 format %{ "b$cmp $lbl \t// counted loop end" %}
15996
15997 ins_encode(aarch64_enc_br_con(cmp, lbl));
15998
15999 ins_pipe(pipe_branch);
16000 %}
16001
16002 // counted loop end branch far
16003 // TODO: fixme
16004
16005 // ============================================================================
16006 // inlined locking and unlocking
16007
16008 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16009 %{
16010 predicate(LockingMode != LM_LIGHTWEIGHT);
16011 match(Set cr (FastLock object box));
16012 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16013
16014 ins_cost(5 * INSN_COST);
16015 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16016
16017 ins_encode %{
16018 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16019 %}
16020
16021 ins_pipe(pipe_serial);
16022 %}
16023
16024 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16025 %{
16026 predicate(LockingMode != LM_LIGHTWEIGHT);
16027 match(Set cr (FastUnlock object box));
16028 effect(TEMP tmp, TEMP tmp2);
16029
16030 ins_cost(5 * INSN_COST);
16031 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16032
16033 ins_encode %{
16034 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16035 %}
16036
16037 ins_pipe(pipe_serial);
16038 %}
16039
16040 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16041 %{
16042 predicate(LockingMode == LM_LIGHTWEIGHT);
16043 match(Set cr (FastLock object box));
16044 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16045
16046 ins_cost(5 * INSN_COST);
16047 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16048
16049 ins_encode %{
16050 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16051 %}
16052
16053 ins_pipe(pipe_serial);
16054 %}
16055
16056 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16057 %{
16058 predicate(LockingMode == LM_LIGHTWEIGHT);
16059 match(Set cr (FastUnlock object box));
16060 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16061
16062 ins_cost(5 * INSN_COST);
16063 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16064
16065 ins_encode %{
16066 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16067 %}
16068
16069 ins_pipe(pipe_serial);
16070 %}
16071
16072 // ============================================================================
16073 // Safepoint Instructions
16074
16075 // TODO
16076 // provide a near and far version of this code
16077
16078 instruct safePoint(rFlagsReg cr, iRegP poll)
16079 %{
16080 match(SafePoint poll);
16081 effect(KILL cr);
16082
16083 format %{
16084 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16085 %}
16086 ins_encode %{
16087 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16088 %}
16089 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16090 %}
16091
16092
16093 // ============================================================================
16094 // Procedure Call/Return Instructions
16095
16096 // Call Java Static Instruction
16097
16098 instruct CallStaticJavaDirect(method meth)
16099 %{
16100 match(CallStaticJava);
16101
16102 effect(USE meth);
16103
16104 ins_cost(CALL_COST);
16105
16106 format %{ "call,static $meth \t// ==> " %}
16107
16108 ins_encode(aarch64_enc_java_static_call(meth),
16109 aarch64_enc_call_epilog);
16110
16111 ins_pipe(pipe_class_call);
16112 %}
16113
16114 // TO HERE
16115
16116 // Call Java Dynamic Instruction
16117 instruct CallDynamicJavaDirect(method meth)
16118 %{
16119 match(CallDynamicJava);
16120
16121 effect(USE meth);
16122
16123 ins_cost(CALL_COST);
16124
16125 format %{ "CALL,dynamic $meth \t// ==> " %}
16126
16127 ins_encode(aarch64_enc_java_dynamic_call(meth),
16128 aarch64_enc_call_epilog);
16129
16130 ins_pipe(pipe_class_call);
16131 %}
16132
16133 // Call Runtime Instruction
16134
16135 instruct CallRuntimeDirect(method meth)
16136 %{
16137 match(CallRuntime);
16138
16139 effect(USE meth);
16140
16141 ins_cost(CALL_COST);
16142
16143 format %{ "CALL, runtime $meth" %}
16144
16145 ins_encode( aarch64_enc_java_to_runtime(meth) );
16146
16147 ins_pipe(pipe_class_call);
16148 %}
16149
16150 // Call Runtime Instruction
16151
16152 instruct CallLeafDirect(method meth)
16153 %{
16154 match(CallLeaf);
16155
16156 effect(USE meth);
16157
16158 ins_cost(CALL_COST);
16159
16160 format %{ "CALL, runtime leaf $meth" %}
16161
16162 ins_encode( aarch64_enc_java_to_runtime(meth) );
16163
16164 ins_pipe(pipe_class_call);
16165 %}
16166
16167 // Call Runtime Instruction
16168
16169 // entry point is null, target holds the address to call
16170 instruct CallLeafNoFPIndirect(iRegP target)
16171 %{
16172 predicate(n->as_Call()->entry_point() == nullptr);
16173
16174 match(CallLeafNoFP target);
16175
16176 ins_cost(CALL_COST);
16177
16178 format %{ "CALL, runtime leaf nofp indirect $target" %}
16179
16180 ins_encode %{
16181 __ blr($target$$Register);
16182 %}
16183
16184 ins_pipe(pipe_class_call);
16185 %}
16186
16187 instruct CallLeafNoFPDirect(method meth)
16188 %{
16189 predicate(n->as_Call()->entry_point() != nullptr);
16190
16191 match(CallLeafNoFP);
16192
16193 effect(USE meth);
16194
16195 ins_cost(CALL_COST);
16196
16197 format %{ "CALL, runtime leaf nofp $meth" %}
16198
16199 ins_encode( aarch64_enc_java_to_runtime(meth) );
16200
16201 ins_pipe(pipe_class_call);
16202 %}
16203
16204 // Tail Call; Jump from runtime stub to Java code.
16205 // Also known as an 'interprocedural jump'.
16206 // Target of jump will eventually return to caller.
16207 // TailJump below removes the return address.
16208 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16209 // emitted just above the TailCall which has reset rfp to the caller state.
16210 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16211 %{
16212 match(TailCall jump_target method_ptr);
16213
16214 ins_cost(CALL_COST);
16215
16216 format %{ "br $jump_target\t# $method_ptr holds method" %}
16217
16218 ins_encode(aarch64_enc_tail_call(jump_target));
16219
16220 ins_pipe(pipe_class_call);
16221 %}
16222
16223 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16224 %{
16225 match(TailJump jump_target ex_oop);
16226
16227 ins_cost(CALL_COST);
16228
16229 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16230
16231 ins_encode(aarch64_enc_tail_jmp(jump_target));
16232
16233 ins_pipe(pipe_class_call);
16234 %}
16235
16236 // Forward exception.
16237 instruct ForwardExceptionjmp()
16238 %{
16239 match(ForwardException);
16240 ins_cost(CALL_COST);
16241
16242 format %{ "b forward_exception_stub" %}
16243 ins_encode %{
16244 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16245 %}
16246 ins_pipe(pipe_class_call);
16247 %}
16248
16249 // Create exception oop: created by stack-crawling runtime code.
16250 // Created exception is now available to this handler, and is setup
16251 // just prior to jumping to this handler. No code emitted.
16252 // TODO check
16253 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16254 instruct CreateException(iRegP_R0 ex_oop)
16255 %{
16256 match(Set ex_oop (CreateEx));
16257
16258 format %{ " -- \t// exception oop; no code emitted" %}
16259
16260 size(0);
16261
16262 ins_encode( /*empty*/ );
16263
16264 ins_pipe(pipe_class_empty);
16265 %}
16266
16267 // Rethrow exception: The exception oop will come in the first
16268 // argument position. Then JUMP (not call) to the rethrow stub code.
16269 instruct RethrowException() %{
16270 match(Rethrow);
16271 ins_cost(CALL_COST);
16272
16273 format %{ "b rethrow_stub" %}
16274
16275 ins_encode( aarch64_enc_rethrow() );
16276
16277 ins_pipe(pipe_class_call);
16278 %}
16279
16280
16281 // Return Instruction
16282 // epilog node loads ret address into lr as part of frame pop
16283 instruct Ret()
16284 %{
16285 match(Return);
16286
16287 format %{ "ret\t// return register" %}
16288
16289 ins_encode( aarch64_enc_ret() );
16290
16291 ins_pipe(pipe_branch);
16292 %}
16293
16294 // Die now.
16295 instruct ShouldNotReachHere() %{
16296 match(Halt);
16297
16298 ins_cost(CALL_COST);
16299 format %{ "ShouldNotReachHere" %}
16300
16301 ins_encode %{
16302 if (is_reachable()) {
16303 __ stop(_halt_reason);
16304 }
16305 %}
16306
16307 ins_pipe(pipe_class_default);
16308 %}
16309
16310 // ============================================================================
16311 // Partial Subtype Check
16312 //
16313 // superklass array for an instance of the superklass. Set a hidden
16314 // internal cache on a hit (cache is checked with exposed code in
16315 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16316 // encoding ALSO sets flags.
16317
16318 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16319 %{
16320 match(Set result (PartialSubtypeCheck sub super));
16321 effect(KILL cr, KILL temp);
16322
16323 ins_cost(1100); // slightly larger than the next version
16324 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16325
16326 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16327
16328 opcode(0x1); // Force zero of result reg on hit
16329
16330 ins_pipe(pipe_class_memory);
16331 %}
16332
16333 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16334 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16335 rFlagsReg cr)
16336 %{
16337 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16338 predicate(UseSecondarySupersTable);
16339 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16340
16341 ins_cost(700); // smaller than the next version
16342 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16343
16344 ins_encode %{
16345 bool success = false;
16346 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16347 if (InlineSecondarySupersTest) {
16348 success = __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register,
16349 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16350 $vtemp$$FloatRegister,
16351 $result$$Register,
16352 super_klass_slot);
16353 } else {
16354 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16355 success = (call != nullptr);
16356 }
16357 if (!success) {
16358 ciEnv::current()->record_failure("CodeCache is full");
16359 return;
16360 }
16361 %}
16362
16363 ins_pipe(pipe_class_memory);
16364 %}
16365
16366 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16367 %{
16368 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16369 effect(KILL temp, KILL result);
16370
16371 ins_cost(1100); // slightly larger than the next version
16372 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16373
16374 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16375
16376 opcode(0x0); // Don't zero result reg on hit
16377
16378 ins_pipe(pipe_class_memory);
16379 %}
16380
16381 // Intrisics for String.compareTo()
16382
16383 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16384 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16385 %{
16386 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16387 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16388 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16389
16390 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16391 ins_encode %{
16392 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16393 __ string_compare($str1$$Register, $str2$$Register,
16394 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16395 $tmp1$$Register, $tmp2$$Register,
16396 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16397 %}
16398 ins_pipe(pipe_class_memory);
16399 %}
16400
16401 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16402 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16403 %{
16404 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16405 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16406 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16407
16408 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16409 ins_encode %{
16410 __ string_compare($str1$$Register, $str2$$Register,
16411 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16412 $tmp1$$Register, $tmp2$$Register,
16413 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16414 %}
16415 ins_pipe(pipe_class_memory);
16416 %}
16417
16418 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16419 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16420 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16421 %{
16422 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16423 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16424 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16425 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16426
16427 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16428 ins_encode %{
16429 __ string_compare($str1$$Register, $str2$$Register,
16430 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16431 $tmp1$$Register, $tmp2$$Register,
16432 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16433 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16434 %}
16435 ins_pipe(pipe_class_memory);
16436 %}
16437
16438 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16439 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16440 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16441 %{
16442 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16443 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16444 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16445 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16446
16447 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16448 ins_encode %{
16449 __ string_compare($str1$$Register, $str2$$Register,
16450 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16451 $tmp1$$Register, $tmp2$$Register,
16452 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16453 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16454 %}
16455 ins_pipe(pipe_class_memory);
16456 %}
16457
16458 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16459 // these string_compare variants as NEON register type for convenience so that the prototype of
16460 // string_compare can be shared with all variants.
16461
16462 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16463 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16464 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16465 pRegGov_P1 pgtmp2, rFlagsReg cr)
16466 %{
16467 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16468 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16469 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16470 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16471
16472 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16473 ins_encode %{
16474 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16475 __ string_compare($str1$$Register, $str2$$Register,
16476 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16477 $tmp1$$Register, $tmp2$$Register,
16478 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16479 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16480 StrIntrinsicNode::LL);
16481 %}
16482 ins_pipe(pipe_class_memory);
16483 %}
16484
16485 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16486 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16487 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16488 pRegGov_P1 pgtmp2, rFlagsReg cr)
16489 %{
16490 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16491 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16492 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16493 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16494
16495 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16496 ins_encode %{
16497 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16498 __ string_compare($str1$$Register, $str2$$Register,
16499 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16500 $tmp1$$Register, $tmp2$$Register,
16501 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16502 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16503 StrIntrinsicNode::LU);
16504 %}
16505 ins_pipe(pipe_class_memory);
16506 %}
16507
16508 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16509 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16510 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16511 pRegGov_P1 pgtmp2, rFlagsReg cr)
16512 %{
16513 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16514 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16515 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16516 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16517
16518 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16519 ins_encode %{
16520 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16521 __ string_compare($str1$$Register, $str2$$Register,
16522 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16523 $tmp1$$Register, $tmp2$$Register,
16524 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16525 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16526 StrIntrinsicNode::UL);
16527 %}
16528 ins_pipe(pipe_class_memory);
16529 %}
16530
16531 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16532 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16533 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16534 pRegGov_P1 pgtmp2, rFlagsReg cr)
16535 %{
16536 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16537 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16538 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16539 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16540
16541 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16542 ins_encode %{
16543 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16544 __ string_compare($str1$$Register, $str2$$Register,
16545 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16546 $tmp1$$Register, $tmp2$$Register,
16547 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16548 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16549 StrIntrinsicNode::UU);
16550 %}
16551 ins_pipe(pipe_class_memory);
16552 %}
16553
16554 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16555 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16556 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16557 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16558 %{
16559 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16560 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16561 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16562 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16563 TEMP vtmp0, TEMP vtmp1, KILL cr);
16564 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16565 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16566
16567 ins_encode %{
16568 __ string_indexof($str1$$Register, $str2$$Register,
16569 $cnt1$$Register, $cnt2$$Register,
16570 $tmp1$$Register, $tmp2$$Register,
16571 $tmp3$$Register, $tmp4$$Register,
16572 $tmp5$$Register, $tmp6$$Register,
16573 -1, $result$$Register, StrIntrinsicNode::UU);
16574 %}
16575 ins_pipe(pipe_class_memory);
16576 %}
16577
16578 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16579 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16580 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16581 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16582 %{
16583 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16584 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16585 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16586 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16587 TEMP vtmp0, TEMP vtmp1, KILL cr);
16588 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16589 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16590
16591 ins_encode %{
16592 __ string_indexof($str1$$Register, $str2$$Register,
16593 $cnt1$$Register, $cnt2$$Register,
16594 $tmp1$$Register, $tmp2$$Register,
16595 $tmp3$$Register, $tmp4$$Register,
16596 $tmp5$$Register, $tmp6$$Register,
16597 -1, $result$$Register, StrIntrinsicNode::LL);
16598 %}
16599 ins_pipe(pipe_class_memory);
16600 %}
16601
16602 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16603 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16604 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16605 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16606 %{
16607 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16608 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16609 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16610 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16611 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16612 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16613 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16614
16615 ins_encode %{
16616 __ string_indexof($str1$$Register, $str2$$Register,
16617 $cnt1$$Register, $cnt2$$Register,
16618 $tmp1$$Register, $tmp2$$Register,
16619 $tmp3$$Register, $tmp4$$Register,
16620 $tmp5$$Register, $tmp6$$Register,
16621 -1, $result$$Register, StrIntrinsicNode::UL);
16622 %}
16623 ins_pipe(pipe_class_memory);
16624 %}
16625
16626 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16627 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16628 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16629 %{
16630 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16631 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16632 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16633 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16634 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16635 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16636
16637 ins_encode %{
16638 int icnt2 = (int)$int_cnt2$$constant;
16639 __ string_indexof($str1$$Register, $str2$$Register,
16640 $cnt1$$Register, zr,
16641 $tmp1$$Register, $tmp2$$Register,
16642 $tmp3$$Register, $tmp4$$Register, zr, zr,
16643 icnt2, $result$$Register, StrIntrinsicNode::UU);
16644 %}
16645 ins_pipe(pipe_class_memory);
16646 %}
16647
16648 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16649 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16650 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16651 %{
16652 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16653 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16654 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16655 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16656 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16657 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16658
16659 ins_encode %{
16660 int icnt2 = (int)$int_cnt2$$constant;
16661 __ string_indexof($str1$$Register, $str2$$Register,
16662 $cnt1$$Register, zr,
16663 $tmp1$$Register, $tmp2$$Register,
16664 $tmp3$$Register, $tmp4$$Register, zr, zr,
16665 icnt2, $result$$Register, StrIntrinsicNode::LL);
16666 %}
16667 ins_pipe(pipe_class_memory);
16668 %}
16669
16670 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16671 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16672 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16673 %{
16674 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16675 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16676 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16677 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16678 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16679 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16680
16681 ins_encode %{
16682 int icnt2 = (int)$int_cnt2$$constant;
16683 __ string_indexof($str1$$Register, $str2$$Register,
16684 $cnt1$$Register, zr,
16685 $tmp1$$Register, $tmp2$$Register,
16686 $tmp3$$Register, $tmp4$$Register, zr, zr,
16687 icnt2, $result$$Register, StrIntrinsicNode::UL);
16688 %}
16689 ins_pipe(pipe_class_memory);
16690 %}
16691
16692 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16693 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16694 iRegINoSp tmp3, rFlagsReg cr)
16695 %{
16696 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16697 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16698 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16699 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16700
16701 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16702
16703 ins_encode %{
16704 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16705 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16706 $tmp3$$Register);
16707 %}
16708 ins_pipe(pipe_class_memory);
16709 %}
16710
16711 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16712 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16713 iRegINoSp tmp3, rFlagsReg cr)
16714 %{
16715 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16716 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16717 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16718 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16719
16720 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16721
16722 ins_encode %{
16723 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16724 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16725 $tmp3$$Register);
16726 %}
16727 ins_pipe(pipe_class_memory);
16728 %}
16729
16730 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16731 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16732 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16733 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16734 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16735 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16736 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16737 ins_encode %{
16738 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16739 $result$$Register, $ztmp1$$FloatRegister,
16740 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16741 $ptmp$$PRegister, true /* isL */);
16742 %}
16743 ins_pipe(pipe_class_memory);
16744 %}
16745
16746 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16747 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16748 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16749 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16750 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16751 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16752 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16753 ins_encode %{
16754 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16755 $result$$Register, $ztmp1$$FloatRegister,
16756 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16757 $ptmp$$PRegister, false /* isL */);
16758 %}
16759 ins_pipe(pipe_class_memory);
16760 %}
16761
16762 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16763 iRegI_R0 result, rFlagsReg cr)
16764 %{
16765 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16766 match(Set result (StrEquals (Binary str1 str2) cnt));
16767 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16768
16769 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16770 ins_encode %{
16771 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16772 __ string_equals($str1$$Register, $str2$$Register,
16773 $result$$Register, $cnt$$Register);
16774 %}
16775 ins_pipe(pipe_class_memory);
16776 %}
16777
16778 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16779 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16780 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16781 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16782 iRegP_R10 tmp, rFlagsReg cr)
16783 %{
16784 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16785 match(Set result (AryEq ary1 ary2));
16786 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16787 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16788 TEMP vtmp6, TEMP vtmp7, KILL cr);
16789
16790 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16791 ins_encode %{
16792 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16793 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16794 $result$$Register, $tmp$$Register, 1);
16795 if (tpc == nullptr) {
16796 ciEnv::current()->record_failure("CodeCache is full");
16797 return;
16798 }
16799 %}
16800 ins_pipe(pipe_class_memory);
16801 %}
16802
16803 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16804 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16805 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16806 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16807 iRegP_R10 tmp, rFlagsReg cr)
16808 %{
16809 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16810 match(Set result (AryEq ary1 ary2));
16811 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16812 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16813 TEMP vtmp6, TEMP vtmp7, KILL cr);
16814
16815 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16816 ins_encode %{
16817 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16818 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16819 $result$$Register, $tmp$$Register, 2);
16820 if (tpc == nullptr) {
16821 ciEnv::current()->record_failure("CodeCache is full");
16822 return;
16823 }
16824 %}
16825 ins_pipe(pipe_class_memory);
16826 %}
16827
16828 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16829 %{
16830 match(Set result (CountPositives ary1 len));
16831 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16832 format %{ "count positives byte[] $ary1,$len -> $result" %}
16833 ins_encode %{
16834 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16835 if (tpc == nullptr) {
16836 ciEnv::current()->record_failure("CodeCache is full");
16837 return;
16838 }
16839 %}
16840 ins_pipe( pipe_slow );
16841 %}
16842
16843 // fast char[] to byte[] compression
16844 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16845 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16846 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16847 iRegI_R0 result, rFlagsReg cr)
16848 %{
16849 match(Set result (StrCompressedCopy src (Binary dst len)));
16850 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16851 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16852
16853 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16854 ins_encode %{
16855 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16856 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16857 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16858 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16859 %}
16860 ins_pipe(pipe_slow);
16861 %}
16862
16863 // fast byte[] to char[] inflation
16864 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16865 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16866 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16867 %{
16868 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16869 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16870 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16871 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16872
16873 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16874 ins_encode %{
16875 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16876 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16877 $vtmp2$$FloatRegister, $tmp$$Register);
16878 if (tpc == nullptr) {
16879 ciEnv::current()->record_failure("CodeCache is full");
16880 return;
16881 }
16882 %}
16883 ins_pipe(pipe_class_memory);
16884 %}
16885
16886 // encode char[] to byte[] in ISO_8859_1
16887 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16888 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16889 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16890 iRegI_R0 result, rFlagsReg cr)
16891 %{
16892 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16893 match(Set result (EncodeISOArray src (Binary dst len)));
16894 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16895 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16896
16897 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16898 ins_encode %{
16899 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16900 $result$$Register, false,
16901 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16902 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16903 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16904 %}
16905 ins_pipe(pipe_class_memory);
16906 %}
16907
16908 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16909 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16910 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16911 iRegI_R0 result, rFlagsReg cr)
16912 %{
16913 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16914 match(Set result (EncodeISOArray src (Binary dst len)));
16915 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16916 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16917
16918 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16919 ins_encode %{
16920 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16921 $result$$Register, true,
16922 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16923 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16924 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16925 %}
16926 ins_pipe(pipe_class_memory);
16927 %}
16928
16929 //----------------------------- CompressBits/ExpandBits ------------------------
16930
16931 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16932 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16933 match(Set dst (CompressBits src mask));
16934 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16935 format %{ "mov $tsrc, $src\n\t"
16936 "mov $tmask, $mask\n\t"
16937 "bext $tdst, $tsrc, $tmask\n\t"
16938 "mov $dst, $tdst"
16939 %}
16940 ins_encode %{
16941 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16942 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16943 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16944 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16945 %}
16946 ins_pipe(pipe_slow);
16947 %}
16948
16949 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16950 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16951 match(Set dst (CompressBits (LoadI mem) mask));
16952 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16953 format %{ "ldrs $tsrc, $mem\n\t"
16954 "ldrs $tmask, $mask\n\t"
16955 "bext $tdst, $tsrc, $tmask\n\t"
16956 "mov $dst, $tdst"
16957 %}
16958 ins_encode %{
16959 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16961 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16962 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16963 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16964 %}
16965 ins_pipe(pipe_slow);
16966 %}
16967
16968 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16969 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16970 match(Set dst (CompressBits src mask));
16971 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16972 format %{ "mov $tsrc, $src\n\t"
16973 "mov $tmask, $mask\n\t"
16974 "bext $tdst, $tsrc, $tmask\n\t"
16975 "mov $dst, $tdst"
16976 %}
16977 ins_encode %{
16978 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16979 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16980 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16981 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16982 %}
16983 ins_pipe(pipe_slow);
16984 %}
16985
16986 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16987 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16988 match(Set dst (CompressBits (LoadL mem) mask));
16989 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16990 format %{ "ldrd $tsrc, $mem\n\t"
16991 "ldrd $tmask, $mask\n\t"
16992 "bext $tdst, $tsrc, $tmask\n\t"
16993 "mov $dst, $tdst"
16994 %}
16995 ins_encode %{
16996 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16997 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16998 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16999 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17000 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17001 %}
17002 ins_pipe(pipe_slow);
17003 %}
17004
17005 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17006 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17007 match(Set dst (ExpandBits src mask));
17008 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17009 format %{ "mov $tsrc, $src\n\t"
17010 "mov $tmask, $mask\n\t"
17011 "bdep $tdst, $tsrc, $tmask\n\t"
17012 "mov $dst, $tdst"
17013 %}
17014 ins_encode %{
17015 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17016 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17017 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17018 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17019 %}
17020 ins_pipe(pipe_slow);
17021 %}
17022
17023 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17024 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17025 match(Set dst (ExpandBits (LoadI mem) mask));
17026 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17027 format %{ "ldrs $tsrc, $mem\n\t"
17028 "ldrs $tmask, $mask\n\t"
17029 "bdep $tdst, $tsrc, $tmask\n\t"
17030 "mov $dst, $tdst"
17031 %}
17032 ins_encode %{
17033 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17034 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17035 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17036 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17037 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17038 %}
17039 ins_pipe(pipe_slow);
17040 %}
17041
17042 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17043 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17044 match(Set dst (ExpandBits src mask));
17045 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17046 format %{ "mov $tsrc, $src\n\t"
17047 "mov $tmask, $mask\n\t"
17048 "bdep $tdst, $tsrc, $tmask\n\t"
17049 "mov $dst, $tdst"
17050 %}
17051 ins_encode %{
17052 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17053 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17054 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17055 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17056 %}
17057 ins_pipe(pipe_slow);
17058 %}
17059
17060
17061 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17062 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17063 match(Set dst (ExpandBits (LoadL mem) mask));
17064 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17065 format %{ "ldrd $tsrc, $mem\n\t"
17066 "ldrd $tmask, $mask\n\t"
17067 "bdep $tdst, $tsrc, $tmask\n\t"
17068 "mov $dst, $tdst"
17069 %}
17070 ins_encode %{
17071 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17072 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17073 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17074 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17075 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17076 %}
17077 ins_pipe(pipe_slow);
17078 %}
17079
17080 // ============================================================================
17081 // This name is KNOWN by the ADLC and cannot be changed.
17082 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17083 // for this guy.
17084 instruct tlsLoadP(thread_RegP dst)
17085 %{
17086 match(Set dst (ThreadLocal));
17087
17088 ins_cost(0);
17089
17090 format %{ " -- \t// $dst=Thread::current(), empty" %}
17091
17092 size(0);
17093
17094 ins_encode( /*empty*/ );
17095
17096 ins_pipe(pipe_class_empty);
17097 %}
17098
17099 //----------PEEPHOLE RULES-----------------------------------------------------
17100 // These must follow all instruction definitions as they use the names
17101 // defined in the instructions definitions.
17102 //
17103 // peepmatch ( root_instr_name [preceding_instruction]* );
17104 //
17105 // peepconstraint %{
17106 // (instruction_number.operand_name relational_op instruction_number.operand_name
17107 // [, ...] );
17108 // // instruction numbers are zero-based using left to right order in peepmatch
17109 //
17110 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17111 // // provide an instruction_number.operand_name for each operand that appears
17112 // // in the replacement instruction's match rule
17113 //
17114 // ---------VM FLAGS---------------------------------------------------------
17115 //
17116 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17117 //
17118 // Each peephole rule is given an identifying number starting with zero and
17119 // increasing by one in the order seen by the parser. An individual peephole
17120 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17121 // on the command-line.
17122 //
17123 // ---------CURRENT LIMITATIONS----------------------------------------------
17124 //
17125 // Only match adjacent instructions in same basic block
17126 // Only equality constraints
17127 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17128 // Only one replacement instruction
17129 //
17130 // ---------EXAMPLE----------------------------------------------------------
17131 //
17132 // // pertinent parts of existing instructions in architecture description
17133 // instruct movI(iRegINoSp dst, iRegI src)
17134 // %{
17135 // match(Set dst (CopyI src));
17136 // %}
17137 //
17138 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17139 // %{
17140 // match(Set dst (AddI dst src));
17141 // effect(KILL cr);
17142 // %}
17143 //
17144 // // Change (inc mov) to lea
17145 // peephole %{
17146 // // increment preceded by register-register move
17147 // peepmatch ( incI_iReg movI );
17148 // // require that the destination register of the increment
17149 // // match the destination register of the move
17150 // peepconstraint ( 0.dst == 1.dst );
17151 // // construct a replacement instruction that sets
17152 // // the destination to ( move's source register + one )
17153 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17154 // %}
17155 //
17156
17157 // Implementation no longer uses movX instructions since
17158 // machine-independent system no longer uses CopyX nodes.
17159 //
17160 // peephole
17161 // %{
17162 // peepmatch (incI_iReg movI);
17163 // peepconstraint (0.dst == 1.dst);
17164 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17165 // %}
17166
17167 // peephole
17168 // %{
17169 // peepmatch (decI_iReg movI);
17170 // peepconstraint (0.dst == 1.dst);
17171 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17172 // %}
17173
17174 // peephole
17175 // %{
17176 // peepmatch (addI_iReg_imm movI);
17177 // peepconstraint (0.dst == 1.dst);
17178 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17179 // %}
17180
17181 // peephole
17182 // %{
17183 // peepmatch (incL_iReg movL);
17184 // peepconstraint (0.dst == 1.dst);
17185 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17186 // %}
17187
17188 // peephole
17189 // %{
17190 // peepmatch (decL_iReg movL);
17191 // peepconstraint (0.dst == 1.dst);
17192 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17193 // %}
17194
17195 // peephole
17196 // %{
17197 // peepmatch (addL_iReg_imm movL);
17198 // peepconstraint (0.dst == 1.dst);
17199 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17200 // %}
17201
17202 // peephole
17203 // %{
17204 // peepmatch (addP_iReg_imm movP);
17205 // peepconstraint (0.dst == 1.dst);
17206 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17207 // %}
17208
17209 // // Change load of spilled value to only a spill
17210 // instruct storeI(memory mem, iRegI src)
17211 // %{
17212 // match(Set mem (StoreI mem src));
17213 // %}
17214 //
17215 // instruct loadI(iRegINoSp dst, memory mem)
17216 // %{
17217 // match(Set dst (LoadI mem));
17218 // %}
17219 //
17220
17221 //----------SMARTSPILL RULES---------------------------------------------------
17222 // These must follow all instruction definitions as they use the names
17223 // defined in the instructions definitions.
17224
17225 // Local Variables:
17226 // mode: c++
17227 // End: