1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 bool Matcher::supports_vector_calling_convention(void) {
2311 return EnableVectorSupport && UseVectorStubs;
2312 }
2313
2314 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2315 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2316 int lo = V0_num;
2317 int hi = V0_H_num;
2318 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2319 hi = V0_K_num;
2320 }
2321 return OptoRegPair(hi, lo);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635
2636 // Loads and stores with indirect memory input (e.g., volatile loads and
2637 // stores) do not subsume the input into complex addressing expressions. If
2638 // the addressing expression is input to at least one such load or store, do
2639 // not clone the addressing expression. Query needs_acquiring_load and
2640 // needs_releasing_store as a proxy for indirect memory input, as it is not
2641 // possible to directly query for indirect memory input at this stage.
2642 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2643 Node* n = m->fast_out(i);
2644 if (n->is_Load() && needs_acquiring_load(n)) {
2645 return false;
2646 }
2647 if (n->is_Store() && needs_releasing_store(n)) {
2648 return false;
2649 }
2650 }
2651
2652 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2653 return true;
2654 }
2655
2656 Node *off = m->in(AddPNode::Offset);
2657 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2658 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2659 // Are there other uses besides address expressions?
2660 !is_visited(off)) {
2661 address_visited.set(off->_idx); // Flag as address_visited
2662 mstack.push(off->in(2), Visit);
2663 Node *conv = off->in(1);
2664 if (conv->Opcode() == Op_ConvI2L &&
2665 // Are there other uses besides address expressions?
2666 !is_visited(conv)) {
2667 address_visited.set(conv->_idx); // Flag as address_visited
2668 mstack.push(conv->in(1), Pre_Visit);
2669 } else {
2670 mstack.push(conv, Pre_Visit);
2671 }
2672 address_visited.test_set(m->_idx); // Flag as address_visited
2673 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2674 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2675 return true;
2676 } else if (off->Opcode() == Op_ConvI2L &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.test_set(m->_idx); // Flag as address_visited
2680 address_visited.set(off->_idx); // Flag as address_visited
2681 mstack.push(off->in(1), Pre_Visit);
2682 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2683 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2684 return true;
2685 }
2686 return false;
2687 }
2688
2689 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2690 { \
2691 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2692 guarantee(DISP == 0, "mode not permitted for volatile"); \
2693 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2694 __ INSN(REG, as_Register(BASE)); \
2695 }
2696
2697
2698 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2699 {
2700 Address::extend scale;
2701
2702 // Hooboy, this is fugly. We need a way to communicate to the
2703 // encoder that the index needs to be sign extended, so we have to
2704 // enumerate all the cases.
2705 switch (opcode) {
2706 case INDINDEXSCALEDI2L:
2707 case INDINDEXSCALEDI2LN:
2708 case INDINDEXI2L:
2709 case INDINDEXI2LN:
2710 scale = Address::sxtw(size);
2711 break;
2712 default:
2713 scale = Address::lsl(size);
2714 }
2715
2716 if (index == -1) {
2717 return Address(base, disp);
2718 } else {
2719 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2720 return Address(base, as_Register(index), scale);
2721 }
2722 }
2723
2724
2725 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2727 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2728 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2729 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2730
2731 // Used for all non-volatile memory accesses. The use of
2732 // $mem->opcode() to discover whether this pattern uses sign-extended
2733 // offsets is something of a kludge.
2734 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2735 Register reg, int opcode,
2736 Register base, int index, int scale, int disp,
2737 int size_in_memory)
2738 {
2739 Address addr = mem2address(opcode, base, index, scale, disp);
2740 if (addr.getMode() == Address::base_plus_offset) {
2741 /* Fix up any out-of-range offsets. */
2742 assert_different_registers(rscratch1, base);
2743 assert_different_registers(rscratch1, reg);
2744 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2745 }
2746 (masm->*insn)(reg, addr);
2747 }
2748
2749 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2750 FloatRegister reg, int opcode,
2751 Register base, int index, int size, int disp,
2752 int size_in_memory)
2753 {
2754 Address::extend scale;
2755
2756 switch (opcode) {
2757 case INDINDEXSCALEDI2L:
2758 case INDINDEXSCALEDI2LN:
2759 scale = Address::sxtw(size);
2760 break;
2761 default:
2762 scale = Address::lsl(size);
2763 }
2764
2765 if (index == -1) {
2766 // Fix up any out-of-range offsets.
2767 assert_different_registers(rscratch1, base);
2768 Address addr = Address(base, disp);
2769 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2770 (masm->*insn)(reg, addr);
2771 } else {
2772 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2773 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2774 }
2775 }
2776
2777 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2778 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2779 int opcode, Register base, int index, int size, int disp)
2780 {
2781 if (index == -1) {
2782 (masm->*insn)(reg, T, Address(base, disp));
2783 } else {
2784 assert(disp == 0, "unsupported address mode");
2785 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2786 }
2787 }
2788
2789 %}
2790
2791
2792
2793 //----------ENCODING BLOCK-----------------------------------------------------
2794 // This block specifies the encoding classes used by the compiler to
2795 // output byte streams. Encoding classes are parameterized macros
2796 // used by Machine Instruction Nodes in order to generate the bit
2797 // encoding of the instruction. Operands specify their base encoding
2798 // interface with the interface keyword. There are currently
2799 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2800 // COND_INTER. REG_INTER causes an operand to generate a function
2801 // which returns its register number when queried. CONST_INTER causes
2802 // an operand to generate a function which returns the value of the
2803 // constant when queried. MEMORY_INTER causes an operand to generate
2804 // four functions which return the Base Register, the Index Register,
2805 // the Scale Value, and the Offset Value of the operand when queried.
2806 // COND_INTER causes an operand to generate six functions which return
2807 // the encoding code (ie - encoding bits for the instruction)
2808 // associated with each basic boolean condition for a conditional
2809 // instruction.
2810 //
2811 // Instructions specify two basic values for encoding. Again, a
2812 // function is available to check if the constant displacement is an
2813 // oop. They use the ins_encode keyword to specify their encoding
2814 // classes (which must be a sequence of enc_class names, and their
2815 // parameters, specified in the encoding block), and they use the
2816 // opcode keyword to specify, in order, their primary, secondary, and
2817 // tertiary opcode. Only the opcode sections which a particular
2818 // instruction needs for encoding need to be specified.
2819 encode %{
2820 // Build emit functions for each basic byte or larger field in the
2821 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2822 // from C++ code in the enc_class source block. Emit functions will
2823 // live in the main source block for now. In future, we can
2824 // generalize this by adding a syntax that specifies the sizes of
2825 // fields in an order, so that the adlc can build the emit functions
2826 // automagically
2827
2828 // catch all for unimplemented encodings
2829 enc_class enc_unimplemented %{
2830 __ unimplemented("C2 catch all");
2831 %}
2832
2833 // BEGIN Non-volatile memory access
2834
2835 // This encoding class is generated automatically from ad_encode.m4.
2836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2837 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2838 Register dst_reg = as_Register($dst$$reg);
2839 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2841 %}
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2934 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2942 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2950 Register src_reg = as_Register($src$$reg);
2951 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_strb0(memory1 mem) %{
2958 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strh0(memory2 mem) %{
2973 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2980 Register src_reg = as_Register($src$$reg);
2981 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strw0(memory4 mem) %{
2988 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2989 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2990 %}
2991
2992 // This encoding class is generated automatically from ad_encode.m4.
2993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2994 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2995 Register src_reg = as_Register($src$$reg);
2996 // we sometimes get asked to store the stack pointer into the
2997 // current thread -- we cannot do that directly on AArch64
2998 if (src_reg == r31_sp) {
2999 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3000 __ mov(rscratch2, sp);
3001 src_reg = rscratch2;
3002 }
3003 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_str0(memory8 mem) %{
3010 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3017 FloatRegister src_reg = as_FloatRegister($src$$reg);
3018 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3025 FloatRegister src_reg = as_FloatRegister($src$$reg);
3026 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3028 %}
3029
3030 // This encoding class is generated automatically from ad_encode.m4.
3031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3032 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3033 __ membar(Assembler::StoreStore);
3034 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3036 %}
3037
3038 // END Non-volatile memory access
3039
3040 // Vector loads and stores
3041 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3042 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3043 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3044 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3045 %}
3046
3047 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3048 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3049 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3050 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3051 %}
3052
3053 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3066 FloatRegister src_reg = as_FloatRegister($src$$reg);
3067 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3072 FloatRegister src_reg = as_FloatRegister($src$$reg);
3073 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 // volatile loads and stores
3090
3091 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3092 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3093 rscratch1, stlrb);
3094 %}
3095
3096 enc_class aarch64_enc_stlrb0(memory mem) %{
3097 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3102 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrh);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh0(memory mem) %{
3107 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3112 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrw);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw0(memory mem) %{
3117 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3122 Register dst_reg = as_Register($dst$$reg);
3123 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3124 rscratch1, ldarb);
3125 __ sxtbw(dst_reg, dst_reg);
3126 %}
3127
3128 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarb);
3132 __ sxtb(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3136 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, ldarb);
3138 %}
3139
3140 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3146 Register dst_reg = as_Register($dst$$reg);
3147 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, ldarh);
3149 __ sxthw(dst_reg, dst_reg);
3150 %}
3151
3152 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3153 Register dst_reg = as_Register($dst$$reg);
3154 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarh);
3156 __ sxth(dst_reg, dst_reg);
3157 %}
3158
3159 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3160 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarh);
3162 %}
3163
3164 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarw);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldar);
3182 %}
3183
3184 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3185 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldarw);
3187 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3188 %}
3189
3190 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3191 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldar);
3193 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3194 %}
3195
3196 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3197 Register src_reg = as_Register($src$$reg);
3198 // we sometimes get asked to store the stack pointer into the
3199 // current thread -- we cannot do that directly on AArch64
3200 if (src_reg == r31_sp) {
3201 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3202 __ mov(rscratch2, sp);
3203 src_reg = rscratch2;
3204 }
3205 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, stlr);
3207 %}
3208
3209 enc_class aarch64_enc_stlr0(memory mem) %{
3210 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, stlr);
3212 %}
3213
3214 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3215 {
3216 FloatRegister src_reg = as_FloatRegister($src$$reg);
3217 __ fmovs(rscratch2, src_reg);
3218 }
3219 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, stlrw);
3221 %}
3222
3223 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3224 {
3225 FloatRegister src_reg = as_FloatRegister($src$$reg);
3226 __ fmovd(rscratch2, src_reg);
3227 }
3228 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 // synchronized read/update encodings
3233
3234 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3235 Register dst_reg = as_Register($dst$$reg);
3236 Register base = as_Register($mem$$base);
3237 int index = $mem$$index;
3238 int scale = $mem$$scale;
3239 int disp = $mem$$disp;
3240 if (index == -1) {
3241 if (disp != 0) {
3242 __ lea(rscratch1, Address(base, disp));
3243 __ ldaxr(dst_reg, rscratch1);
3244 } else {
3245 // TODO
3246 // should we ever get anything other than this case?
3247 __ ldaxr(dst_reg, base);
3248 }
3249 } else {
3250 Register index_reg = as_Register(index);
3251 if (disp == 0) {
3252 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3253 __ ldaxr(dst_reg, rscratch1);
3254 } else {
3255 __ lea(rscratch1, Address(base, disp));
3256 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3257 __ ldaxr(dst_reg, rscratch1);
3258 }
3259 }
3260 %}
3261
3262 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3263 Register src_reg = as_Register($src$$reg);
3264 Register base = as_Register($mem$$base);
3265 int index = $mem$$index;
3266 int scale = $mem$$scale;
3267 int disp = $mem$$disp;
3268 if (index == -1) {
3269 if (disp != 0) {
3270 __ lea(rscratch2, Address(base, disp));
3271 __ stlxr(rscratch1, src_reg, rscratch2);
3272 } else {
3273 // TODO
3274 // should we ever get anything other than this case?
3275 __ stlxr(rscratch1, src_reg, base);
3276 }
3277 } else {
3278 Register index_reg = as_Register(index);
3279 if (disp == 0) {
3280 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3281 __ stlxr(rscratch1, src_reg, rscratch2);
3282 } else {
3283 __ lea(rscratch2, Address(base, disp));
3284 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3285 __ stlxr(rscratch1, src_reg, rscratch2);
3286 }
3287 }
3288 __ cmpw(rscratch1, zr);
3289 %}
3290
3291 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3292 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3293 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3294 Assembler::xword, /*acquire*/ false, /*release*/ true,
3295 /*weak*/ false, noreg);
3296 %}
3297
3298 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3299 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3300 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3301 Assembler::word, /*acquire*/ false, /*release*/ true,
3302 /*weak*/ false, noreg);
3303 %}
3304
3305 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3306 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3307 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3308 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3309 /*weak*/ false, noreg);
3310 %}
3311
3312 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3313 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3314 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3315 Assembler::byte, /*acquire*/ false, /*release*/ true,
3316 /*weak*/ false, noreg);
3317 %}
3318
3319
3320 // The only difference between aarch64_enc_cmpxchg and
3321 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3322 // CompareAndSwap sequence to serve as a barrier on acquiring a
3323 // lock.
3324 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::xword, /*acquire*/ true, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3332 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3333 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3334 Assembler::word, /*acquire*/ true, /*release*/ true,
3335 /*weak*/ false, noreg);
3336 %}
3337
3338 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3339 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3340 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3341 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3342 /*weak*/ false, noreg);
3343 %}
3344
3345 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3346 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3347 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3348 Assembler::byte, /*acquire*/ true, /*release*/ true,
3349 /*weak*/ false, noreg);
3350 %}
3351
3352 // auxiliary used for CompareAndSwapX to set result register
3353 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3354 Register res_reg = as_Register($res$$reg);
3355 __ cset(res_reg, Assembler::EQ);
3356 %}
3357
3358 // prefetch encodings
3359
3360 enc_class aarch64_enc_prefetchw(memory mem) %{
3361 Register base = as_Register($mem$$base);
3362 int index = $mem$$index;
3363 int scale = $mem$$scale;
3364 int disp = $mem$$disp;
3365 if (index == -1) {
3366 // Fix up any out-of-range offsets.
3367 assert_different_registers(rscratch1, base);
3368 Address addr = Address(base, disp);
3369 addr = __ legitimize_address(addr, 8, rscratch1);
3370 __ prfm(addr, PSTL1KEEP);
3371 } else {
3372 Register index_reg = as_Register(index);
3373 if (disp == 0) {
3374 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3375 } else {
3376 __ lea(rscratch1, Address(base, disp));
3377 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3378 }
3379 }
3380 %}
3381
3382 // mov encodings
3383
3384 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3385 uint32_t con = (uint32_t)$src$$constant;
3386 Register dst_reg = as_Register($dst$$reg);
3387 if (con == 0) {
3388 __ movw(dst_reg, zr);
3389 } else {
3390 __ movw(dst_reg, con);
3391 }
3392 %}
3393
3394 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3395 Register dst_reg = as_Register($dst$$reg);
3396 uint64_t con = (uint64_t)$src$$constant;
3397 if (con == 0) {
3398 __ mov(dst_reg, zr);
3399 } else {
3400 __ mov(dst_reg, con);
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 address con = (address)$src$$constant;
3407 if (con == nullptr || con == (address)1) {
3408 ShouldNotReachHere();
3409 } else {
3410 relocInfo::relocType rtype = $src->constant_reloc();
3411 if (rtype == relocInfo::oop_type) {
3412 __ movoop(dst_reg, (jobject)con);
3413 } else if (rtype == relocInfo::metadata_type) {
3414 __ mov_metadata(dst_reg, (Metadata*)con);
3415 } else {
3416 assert(rtype == relocInfo::none, "unexpected reloc type");
3417 if (! __ is_valid_AArch64_address(con) ||
3418 con < (address)(uintptr_t)os::vm_page_size()) {
3419 __ mov(dst_reg, con);
3420 } else {
3421 uint64_t offset;
3422 __ adrp(dst_reg, con, offset);
3423 __ add(dst_reg, dst_reg, offset);
3424 }
3425 }
3426 }
3427 %}
3428
3429 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3430 Register dst_reg = as_Register($dst$$reg);
3431 __ mov(dst_reg, zr);
3432 %}
3433
3434 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, (uint64_t)1);
3437 %}
3438
3439 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3440 __ load_byte_map_base($dst$$Register);
3441 %}
3442
3443 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 address con = (address)$src$$constant;
3446 if (con == nullptr) {
3447 ShouldNotReachHere();
3448 } else {
3449 relocInfo::relocType rtype = $src->constant_reloc();
3450 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3451 __ set_narrow_oop(dst_reg, (jobject)con);
3452 }
3453 %}
3454
3455 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 __ mov(dst_reg, zr);
3458 %}
3459
3460 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3461 Register dst_reg = as_Register($dst$$reg);
3462 address con = (address)$src$$constant;
3463 if (con == nullptr) {
3464 ShouldNotReachHere();
3465 } else {
3466 relocInfo::relocType rtype = $src->constant_reloc();
3467 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3468 __ set_narrow_klass(dst_reg, (Klass *)con);
3469 }
3470 %}
3471
3472 // arithmetic encodings
3473
3474 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3475 Register dst_reg = as_Register($dst$$reg);
3476 Register src_reg = as_Register($src1$$reg);
3477 int32_t con = (int32_t)$src2$$constant;
3478 // add has primary == 0, subtract has primary == 1
3479 if ($primary) { con = -con; }
3480 if (con < 0) {
3481 __ subw(dst_reg, src_reg, -con);
3482 } else {
3483 __ addw(dst_reg, src_reg, con);
3484 }
3485 %}
3486
3487 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3488 Register dst_reg = as_Register($dst$$reg);
3489 Register src_reg = as_Register($src1$$reg);
3490 int32_t con = (int32_t)$src2$$constant;
3491 // add has primary == 0, subtract has primary == 1
3492 if ($primary) { con = -con; }
3493 if (con < 0) {
3494 __ sub(dst_reg, src_reg, -con);
3495 } else {
3496 __ add(dst_reg, src_reg, con);
3497 }
3498 %}
3499
3500 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3505 %}
3506
3507 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3508 Register dst_reg = as_Register($dst$$reg);
3509 Register src1_reg = as_Register($src1$$reg);
3510 Register src2_reg = as_Register($src2$$reg);
3511 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3512 %}
3513
3514 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src1_reg = as_Register($src1$$reg);
3517 Register src2_reg = as_Register($src2$$reg);
3518 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3519 %}
3520
3521 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3522 Register dst_reg = as_Register($dst$$reg);
3523 Register src1_reg = as_Register($src1$$reg);
3524 Register src2_reg = as_Register($src2$$reg);
3525 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3526 %}
3527
3528 // compare instruction encodings
3529
3530 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 Register reg2 = as_Register($src2$$reg);
3533 __ cmpw(reg1, reg2);
3534 %}
3535
3536 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3537 Register reg = as_Register($src1$$reg);
3538 int32_t val = $src2$$constant;
3539 if (val >= 0) {
3540 __ subsw(zr, reg, val);
3541 } else {
3542 __ addsw(zr, reg, -val);
3543 }
3544 %}
3545
3546 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3547 Register reg1 = as_Register($src1$$reg);
3548 uint32_t val = (uint32_t)$src2$$constant;
3549 __ movw(rscratch1, val);
3550 __ cmpw(reg1, rscratch1);
3551 %}
3552
3553 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3554 Register reg1 = as_Register($src1$$reg);
3555 Register reg2 = as_Register($src2$$reg);
3556 __ cmp(reg1, reg2);
3557 %}
3558
3559 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3560 Register reg = as_Register($src1$$reg);
3561 int64_t val = $src2$$constant;
3562 if (val >= 0) {
3563 __ subs(zr, reg, val);
3564 } else if (val != -val) {
3565 __ adds(zr, reg, -val);
3566 } else {
3567 // aargh, Long.MIN_VALUE is a special case
3568 __ orr(rscratch1, zr, (uint64_t)val);
3569 __ subs(zr, reg, rscratch1);
3570 }
3571 %}
3572
3573 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3574 Register reg1 = as_Register($src1$$reg);
3575 uint64_t val = (uint64_t)$src2$$constant;
3576 __ mov(rscratch1, val);
3577 __ cmp(reg1, rscratch1);
3578 %}
3579
3580 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3581 Register reg1 = as_Register($src1$$reg);
3582 Register reg2 = as_Register($src2$$reg);
3583 __ cmp(reg1, reg2);
3584 %}
3585
3586 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3587 Register reg1 = as_Register($src1$$reg);
3588 Register reg2 = as_Register($src2$$reg);
3589 __ cmpw(reg1, reg2);
3590 %}
3591
3592 enc_class aarch64_enc_testp(iRegP src) %{
3593 Register reg = as_Register($src$$reg);
3594 __ cmp(reg, zr);
3595 %}
3596
3597 enc_class aarch64_enc_testn(iRegN src) %{
3598 Register reg = as_Register($src$$reg);
3599 __ cmpw(reg, zr);
3600 %}
3601
3602 enc_class aarch64_enc_b(label lbl) %{
3603 Label *L = $lbl$$label;
3604 __ b(*L);
3605 %}
3606
3607 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3608 Label *L = $lbl$$label;
3609 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3610 %}
3611
3612 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3613 Label *L = $lbl$$label;
3614 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3615 %}
3616
3617 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3618 %{
3619 Register sub_reg = as_Register($sub$$reg);
3620 Register super_reg = as_Register($super$$reg);
3621 Register temp_reg = as_Register($temp$$reg);
3622 Register result_reg = as_Register($result$$reg);
3623
3624 Label miss;
3625 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3626 nullptr, &miss,
3627 /*set_cond_codes:*/ true);
3628 if ($primary) {
3629 __ mov(result_reg, zr);
3630 }
3631 __ bind(miss);
3632 %}
3633
3634 enc_class aarch64_enc_java_static_call(method meth) %{
3635 address addr = (address)$meth$$method;
3636 address call;
3637 if (!_method) {
3638 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3639 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3640 if (call == nullptr) {
3641 ciEnv::current()->record_failure("CodeCache is full");
3642 return;
3643 }
3644 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3645 // The NOP here is purely to ensure that eliding a call to
3646 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3647 __ nop();
3648 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3649 } else {
3650 int method_index = resolved_method_index(masm);
3651 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3652 : static_call_Relocation::spec(method_index);
3653 call = __ trampoline_call(Address(addr, rspec));
3654 if (call == nullptr) {
3655 ciEnv::current()->record_failure("CodeCache is full");
3656 return;
3657 }
3658 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3659 // Calls of the same statically bound method can share
3660 // a stub to the interpreter.
3661 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3662 } else {
3663 // Emit stub for static call
3664 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3665 if (stub == nullptr) {
3666 ciEnv::current()->record_failure("CodeCache is full");
3667 return;
3668 }
3669 }
3670 }
3671
3672 __ post_call_nop();
3673
3674 // Only non uncommon_trap calls need to reinitialize ptrue.
3675 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3676 __ reinitialize_ptrue();
3677 }
3678 %}
3679
3680 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3681 int method_index = resolved_method_index(masm);
3682 address call = __ ic_call((address)$meth$$method, method_index);
3683 if (call == nullptr) {
3684 ciEnv::current()->record_failure("CodeCache is full");
3685 return;
3686 }
3687 __ post_call_nop();
3688 if (Compile::current()->max_vector_size() > 0) {
3689 __ reinitialize_ptrue();
3690 }
3691 %}
3692
3693 enc_class aarch64_enc_call_epilog() %{
3694 if (VerifyStackAtCalls) {
3695 // Check that stack depth is unchanged: find majik cookie on stack
3696 __ call_Unimplemented();
3697 }
3698 %}
3699
3700 enc_class aarch64_enc_java_to_runtime(method meth) %{
3701 // some calls to generated routines (arraycopy code) are scheduled
3702 // by C2 as runtime calls. if so we can call them using a br (they
3703 // will be in a reachable segment) otherwise we have to use a blr
3704 // which loads the absolute address into a register.
3705 address entry = (address)$meth$$method;
3706 CodeBlob *cb = CodeCache::find_blob(entry);
3707 if (cb) {
3708 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3709 if (call == nullptr) {
3710 ciEnv::current()->record_failure("CodeCache is full");
3711 return;
3712 }
3713 __ post_call_nop();
3714 } else {
3715 Label retaddr;
3716 // Make the anchor frame walkable
3717 __ adr(rscratch2, retaddr);
3718 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3719 __ lea(rscratch1, RuntimeAddress(entry));
3720 __ blr(rscratch1);
3721 __ bind(retaddr);
3722 __ post_call_nop();
3723 }
3724 if (Compile::current()->max_vector_size() > 0) {
3725 __ reinitialize_ptrue();
3726 }
3727 %}
3728
3729 enc_class aarch64_enc_rethrow() %{
3730 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3731 %}
3732
3733 enc_class aarch64_enc_ret() %{
3734 #ifdef ASSERT
3735 if (Compile::current()->max_vector_size() > 0) {
3736 __ verify_ptrue();
3737 }
3738 #endif
3739 __ ret(lr);
3740 %}
3741
3742 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3743 Register target_reg = as_Register($jump_target$$reg);
3744 __ br(target_reg);
3745 %}
3746
3747 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3748 Register target_reg = as_Register($jump_target$$reg);
3749 // exception oop should be in r0
3750 // ret addr has been popped into lr
3751 // callee expects it in r3
3752 __ mov(r3, lr);
3753 __ br(target_reg);
3754 %}
3755
3756 %}
3757
3758 //----------FRAME--------------------------------------------------------------
3759 // Definition of frame structure and management information.
3760 //
3761 // S T A C K L A Y O U T Allocators stack-slot number
3762 // | (to get allocators register number
3763 // G Owned by | | v add OptoReg::stack0())
3764 // r CALLER | |
3765 // o | +--------+ pad to even-align allocators stack-slot
3766 // w V | pad0 | numbers; owned by CALLER
3767 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3768 // h ^ | in | 5
3769 // | | args | 4 Holes in incoming args owned by SELF
3770 // | | | | 3
3771 // | | +--------+
3772 // V | | old out| Empty on Intel, window on Sparc
3773 // | old |preserve| Must be even aligned.
3774 // | SP-+--------+----> Matcher::_old_SP, even aligned
3775 // | | in | 3 area for Intel ret address
3776 // Owned by |preserve| Empty on Sparc.
3777 // SELF +--------+
3778 // | | pad2 | 2 pad to align old SP
3779 // | +--------+ 1
3780 // | | locks | 0
3781 // | +--------+----> OptoReg::stack0(), even aligned
3782 // | | pad1 | 11 pad to align new SP
3783 // | +--------+
3784 // | | | 10
3785 // | | spills | 9 spills
3786 // V | | 8 (pad0 slot for callee)
3787 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3788 // ^ | out | 7
3789 // | | args | 6 Holes in outgoing args owned by CALLEE
3790 // Owned by +--------+
3791 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3792 // | new |preserve| Must be even-aligned.
3793 // | SP-+--------+----> Matcher::_new_SP, even aligned
3794 // | | |
3795 //
3796 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3797 // known from SELF's arguments and the Java calling convention.
3798 // Region 6-7 is determined per call site.
3799 // Note 2: If the calling convention leaves holes in the incoming argument
3800 // area, those holes are owned by SELF. Holes in the outgoing area
3801 // are owned by the CALLEE. Holes should not be necessary in the
3802 // incoming area, as the Java calling convention is completely under
3803 // the control of the AD file. Doubles can be sorted and packed to
3804 // avoid holes. Holes in the outgoing arguments may be necessary for
3805 // varargs C calling conventions.
3806 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3807 // even aligned with pad0 as needed.
3808 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3809 // (the latter is true on Intel but is it false on AArch64?)
3810 // region 6-11 is even aligned; it may be padded out more so that
3811 // the region from SP to FP meets the minimum stack alignment.
3812 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3813 // alignment. Region 11, pad1, may be dynamically extended so that
3814 // SP meets the minimum alignment.
3815
3816 frame %{
3817 // These three registers define part of the calling convention
3818 // between compiled code and the interpreter.
3819
3820 // Inline Cache Register or Method for I2C.
3821 inline_cache_reg(R12);
3822
3823 // Number of stack slots consumed by locking an object
3824 sync_stack_slots(2);
3825
3826 // Compiled code's Frame Pointer
3827 frame_pointer(R31);
3828
3829 // Interpreter stores its frame pointer in a register which is
3830 // stored to the stack by I2CAdaptors.
3831 // I2CAdaptors convert from interpreted java to compiled java.
3832 interpreter_frame_pointer(R29);
3833
3834 // Stack alignment requirement
3835 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3836
3837 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3838 // for calls to C. Supports the var-args backing area for register parms.
3839 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3840
3841 // The after-PROLOG location of the return address. Location of
3842 // return address specifies a type (REG or STACK) and a number
3843 // representing the register number (i.e. - use a register name) or
3844 // stack slot.
3845 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3846 // Otherwise, it is above the locks and verification slot and alignment word
3847 // TODO this may well be correct but need to check why that - 2 is there
3848 // ppc port uses 0 but we definitely need to allow for fixed_slots
3849 // which folds in the space used for monitors
3850 return_addr(STACK - 2 +
3851 align_up((Compile::current()->in_preserve_stack_slots() +
3852 Compile::current()->fixed_slots()),
3853 stack_alignment_in_slots()));
3854
3855 // Location of compiled Java return values. Same as C for now.
3856 return_value
3857 %{
3858 // TODO do we allow ideal_reg == Op_RegN???
3859 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3860 "only return normal values");
3861
3862 static const int lo[Op_RegL + 1] = { // enum name
3863 0, // Op_Node
3864 0, // Op_Set
3865 R0_num, // Op_RegN
3866 R0_num, // Op_RegI
3867 R0_num, // Op_RegP
3868 V0_num, // Op_RegF
3869 V0_num, // Op_RegD
3870 R0_num // Op_RegL
3871 };
3872
3873 static const int hi[Op_RegL + 1] = { // enum name
3874 0, // Op_Node
3875 0, // Op_Set
3876 OptoReg::Bad, // Op_RegN
3877 OptoReg::Bad, // Op_RegI
3878 R0_H_num, // Op_RegP
3879 OptoReg::Bad, // Op_RegF
3880 V0_H_num, // Op_RegD
3881 R0_H_num // Op_RegL
3882 };
3883
3884 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3885 %}
3886 %}
3887
3888 //----------ATTRIBUTES---------------------------------------------------------
3889 //----------Operand Attributes-------------------------------------------------
3890 op_attrib op_cost(1); // Required cost attribute
3891
3892 //----------Instruction Attributes---------------------------------------------
3893 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3894 ins_attrib ins_size(32); // Required size attribute (in bits)
3895 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3896 // a non-matching short branch variant
3897 // of some long branch?
3898 ins_attrib ins_alignment(4); // Required alignment attribute (must
3899 // be a power of 2) specifies the
3900 // alignment that some part of the
3901 // instruction (not necessarily the
3902 // start) requires. If > 1, a
3903 // compute_padding() function must be
3904 // provided for the instruction
3905
3906 //----------OPERANDS-----------------------------------------------------------
3907 // Operand definitions must precede instruction definitions for correct parsing
3908 // in the ADLC because operands constitute user defined types which are used in
3909 // instruction definitions.
3910
3911 //----------Simple Operands----------------------------------------------------
3912
3913 // Integer operands 32 bit
3914 // 32 bit immediate
3915 operand immI()
3916 %{
3917 match(ConI);
3918
3919 op_cost(0);
3920 format %{ %}
3921 interface(CONST_INTER);
3922 %}
3923
3924 // 32 bit zero
3925 operand immI0()
3926 %{
3927 predicate(n->get_int() == 0);
3928 match(ConI);
3929
3930 op_cost(0);
3931 format %{ %}
3932 interface(CONST_INTER);
3933 %}
3934
3935 // 32 bit unit increment
3936 operand immI_1()
3937 %{
3938 predicate(n->get_int() == 1);
3939 match(ConI);
3940
3941 op_cost(0);
3942 format %{ %}
3943 interface(CONST_INTER);
3944 %}
3945
3946 // 32 bit unit decrement
3947 operand immI_M1()
3948 %{
3949 predicate(n->get_int() == -1);
3950 match(ConI);
3951
3952 op_cost(0);
3953 format %{ %}
3954 interface(CONST_INTER);
3955 %}
3956
3957 // Shift values for add/sub extension shift
3958 operand immIExt()
3959 %{
3960 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_gt_1()
3969 %{
3970 predicate(n->get_int() > 1);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_le_4()
3979 %{
3980 predicate(n->get_int() <= 4);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_16()
3989 %{
3990 predicate(n->get_int() == 16);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_24()
3999 %{
4000 predicate(n->get_int() == 24);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_32()
4009 %{
4010 predicate(n->get_int() == 32);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_48()
4019 %{
4020 predicate(n->get_int() == 48);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_56()
4029 %{
4030 predicate(n->get_int() == 56);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_255()
4039 %{
4040 predicate(n->get_int() == 255);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_65535()
4049 %{
4050 predicate(n->get_int() == 65535);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_positive()
4059 %{
4060 predicate(n->get_int() > 0);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 // BoolTest condition for signed compare
4069 operand immI_cmp_cond()
4070 %{
4071 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4072 match(ConI);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 // BoolTest condition for unsigned compare
4080 operand immI_cmpU_cond()
4081 %{
4082 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4083 match(ConI);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_255()
4091 %{
4092 predicate(n->get_long() == 255L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_65535()
4101 %{
4102 predicate(n->get_long() == 65535L);
4103 match(ConL);
4104
4105 op_cost(0);
4106 format %{ %}
4107 interface(CONST_INTER);
4108 %}
4109
4110 operand immL_4294967295()
4111 %{
4112 predicate(n->get_long() == 4294967295L);
4113 match(ConL);
4114
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 operand immL_bitmask()
4121 %{
4122 predicate((n->get_long() != 0)
4123 && ((n->get_long() & 0xc000000000000000l) == 0)
4124 && is_power_of_2(n->get_long() + 1));
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 operand immI_bitmask()
4133 %{
4134 predicate((n->get_int() != 0)
4135 && ((n->get_int() & 0xc0000000) == 0)
4136 && is_power_of_2(n->get_int() + 1));
4137 match(ConI);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 operand immL_positive_bitmaskI()
4145 %{
4146 predicate((n->get_long() != 0)
4147 && ((julong)n->get_long() < 0x80000000ULL)
4148 && is_power_of_2(n->get_long() + 1));
4149 match(ConL);
4150
4151 op_cost(0);
4152 format %{ %}
4153 interface(CONST_INTER);
4154 %}
4155
4156 // Scale values for scaled offset addressing modes (up to long but not quad)
4157 operand immIScale()
4158 %{
4159 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4160 match(ConI);
4161
4162 op_cost(0);
4163 format %{ %}
4164 interface(CONST_INTER);
4165 %}
4166
4167 // 5 bit signed integer
4168 operand immI5()
4169 %{
4170 predicate(Assembler::is_simm(n->get_int(), 5));
4171 match(ConI);
4172
4173 op_cost(0);
4174 format %{ %}
4175 interface(CONST_INTER);
4176 %}
4177
4178 // 7 bit unsigned integer
4179 operand immIU7()
4180 %{
4181 predicate(Assembler::is_uimm(n->get_int(), 7));
4182 match(ConI);
4183
4184 op_cost(0);
4185 format %{ %}
4186 interface(CONST_INTER);
4187 %}
4188
4189 // Offset for scaled or unscaled immediate loads and stores
4190 operand immIOffset()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset1()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset2()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset4()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immIOffset8()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immIOffset16()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLOffset()
4251 %{
4252 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset1()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset2()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset4()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immLoffset8()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4293 match(ConL);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immLoffset16()
4301 %{
4302 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4303 match(ConL);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 // 5 bit signed long integer
4311 operand immL5()
4312 %{
4313 predicate(Assembler::is_simm(n->get_long(), 5));
4314 match(ConL);
4315
4316 op_cost(0);
4317 format %{ %}
4318 interface(CONST_INTER);
4319 %}
4320
4321 // 7 bit unsigned long integer
4322 operand immLU7()
4323 %{
4324 predicate(Assembler::is_uimm(n->get_long(), 7));
4325 match(ConL);
4326
4327 op_cost(0);
4328 format %{ %}
4329 interface(CONST_INTER);
4330 %}
4331
4332 // 8 bit signed value.
4333 operand immI8()
4334 %{
4335 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4336 match(ConI);
4337
4338 op_cost(0);
4339 format %{ %}
4340 interface(CONST_INTER);
4341 %}
4342
4343 // 8 bit signed value (simm8), or #simm8 LSL 8.
4344 operand immI8_shift8()
4345 %{
4346 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4347 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4348 match(ConI);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value (simm8), or #simm8 LSL 8.
4356 operand immL8_shift8()
4357 %{
4358 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4359 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4360 match(ConL);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit integer valid for vector add sub immediate
4368 operand immBAddSubV()
4369 %{
4370 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit integer valid for add sub immediate
4379 operand immIAddSub()
4380 %{
4381 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4382 match(ConI);
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 32 bit integer valid for vector add sub immediate
4389 operand immIAddSubV()
4390 %{
4391 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4392 match(ConI);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit unsigned integer valid for logical immediate
4400
4401 operand immBLog()
4402 %{
4403 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immSLog()
4412 %{
4413 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immILog()
4422 %{
4423 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // Integer operands 64 bit
4432 // 64 bit immediate
4433 operand immL()
4434 %{
4435 match(ConL);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 64 bit zero
4443 operand immL0()
4444 %{
4445 predicate(n->get_long() == 0);
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit unit decrement
4454 operand immL_M1()
4455 %{
4456 predicate(n->get_long() == -1);
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for add sub immediate
4465 operand immLAddSub()
4466 %{
4467 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit integer valid for addv subv immediate
4475 operand immLAddSubV()
4476 %{
4477 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for logical immediate
4486 operand immLLog()
4487 %{
4488 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4489 match(ConL);
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // Long Immediate: low 32-bit mask
4496 operand immL_32bits()
4497 %{
4498 predicate(n->get_long() == 0xFFFFFFFFL);
4499 match(ConL);
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // Pointer operands
4506 // Pointer Immediate
4507 operand immP()
4508 %{
4509 match(ConP);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // nullptr Pointer Immediate
4517 operand immP0()
4518 %{
4519 predicate(n->get_ptr() == 0);
4520 match(ConP);
4521
4522 op_cost(0);
4523 format %{ %}
4524 interface(CONST_INTER);
4525 %}
4526
4527 // Pointer Immediate One
4528 // this is used in object initialization (initial object header)
4529 operand immP_1()
4530 %{
4531 predicate(n->get_ptr() == 1);
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // Card Table Byte Map Base
4540 operand immByteMapBase()
4541 %{
4542 // Get base of card map
4543 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4544 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4545 match(ConP);
4546
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 // Float and Double operands
4553 // Double Immediate
4554 operand immD()
4555 %{
4556 match(ConD);
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Double Immediate: +0.0d
4563 operand immD0()
4564 %{
4565 predicate(jlong_cast(n->getd()) == 0);
4566 match(ConD);
4567
4568 op_cost(0);
4569 format %{ %}
4570 interface(CONST_INTER);
4571 %}
4572
4573 // constant 'double +0.0'.
4574 operand immDPacked()
4575 %{
4576 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4577 match(ConD);
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // Float Immediate
4584 operand immF()
4585 %{
4586 match(ConF);
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 // Float Immediate: +0.0f.
4593 operand immF0()
4594 %{
4595 predicate(jint_cast(n->getf()) == 0);
4596 match(ConF);
4597
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 //
4604 operand immFPacked()
4605 %{
4606 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4607 match(ConF);
4608 op_cost(0);
4609 format %{ %}
4610 interface(CONST_INTER);
4611 %}
4612
4613 // Narrow pointer operands
4614 // Narrow Pointer Immediate
4615 operand immN()
4616 %{
4617 match(ConN);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // Narrow nullptr Pointer Immediate
4625 operand immN0()
4626 %{
4627 predicate(n->get_narrowcon() == 0);
4628 match(ConN);
4629
4630 op_cost(0);
4631 format %{ %}
4632 interface(CONST_INTER);
4633 %}
4634
4635 operand immNKlass()
4636 %{
4637 match(ConNKlass);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // Integer 32 bit Register Operands
4645 // Integer 32 bitRegister (excludes SP)
4646 operand iRegI()
4647 %{
4648 constraint(ALLOC_IN_RC(any_reg32));
4649 match(RegI);
4650 match(iRegINoSp);
4651 op_cost(0);
4652 format %{ %}
4653 interface(REG_INTER);
4654 %}
4655
4656 // Integer 32 bit Register not Special
4657 operand iRegINoSp()
4658 %{
4659 constraint(ALLOC_IN_RC(no_special_reg32));
4660 match(RegI);
4661 op_cost(0);
4662 format %{ %}
4663 interface(REG_INTER);
4664 %}
4665
4666 // Integer 64 bit Register Operands
4667 // Integer 64 bit Register (includes SP)
4668 operand iRegL()
4669 %{
4670 constraint(ALLOC_IN_RC(any_reg));
4671 match(RegL);
4672 match(iRegLNoSp);
4673 op_cost(0);
4674 format %{ %}
4675 interface(REG_INTER);
4676 %}
4677
4678 // Integer 64 bit Register not Special
4679 operand iRegLNoSp()
4680 %{
4681 constraint(ALLOC_IN_RC(no_special_reg));
4682 match(RegL);
4683 match(iRegL_R0);
4684 format %{ %}
4685 interface(REG_INTER);
4686 %}
4687
4688 // Pointer Register Operands
4689 // Pointer Register
4690 operand iRegP()
4691 %{
4692 constraint(ALLOC_IN_RC(ptr_reg));
4693 match(RegP);
4694 match(iRegPNoSp);
4695 match(iRegP_R0);
4696 //match(iRegP_R2);
4697 //match(iRegP_R4);
4698 match(iRegP_R5);
4699 match(thread_RegP);
4700 op_cost(0);
4701 format %{ %}
4702 interface(REG_INTER);
4703 %}
4704
4705 // Pointer 64 bit Register not Special
4706 operand iRegPNoSp()
4707 %{
4708 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4709 match(RegP);
4710 // match(iRegP);
4711 // match(iRegP_R0);
4712 // match(iRegP_R2);
4713 // match(iRegP_R4);
4714 // match(iRegP_R5);
4715 // match(thread_RegP);
4716 op_cost(0);
4717 format %{ %}
4718 interface(REG_INTER);
4719 %}
4720
4721 // This operand is not allowed to use rfp even if
4722 // rfp is not used to hold the frame pointer.
4723 operand iRegPNoSpNoRfp()
4724 %{
4725 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4726 match(RegP);
4727 match(iRegPNoSp);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Pointer 64 bit Register R0 only
4734 operand iRegP_R0()
4735 %{
4736 constraint(ALLOC_IN_RC(r0_reg));
4737 match(RegP);
4738 // match(iRegP);
4739 match(iRegPNoSp);
4740 op_cost(0);
4741 format %{ %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // Pointer 64 bit Register R1 only
4746 operand iRegP_R1()
4747 %{
4748 constraint(ALLOC_IN_RC(r1_reg));
4749 match(RegP);
4750 // match(iRegP);
4751 match(iRegPNoSp);
4752 op_cost(0);
4753 format %{ %}
4754 interface(REG_INTER);
4755 %}
4756
4757 // Pointer 64 bit Register R2 only
4758 operand iRegP_R2()
4759 %{
4760 constraint(ALLOC_IN_RC(r2_reg));
4761 match(RegP);
4762 // match(iRegP);
4763 match(iRegPNoSp);
4764 op_cost(0);
4765 format %{ %}
4766 interface(REG_INTER);
4767 %}
4768
4769 // Pointer 64 bit Register R3 only
4770 operand iRegP_R3()
4771 %{
4772 constraint(ALLOC_IN_RC(r3_reg));
4773 match(RegP);
4774 // match(iRegP);
4775 match(iRegPNoSp);
4776 op_cost(0);
4777 format %{ %}
4778 interface(REG_INTER);
4779 %}
4780
4781 // Pointer 64 bit Register R4 only
4782 operand iRegP_R4()
4783 %{
4784 constraint(ALLOC_IN_RC(r4_reg));
4785 match(RegP);
4786 // match(iRegP);
4787 match(iRegPNoSp);
4788 op_cost(0);
4789 format %{ %}
4790 interface(REG_INTER);
4791 %}
4792
4793 // Pointer 64 bit Register R5 only
4794 operand iRegP_R5()
4795 %{
4796 constraint(ALLOC_IN_RC(r5_reg));
4797 match(RegP);
4798 // match(iRegP);
4799 match(iRegPNoSp);
4800 op_cost(0);
4801 format %{ %}
4802 interface(REG_INTER);
4803 %}
4804
4805 // Pointer 64 bit Register R10 only
4806 operand iRegP_R10()
4807 %{
4808 constraint(ALLOC_IN_RC(r10_reg));
4809 match(RegP);
4810 // match(iRegP);
4811 match(iRegPNoSp);
4812 op_cost(0);
4813 format %{ %}
4814 interface(REG_INTER);
4815 %}
4816
4817 // Long 64 bit Register R0 only
4818 operand iRegL_R0()
4819 %{
4820 constraint(ALLOC_IN_RC(r0_reg));
4821 match(RegL);
4822 match(iRegLNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Long 64 bit Register R11 only
4829 operand iRegL_R11()
4830 %{
4831 constraint(ALLOC_IN_RC(r11_reg));
4832 match(RegL);
4833 match(iRegLNoSp);
4834 op_cost(0);
4835 format %{ %}
4836 interface(REG_INTER);
4837 %}
4838
4839 // Register R0 only
4840 operand iRegI_R0()
4841 %{
4842 constraint(ALLOC_IN_RC(int_r0_reg));
4843 match(RegI);
4844 match(iRegINoSp);
4845 op_cost(0);
4846 format %{ %}
4847 interface(REG_INTER);
4848 %}
4849
4850 // Register R2 only
4851 operand iRegI_R2()
4852 %{
4853 constraint(ALLOC_IN_RC(int_r2_reg));
4854 match(RegI);
4855 match(iRegINoSp);
4856 op_cost(0);
4857 format %{ %}
4858 interface(REG_INTER);
4859 %}
4860
4861 // Register R3 only
4862 operand iRegI_R3()
4863 %{
4864 constraint(ALLOC_IN_RC(int_r3_reg));
4865 match(RegI);
4866 match(iRegINoSp);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872
4873 // Register R4 only
4874 operand iRegI_R4()
4875 %{
4876 constraint(ALLOC_IN_RC(int_r4_reg));
4877 match(RegI);
4878 match(iRegINoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884
4885 // Pointer Register Operands
4886 // Narrow Pointer Register
4887 operand iRegN()
4888 %{
4889 constraint(ALLOC_IN_RC(any_reg32));
4890 match(RegN);
4891 match(iRegNNoSp);
4892 op_cost(0);
4893 format %{ %}
4894 interface(REG_INTER);
4895 %}
4896
4897 // Integer 64 bit Register not Special
4898 operand iRegNNoSp()
4899 %{
4900 constraint(ALLOC_IN_RC(no_special_reg32));
4901 match(RegN);
4902 op_cost(0);
4903 format %{ %}
4904 interface(REG_INTER);
4905 %}
4906
4907 // Float Register
4908 // Float register operands
4909 operand vRegF()
4910 %{
4911 constraint(ALLOC_IN_RC(float_reg));
4912 match(RegF);
4913
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919 // Double Register
4920 // Double register operands
4921 operand vRegD()
4922 %{
4923 constraint(ALLOC_IN_RC(double_reg));
4924 match(RegD);
4925
4926 op_cost(0);
4927 format %{ %}
4928 interface(REG_INTER);
4929 %}
4930
4931 // Generic vector class. This will be used for
4932 // all vector operands, including NEON and SVE.
4933 operand vReg()
4934 %{
4935 constraint(ALLOC_IN_RC(dynamic));
4936 match(VecA);
4937 match(VecD);
4938 match(VecX);
4939
4940 op_cost(0);
4941 format %{ %}
4942 interface(REG_INTER);
4943 %}
4944
4945 operand vecA()
4946 %{
4947 constraint(ALLOC_IN_RC(vectora_reg));
4948 match(VecA);
4949
4950 op_cost(0);
4951 format %{ %}
4952 interface(REG_INTER);
4953 %}
4954
4955 operand vecD()
4956 %{
4957 constraint(ALLOC_IN_RC(vectord_reg));
4958 match(VecD);
4959
4960 op_cost(0);
4961 format %{ %}
4962 interface(REG_INTER);
4963 %}
4964
4965 operand vecX()
4966 %{
4967 constraint(ALLOC_IN_RC(vectorx_reg));
4968 match(VecX);
4969
4970 op_cost(0);
4971 format %{ %}
4972 interface(REG_INTER);
4973 %}
4974
4975 operand vRegD_V0()
4976 %{
4977 constraint(ALLOC_IN_RC(v0_reg));
4978 match(RegD);
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vRegD_V1()
4985 %{
4986 constraint(ALLOC_IN_RC(v1_reg));
4987 match(RegD);
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vRegD_V2()
4994 %{
4995 constraint(ALLOC_IN_RC(v2_reg));
4996 match(RegD);
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vRegD_V3()
5003 %{
5004 constraint(ALLOC_IN_RC(v3_reg));
5005 match(RegD);
5006 op_cost(0);
5007 format %{ %}
5008 interface(REG_INTER);
5009 %}
5010
5011 operand vRegD_V4()
5012 %{
5013 constraint(ALLOC_IN_RC(v4_reg));
5014 match(RegD);
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vRegD_V5()
5021 %{
5022 constraint(ALLOC_IN_RC(v5_reg));
5023 match(RegD);
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 operand vRegD_V6()
5030 %{
5031 constraint(ALLOC_IN_RC(v6_reg));
5032 match(RegD);
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 operand vRegD_V7()
5039 %{
5040 constraint(ALLOC_IN_RC(v7_reg));
5041 match(RegD);
5042 op_cost(0);
5043 format %{ %}
5044 interface(REG_INTER);
5045 %}
5046
5047 operand vRegD_V12()
5048 %{
5049 constraint(ALLOC_IN_RC(v12_reg));
5050 match(RegD);
5051 op_cost(0);
5052 format %{ %}
5053 interface(REG_INTER);
5054 %}
5055
5056 operand vRegD_V13()
5057 %{
5058 constraint(ALLOC_IN_RC(v13_reg));
5059 match(RegD);
5060 op_cost(0);
5061 format %{ %}
5062 interface(REG_INTER);
5063 %}
5064
5065 operand pReg()
5066 %{
5067 constraint(ALLOC_IN_RC(pr_reg));
5068 match(RegVectMask);
5069 match(pRegGov);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075 operand pRegGov()
5076 %{
5077 constraint(ALLOC_IN_RC(gov_pr));
5078 match(RegVectMask);
5079 match(pReg);
5080 op_cost(0);
5081 format %{ %}
5082 interface(REG_INTER);
5083 %}
5084
5085 operand pRegGov_P0()
5086 %{
5087 constraint(ALLOC_IN_RC(p0_reg));
5088 match(RegVectMask);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 operand pRegGov_P1()
5095 %{
5096 constraint(ALLOC_IN_RC(p1_reg));
5097 match(RegVectMask);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 // Flags register, used as output of signed compare instructions
5104
5105 // note that on AArch64 we also use this register as the output for
5106 // for floating point compare instructions (CmpF CmpD). this ensures
5107 // that ordered inequality tests use GT, GE, LT or LE none of which
5108 // pass through cases where the result is unordered i.e. one or both
5109 // inputs to the compare is a NaN. this means that the ideal code can
5110 // replace e.g. a GT with an LE and not end up capturing the NaN case
5111 // (where the comparison should always fail). EQ and NE tests are
5112 // always generated in ideal code so that unordered folds into the NE
5113 // case, matching the behaviour of AArch64 NE.
5114 //
5115 // This differs from x86 where the outputs of FP compares use a
5116 // special FP flags registers and where compares based on this
5117 // register are distinguished into ordered inequalities (cmpOpUCF) and
5118 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5119 // to explicitly handle the unordered case in branches. x86 also has
5120 // to include extra CMoveX rules to accept a cmpOpUCF input.
5121
5122 operand rFlagsReg()
5123 %{
5124 constraint(ALLOC_IN_RC(int_flags));
5125 match(RegFlags);
5126
5127 op_cost(0);
5128 format %{ "RFLAGS" %}
5129 interface(REG_INTER);
5130 %}
5131
5132 // Flags register, used as output of unsigned compare instructions
5133 operand rFlagsRegU()
5134 %{
5135 constraint(ALLOC_IN_RC(int_flags));
5136 match(RegFlags);
5137
5138 op_cost(0);
5139 format %{ "RFLAGSU" %}
5140 interface(REG_INTER);
5141 %}
5142
5143 // Special Registers
5144
5145 // Method Register
5146 operand inline_cache_RegP(iRegP reg)
5147 %{
5148 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5149 match(reg);
5150 match(iRegPNoSp);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Thread Register
5157 operand thread_RegP(iRegP reg)
5158 %{
5159 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5160 match(reg);
5161 op_cost(0);
5162 format %{ %}
5163 interface(REG_INTER);
5164 %}
5165
5166 //----------Memory Operands----------------------------------------------------
5167
5168 operand indirect(iRegP reg)
5169 %{
5170 constraint(ALLOC_IN_RC(ptr_reg));
5171 match(reg);
5172 op_cost(0);
5173 format %{ "[$reg]" %}
5174 interface(MEMORY_INTER) %{
5175 base($reg);
5176 index(0xffffffff);
5177 scale(0x0);
5178 disp(0x0);
5179 %}
5180 %}
5181
5182 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5183 %{
5184 constraint(ALLOC_IN_RC(ptr_reg));
5185 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5186 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5187 op_cost(0);
5188 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5189 interface(MEMORY_INTER) %{
5190 base($reg);
5191 index($ireg);
5192 scale($scale);
5193 disp(0x0);
5194 %}
5195 %}
5196
5197 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5198 %{
5199 constraint(ALLOC_IN_RC(ptr_reg));
5200 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5201 match(AddP reg (LShiftL lreg scale));
5202 op_cost(0);
5203 format %{ "$reg, $lreg lsl($scale)" %}
5204 interface(MEMORY_INTER) %{
5205 base($reg);
5206 index($lreg);
5207 scale($scale);
5208 disp(0x0);
5209 %}
5210 %}
5211
5212 operand indIndexI2L(iRegP reg, iRegI ireg)
5213 %{
5214 constraint(ALLOC_IN_RC(ptr_reg));
5215 match(AddP reg (ConvI2L ireg));
5216 op_cost(0);
5217 format %{ "$reg, $ireg, 0, I2L" %}
5218 interface(MEMORY_INTER) %{
5219 base($reg);
5220 index($ireg);
5221 scale(0x0);
5222 disp(0x0);
5223 %}
5224 %}
5225
5226 operand indIndex(iRegP reg, iRegL lreg)
5227 %{
5228 constraint(ALLOC_IN_RC(ptr_reg));
5229 match(AddP reg lreg);
5230 op_cost(0);
5231 format %{ "$reg, $lreg" %}
5232 interface(MEMORY_INTER) %{
5233 base($reg);
5234 index($lreg);
5235 scale(0x0);
5236 disp(0x0);
5237 %}
5238 %}
5239
5240 operand indOffI1(iRegP reg, immIOffset1 off)
5241 %{
5242 constraint(ALLOC_IN_RC(ptr_reg));
5243 match(AddP reg off);
5244 op_cost(0);
5245 format %{ "[$reg, $off]" %}
5246 interface(MEMORY_INTER) %{
5247 base($reg);
5248 index(0xffffffff);
5249 scale(0x0);
5250 disp($off);
5251 %}
5252 %}
5253
5254 operand indOffI2(iRegP reg, immIOffset2 off)
5255 %{
5256 constraint(ALLOC_IN_RC(ptr_reg));
5257 match(AddP reg off);
5258 op_cost(0);
5259 format %{ "[$reg, $off]" %}
5260 interface(MEMORY_INTER) %{
5261 base($reg);
5262 index(0xffffffff);
5263 scale(0x0);
5264 disp($off);
5265 %}
5266 %}
5267
5268 operand indOffI4(iRegP reg, immIOffset4 off)
5269 %{
5270 constraint(ALLOC_IN_RC(ptr_reg));
5271 match(AddP reg off);
5272 op_cost(0);
5273 format %{ "[$reg, $off]" %}
5274 interface(MEMORY_INTER) %{
5275 base($reg);
5276 index(0xffffffff);
5277 scale(0x0);
5278 disp($off);
5279 %}
5280 %}
5281
5282 operand indOffI8(iRegP reg, immIOffset8 off)
5283 %{
5284 constraint(ALLOC_IN_RC(ptr_reg));
5285 match(AddP reg off);
5286 op_cost(0);
5287 format %{ "[$reg, $off]" %}
5288 interface(MEMORY_INTER) %{
5289 base($reg);
5290 index(0xffffffff);
5291 scale(0x0);
5292 disp($off);
5293 %}
5294 %}
5295
5296 operand indOffI16(iRegP reg, immIOffset16 off)
5297 %{
5298 constraint(ALLOC_IN_RC(ptr_reg));
5299 match(AddP reg off);
5300 op_cost(0);
5301 format %{ "[$reg, $off]" %}
5302 interface(MEMORY_INTER) %{
5303 base($reg);
5304 index(0xffffffff);
5305 scale(0x0);
5306 disp($off);
5307 %}
5308 %}
5309
5310 operand indOffL1(iRegP reg, immLoffset1 off)
5311 %{
5312 constraint(ALLOC_IN_RC(ptr_reg));
5313 match(AddP reg off);
5314 op_cost(0);
5315 format %{ "[$reg, $off]" %}
5316 interface(MEMORY_INTER) %{
5317 base($reg);
5318 index(0xffffffff);
5319 scale(0x0);
5320 disp($off);
5321 %}
5322 %}
5323
5324 operand indOffL2(iRegP reg, immLoffset2 off)
5325 %{
5326 constraint(ALLOC_IN_RC(ptr_reg));
5327 match(AddP reg off);
5328 op_cost(0);
5329 format %{ "[$reg, $off]" %}
5330 interface(MEMORY_INTER) %{
5331 base($reg);
5332 index(0xffffffff);
5333 scale(0x0);
5334 disp($off);
5335 %}
5336 %}
5337
5338 operand indOffL4(iRegP reg, immLoffset4 off)
5339 %{
5340 constraint(ALLOC_IN_RC(ptr_reg));
5341 match(AddP reg off);
5342 op_cost(0);
5343 format %{ "[$reg, $off]" %}
5344 interface(MEMORY_INTER) %{
5345 base($reg);
5346 index(0xffffffff);
5347 scale(0x0);
5348 disp($off);
5349 %}
5350 %}
5351
5352 operand indOffL8(iRegP reg, immLoffset8 off)
5353 %{
5354 constraint(ALLOC_IN_RC(ptr_reg));
5355 match(AddP reg off);
5356 op_cost(0);
5357 format %{ "[$reg, $off]" %}
5358 interface(MEMORY_INTER) %{
5359 base($reg);
5360 index(0xffffffff);
5361 scale(0x0);
5362 disp($off);
5363 %}
5364 %}
5365
5366 operand indOffL16(iRegP reg, immLoffset16 off)
5367 %{
5368 constraint(ALLOC_IN_RC(ptr_reg));
5369 match(AddP reg off);
5370 op_cost(0);
5371 format %{ "[$reg, $off]" %}
5372 interface(MEMORY_INTER) %{
5373 base($reg);
5374 index(0xffffffff);
5375 scale(0x0);
5376 disp($off);
5377 %}
5378 %}
5379
5380 operand indirectX2P(iRegL reg)
5381 %{
5382 constraint(ALLOC_IN_RC(ptr_reg));
5383 match(CastX2P reg);
5384 op_cost(0);
5385 format %{ "[$reg]\t# long -> ptr" %}
5386 interface(MEMORY_INTER) %{
5387 base($reg);
5388 index(0xffffffff);
5389 scale(0x0);
5390 disp(0x0);
5391 %}
5392 %}
5393
5394 operand indOffX2P(iRegL reg, immLOffset off)
5395 %{
5396 constraint(ALLOC_IN_RC(ptr_reg));
5397 match(AddP (CastX2P reg) off);
5398 op_cost(0);
5399 format %{ "[$reg, $off]\t# long -> ptr" %}
5400 interface(MEMORY_INTER) %{
5401 base($reg);
5402 index(0xffffffff);
5403 scale(0x0);
5404 disp($off);
5405 %}
5406 %}
5407
5408 operand indirectN(iRegN reg)
5409 %{
5410 predicate(CompressedOops::shift() == 0);
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(DecodeN reg);
5413 op_cost(0);
5414 format %{ "[$reg]\t# narrow" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp(0x0);
5420 %}
5421 %}
5422
5423 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5424 %{
5425 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5426 constraint(ALLOC_IN_RC(ptr_reg));
5427 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5428 op_cost(0);
5429 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5430 interface(MEMORY_INTER) %{
5431 base($reg);
5432 index($ireg);
5433 scale($scale);
5434 disp(0x0);
5435 %}
5436 %}
5437
5438 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5439 %{
5440 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5441 constraint(ALLOC_IN_RC(ptr_reg));
5442 match(AddP (DecodeN reg) (LShiftL lreg scale));
5443 op_cost(0);
5444 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5445 interface(MEMORY_INTER) %{
5446 base($reg);
5447 index($lreg);
5448 scale($scale);
5449 disp(0x0);
5450 %}
5451 %}
5452
5453 operand indIndexI2LN(iRegN reg, iRegI ireg)
5454 %{
5455 predicate(CompressedOops::shift() == 0);
5456 constraint(ALLOC_IN_RC(ptr_reg));
5457 match(AddP (DecodeN reg) (ConvI2L ireg));
5458 op_cost(0);
5459 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5460 interface(MEMORY_INTER) %{
5461 base($reg);
5462 index($ireg);
5463 scale(0x0);
5464 disp(0x0);
5465 %}
5466 %}
5467
5468 operand indIndexN(iRegN reg, iRegL lreg)
5469 %{
5470 predicate(CompressedOops::shift() == 0);
5471 constraint(ALLOC_IN_RC(ptr_reg));
5472 match(AddP (DecodeN reg) lreg);
5473 op_cost(0);
5474 format %{ "$reg, $lreg\t# narrow" %}
5475 interface(MEMORY_INTER) %{
5476 base($reg);
5477 index($lreg);
5478 scale(0x0);
5479 disp(0x0);
5480 %}
5481 %}
5482
5483 operand indOffIN(iRegN reg, immIOffset off)
5484 %{
5485 predicate(CompressedOops::shift() == 0);
5486 constraint(ALLOC_IN_RC(ptr_reg));
5487 match(AddP (DecodeN reg) off);
5488 op_cost(0);
5489 format %{ "[$reg, $off]\t# narrow" %}
5490 interface(MEMORY_INTER) %{
5491 base($reg);
5492 index(0xffffffff);
5493 scale(0x0);
5494 disp($off);
5495 %}
5496 %}
5497
5498 operand indOffLN(iRegN reg, immLOffset off)
5499 %{
5500 predicate(CompressedOops::shift() == 0);
5501 constraint(ALLOC_IN_RC(ptr_reg));
5502 match(AddP (DecodeN reg) off);
5503 op_cost(0);
5504 format %{ "[$reg, $off]\t# narrow" %}
5505 interface(MEMORY_INTER) %{
5506 base($reg);
5507 index(0xffffffff);
5508 scale(0x0);
5509 disp($off);
5510 %}
5511 %}
5512
5513
5514 //----------Special Memory Operands--------------------------------------------
5515 // Stack Slot Operand - This operand is used for loading and storing temporary
5516 // values on the stack where a match requires a value to
5517 // flow through memory.
5518 operand stackSlotP(sRegP reg)
5519 %{
5520 constraint(ALLOC_IN_RC(stack_slots));
5521 op_cost(100);
5522 // No match rule because this operand is only generated in matching
5523 // match(RegP);
5524 format %{ "[$reg]" %}
5525 interface(MEMORY_INTER) %{
5526 base(0x1e); // RSP
5527 index(0x0); // No Index
5528 scale(0x0); // No Scale
5529 disp($reg); // Stack Offset
5530 %}
5531 %}
5532
5533 operand stackSlotI(sRegI reg)
5534 %{
5535 constraint(ALLOC_IN_RC(stack_slots));
5536 // No match rule because this operand is only generated in matching
5537 // match(RegI);
5538 format %{ "[$reg]" %}
5539 interface(MEMORY_INTER) %{
5540 base(0x1e); // RSP
5541 index(0x0); // No Index
5542 scale(0x0); // No Scale
5543 disp($reg); // Stack Offset
5544 %}
5545 %}
5546
5547 operand stackSlotF(sRegF reg)
5548 %{
5549 constraint(ALLOC_IN_RC(stack_slots));
5550 // No match rule because this operand is only generated in matching
5551 // match(RegF);
5552 format %{ "[$reg]" %}
5553 interface(MEMORY_INTER) %{
5554 base(0x1e); // RSP
5555 index(0x0); // No Index
5556 scale(0x0); // No Scale
5557 disp($reg); // Stack Offset
5558 %}
5559 %}
5560
5561 operand stackSlotD(sRegD reg)
5562 %{
5563 constraint(ALLOC_IN_RC(stack_slots));
5564 // No match rule because this operand is only generated in matching
5565 // match(RegD);
5566 format %{ "[$reg]" %}
5567 interface(MEMORY_INTER) %{
5568 base(0x1e); // RSP
5569 index(0x0); // No Index
5570 scale(0x0); // No Scale
5571 disp($reg); // Stack Offset
5572 %}
5573 %}
5574
5575 operand stackSlotL(sRegL reg)
5576 %{
5577 constraint(ALLOC_IN_RC(stack_slots));
5578 // No match rule because this operand is only generated in matching
5579 // match(RegL);
5580 format %{ "[$reg]" %}
5581 interface(MEMORY_INTER) %{
5582 base(0x1e); // RSP
5583 index(0x0); // No Index
5584 scale(0x0); // No Scale
5585 disp($reg); // Stack Offset
5586 %}
5587 %}
5588
5589 // Operands for expressing Control Flow
5590 // NOTE: Label is a predefined operand which should not be redefined in
5591 // the AD file. It is generically handled within the ADLC.
5592
5593 //----------Conditional Branch Operands----------------------------------------
5594 // Comparison Op - This is the operation of the comparison, and is limited to
5595 // the following set of codes:
5596 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5597 //
5598 // Other attributes of the comparison, such as unsignedness, are specified
5599 // by the comparison instruction that sets a condition code flags register.
5600 // That result is represented by a flags operand whose subtype is appropriate
5601 // to the unsignedness (etc.) of the comparison.
5602 //
5603 // Later, the instruction which matches both the Comparison Op (a Bool) and
5604 // the flags (produced by the Cmp) specifies the coding of the comparison op
5605 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5606
5607 // used for signed integral comparisons and fp comparisons
5608
5609 operand cmpOp()
5610 %{
5611 match(Bool);
5612
5613 format %{ "" %}
5614 interface(COND_INTER) %{
5615 equal(0x0, "eq");
5616 not_equal(0x1, "ne");
5617 less(0xb, "lt");
5618 greater_equal(0xa, "ge");
5619 less_equal(0xd, "le");
5620 greater(0xc, "gt");
5621 overflow(0x6, "vs");
5622 no_overflow(0x7, "vc");
5623 %}
5624 %}
5625
5626 // used for unsigned integral comparisons
5627
5628 operand cmpOpU()
5629 %{
5630 match(Bool);
5631
5632 format %{ "" %}
5633 interface(COND_INTER) %{
5634 equal(0x0, "eq");
5635 not_equal(0x1, "ne");
5636 less(0x3, "lo");
5637 greater_equal(0x2, "hs");
5638 less_equal(0x9, "ls");
5639 greater(0x8, "hi");
5640 overflow(0x6, "vs");
5641 no_overflow(0x7, "vc");
5642 %}
5643 %}
5644
5645 // used for certain integral comparisons which can be
5646 // converted to cbxx or tbxx instructions
5647
5648 operand cmpOpEqNe()
5649 %{
5650 match(Bool);
5651 op_cost(0);
5652 predicate(n->as_Bool()->_test._test == BoolTest::ne
5653 || n->as_Bool()->_test._test == BoolTest::eq);
5654
5655 format %{ "" %}
5656 interface(COND_INTER) %{
5657 equal(0x0, "eq");
5658 not_equal(0x1, "ne");
5659 less(0xb, "lt");
5660 greater_equal(0xa, "ge");
5661 less_equal(0xd, "le");
5662 greater(0xc, "gt");
5663 overflow(0x6, "vs");
5664 no_overflow(0x7, "vc");
5665 %}
5666 %}
5667
5668 // used for certain integral comparisons which can be
5669 // converted to cbxx or tbxx instructions
5670
5671 operand cmpOpLtGe()
5672 %{
5673 match(Bool);
5674 op_cost(0);
5675
5676 predicate(n->as_Bool()->_test._test == BoolTest::lt
5677 || n->as_Bool()->_test._test == BoolTest::ge);
5678
5679 format %{ "" %}
5680 interface(COND_INTER) %{
5681 equal(0x0, "eq");
5682 not_equal(0x1, "ne");
5683 less(0xb, "lt");
5684 greater_equal(0xa, "ge");
5685 less_equal(0xd, "le");
5686 greater(0xc, "gt");
5687 overflow(0x6, "vs");
5688 no_overflow(0x7, "vc");
5689 %}
5690 %}
5691
5692 // used for certain unsigned integral comparisons which can be
5693 // converted to cbxx or tbxx instructions
5694
5695 operand cmpOpUEqNeLeGt()
5696 %{
5697 match(Bool);
5698 op_cost(0);
5699
5700 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5701 n->as_Bool()->_test._test == BoolTest::ne ||
5702 n->as_Bool()->_test._test == BoolTest::le ||
5703 n->as_Bool()->_test._test == BoolTest::gt);
5704
5705 format %{ "" %}
5706 interface(COND_INTER) %{
5707 equal(0x0, "eq");
5708 not_equal(0x1, "ne");
5709 less(0x3, "lo");
5710 greater_equal(0x2, "hs");
5711 less_equal(0x9, "ls");
5712 greater(0x8, "hi");
5713 overflow(0x6, "vs");
5714 no_overflow(0x7, "vc");
5715 %}
5716 %}
5717
5718 // Special operand allowing long args to int ops to be truncated for free
5719
5720 operand iRegL2I(iRegL reg) %{
5721
5722 op_cost(0);
5723
5724 match(ConvL2I reg);
5725
5726 format %{ "l2i($reg)" %}
5727
5728 interface(REG_INTER)
5729 %}
5730
5731 operand iRegL2P(iRegL reg) %{
5732
5733 op_cost(0);
5734
5735 match(CastX2P reg);
5736
5737 format %{ "l2p($reg)" %}
5738
5739 interface(REG_INTER)
5740 %}
5741
5742 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5743 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5744 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5745 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5746
5747 //----------OPERAND CLASSES----------------------------------------------------
5748 // Operand Classes are groups of operands that are used as to simplify
5749 // instruction definitions by not requiring the AD writer to specify
5750 // separate instructions for every form of operand when the
5751 // instruction accepts multiple operand types with the same basic
5752 // encoding and format. The classic case of this is memory operands.
5753
5754 // memory is used to define read/write location for load/store
5755 // instruction defs. we can turn a memory op into an Address
5756
5757 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5758 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5759
5760 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5761 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5762
5763 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5764 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5765
5766 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5767 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5768
5769 // All of the memory operands. For the pipeline description.
5770 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5771 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5772 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5773
5774
5775 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5776 // operations. it allows the src to be either an iRegI or a (ConvL2I
5777 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5778 // can be elided because the 32-bit instruction will just employ the
5779 // lower 32 bits anyway.
5780 //
5781 // n.b. this does not elide all L2I conversions. if the truncated
5782 // value is consumed by more than one operation then the ConvL2I
5783 // cannot be bundled into the consuming nodes so an l2i gets planted
5784 // (actually a movw $dst $src) and the downstream instructions consume
5785 // the result of the l2i as an iRegI input. That's a shame since the
5786 // movw is actually redundant but its not too costly.
5787
5788 opclass iRegIorL2I(iRegI, iRegL2I);
5789 opclass iRegPorL2P(iRegP, iRegL2P);
5790
5791 //----------PIPELINE-----------------------------------------------------------
5792 // Rules which define the behavior of the target architectures pipeline.
5793
5794 // For specific pipelines, eg A53, define the stages of that pipeline
5795 //pipe_desc(ISS, EX1, EX2, WR);
5796 #define ISS S0
5797 #define EX1 S1
5798 #define EX2 S2
5799 #define WR S3
5800
5801 // Integer ALU reg operation
5802 pipeline %{
5803
5804 attributes %{
5805 // ARM instructions are of fixed length
5806 fixed_size_instructions; // Fixed size instructions TODO does
5807 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5808 // ARM instructions come in 32-bit word units
5809 instruction_unit_size = 4; // An instruction is 4 bytes long
5810 instruction_fetch_unit_size = 64; // The processor fetches one line
5811 instruction_fetch_units = 1; // of 64 bytes
5812
5813 // List of nop instructions
5814 nops( MachNop );
5815 %}
5816
5817 // We don't use an actual pipeline model so don't care about resources
5818 // or description. we do use pipeline classes to introduce fixed
5819 // latencies
5820
5821 //----------RESOURCES----------------------------------------------------------
5822 // Resources are the functional units available to the machine
5823
5824 resources( INS0, INS1, INS01 = INS0 | INS1,
5825 ALU0, ALU1, ALU = ALU0 | ALU1,
5826 MAC,
5827 DIV,
5828 BRANCH,
5829 LDST,
5830 NEON_FP);
5831
5832 //----------PIPELINE DESCRIPTION-----------------------------------------------
5833 // Pipeline Description specifies the stages in the machine's pipeline
5834
5835 // Define the pipeline as a generic 6 stage pipeline
5836 pipe_desc(S0, S1, S2, S3, S4, S5);
5837
5838 //----------PIPELINE CLASSES---------------------------------------------------
5839 // Pipeline Classes describe the stages in which input and output are
5840 // referenced by the hardware pipeline.
5841
5842 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5843 %{
5844 single_instruction;
5845 src1 : S1(read);
5846 src2 : S2(read);
5847 dst : S5(write);
5848 INS01 : ISS;
5849 NEON_FP : S5;
5850 %}
5851
5852 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5853 %{
5854 single_instruction;
5855 src1 : S1(read);
5856 src2 : S2(read);
5857 dst : S5(write);
5858 INS01 : ISS;
5859 NEON_FP : S5;
5860 %}
5861
5862 pipe_class fp_uop_s(vRegF dst, vRegF src)
5863 %{
5864 single_instruction;
5865 src : S1(read);
5866 dst : S5(write);
5867 INS01 : ISS;
5868 NEON_FP : S5;
5869 %}
5870
5871 pipe_class fp_uop_d(vRegD dst, vRegD src)
5872 %{
5873 single_instruction;
5874 src : S1(read);
5875 dst : S5(write);
5876 INS01 : ISS;
5877 NEON_FP : S5;
5878 %}
5879
5880 pipe_class fp_d2f(vRegF dst, vRegD src)
5881 %{
5882 single_instruction;
5883 src : S1(read);
5884 dst : S5(write);
5885 INS01 : ISS;
5886 NEON_FP : S5;
5887 %}
5888
5889 pipe_class fp_f2d(vRegD dst, vRegF src)
5890 %{
5891 single_instruction;
5892 src : S1(read);
5893 dst : S5(write);
5894 INS01 : ISS;
5895 NEON_FP : S5;
5896 %}
5897
5898 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5899 %{
5900 single_instruction;
5901 src : S1(read);
5902 dst : S5(write);
5903 INS01 : ISS;
5904 NEON_FP : S5;
5905 %}
5906
5907 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5908 %{
5909 single_instruction;
5910 src : S1(read);
5911 dst : S5(write);
5912 INS01 : ISS;
5913 NEON_FP : S5;
5914 %}
5915
5916 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5917 %{
5918 single_instruction;
5919 src : S1(read);
5920 dst : S5(write);
5921 INS01 : ISS;
5922 NEON_FP : S5;
5923 %}
5924
5925 pipe_class fp_l2f(vRegF dst, iRegL src)
5926 %{
5927 single_instruction;
5928 src : S1(read);
5929 dst : S5(write);
5930 INS01 : ISS;
5931 NEON_FP : S5;
5932 %}
5933
5934 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5935 %{
5936 single_instruction;
5937 src : S1(read);
5938 dst : S5(write);
5939 INS01 : ISS;
5940 NEON_FP : S5;
5941 %}
5942
5943 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5944 %{
5945 single_instruction;
5946 src : S1(read);
5947 dst : S5(write);
5948 INS01 : ISS;
5949 NEON_FP : S5;
5950 %}
5951
5952 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5953 %{
5954 single_instruction;
5955 src : S1(read);
5956 dst : S5(write);
5957 INS01 : ISS;
5958 NEON_FP : S5;
5959 %}
5960
5961 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5962 %{
5963 single_instruction;
5964 src : S1(read);
5965 dst : S5(write);
5966 INS01 : ISS;
5967 NEON_FP : S5;
5968 %}
5969
5970 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5971 %{
5972 single_instruction;
5973 src1 : S1(read);
5974 src2 : S2(read);
5975 dst : S5(write);
5976 INS0 : ISS;
5977 NEON_FP : S5;
5978 %}
5979
5980 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5981 %{
5982 single_instruction;
5983 src1 : S1(read);
5984 src2 : S2(read);
5985 dst : S5(write);
5986 INS0 : ISS;
5987 NEON_FP : S5;
5988 %}
5989
5990 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5991 %{
5992 single_instruction;
5993 cr : S1(read);
5994 src1 : S1(read);
5995 src2 : S1(read);
5996 dst : S3(write);
5997 INS01 : ISS;
5998 NEON_FP : S3;
5999 %}
6000
6001 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6002 %{
6003 single_instruction;
6004 cr : S1(read);
6005 src1 : S1(read);
6006 src2 : S1(read);
6007 dst : S3(write);
6008 INS01 : ISS;
6009 NEON_FP : S3;
6010 %}
6011
6012 pipe_class fp_imm_s(vRegF dst)
6013 %{
6014 single_instruction;
6015 dst : S3(write);
6016 INS01 : ISS;
6017 NEON_FP : S3;
6018 %}
6019
6020 pipe_class fp_imm_d(vRegD dst)
6021 %{
6022 single_instruction;
6023 dst : S3(write);
6024 INS01 : ISS;
6025 NEON_FP : S3;
6026 %}
6027
6028 pipe_class fp_load_constant_s(vRegF dst)
6029 %{
6030 single_instruction;
6031 dst : S4(write);
6032 INS01 : ISS;
6033 NEON_FP : S4;
6034 %}
6035
6036 pipe_class fp_load_constant_d(vRegD dst)
6037 %{
6038 single_instruction;
6039 dst : S4(write);
6040 INS01 : ISS;
6041 NEON_FP : S4;
6042 %}
6043
6044 //------- Integer ALU operations --------------------------
6045
6046 // Integer ALU reg-reg operation
6047 // Operands needed in EX1, result generated in EX2
6048 // Eg. ADD x0, x1, x2
6049 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6050 %{
6051 single_instruction;
6052 dst : EX2(write);
6053 src1 : EX1(read);
6054 src2 : EX1(read);
6055 INS01 : ISS; // Dual issue as instruction 0 or 1
6056 ALU : EX2;
6057 %}
6058
6059 // Integer ALU reg-reg operation with constant shift
6060 // Shifted register must be available in LATE_ISS instead of EX1
6061 // Eg. ADD x0, x1, x2, LSL #2
6062 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6063 %{
6064 single_instruction;
6065 dst : EX2(write);
6066 src1 : EX1(read);
6067 src2 : ISS(read);
6068 INS01 : ISS;
6069 ALU : EX2;
6070 %}
6071
6072 // Integer ALU reg operation with constant shift
6073 // Eg. LSL x0, x1, #shift
6074 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6075 %{
6076 single_instruction;
6077 dst : EX2(write);
6078 src1 : ISS(read);
6079 INS01 : ISS;
6080 ALU : EX2;
6081 %}
6082
6083 // Integer ALU reg-reg operation with variable shift
6084 // Both operands must be available in LATE_ISS instead of EX1
6085 // Result is available in EX1 instead of EX2
6086 // Eg. LSLV x0, x1, x2
6087 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6088 %{
6089 single_instruction;
6090 dst : EX1(write);
6091 src1 : ISS(read);
6092 src2 : ISS(read);
6093 INS01 : ISS;
6094 ALU : EX1;
6095 %}
6096
6097 // Integer ALU reg-reg operation with extract
6098 // As for _vshift above, but result generated in EX2
6099 // Eg. EXTR x0, x1, x2, #N
6100 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6101 %{
6102 single_instruction;
6103 dst : EX2(write);
6104 src1 : ISS(read);
6105 src2 : ISS(read);
6106 INS1 : ISS; // Can only dual issue as Instruction 1
6107 ALU : EX1;
6108 %}
6109
6110 // Integer ALU reg operation
6111 // Eg. NEG x0, x1
6112 pipe_class ialu_reg(iRegI dst, iRegI src)
6113 %{
6114 single_instruction;
6115 dst : EX2(write);
6116 src : EX1(read);
6117 INS01 : ISS;
6118 ALU : EX2;
6119 %}
6120
6121 // Integer ALU reg mmediate operation
6122 // Eg. ADD x0, x1, #N
6123 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6124 %{
6125 single_instruction;
6126 dst : EX2(write);
6127 src1 : EX1(read);
6128 INS01 : ISS;
6129 ALU : EX2;
6130 %}
6131
6132 // Integer ALU immediate operation (no source operands)
6133 // Eg. MOV x0, #N
6134 pipe_class ialu_imm(iRegI dst)
6135 %{
6136 single_instruction;
6137 dst : EX1(write);
6138 INS01 : ISS;
6139 ALU : EX1;
6140 %}
6141
6142 //------- Compare operation -------------------------------
6143
6144 // Compare reg-reg
6145 // Eg. CMP x0, x1
6146 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6147 %{
6148 single_instruction;
6149 // fixed_latency(16);
6150 cr : EX2(write);
6151 op1 : EX1(read);
6152 op2 : EX1(read);
6153 INS01 : ISS;
6154 ALU : EX2;
6155 %}
6156
6157 // Compare reg-reg
6158 // Eg. CMP x0, #N
6159 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6160 %{
6161 single_instruction;
6162 // fixed_latency(16);
6163 cr : EX2(write);
6164 op1 : EX1(read);
6165 INS01 : ISS;
6166 ALU : EX2;
6167 %}
6168
6169 //------- Conditional instructions ------------------------
6170
6171 // Conditional no operands
6172 // Eg. CSINC x0, zr, zr, <cond>
6173 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6174 %{
6175 single_instruction;
6176 cr : EX1(read);
6177 dst : EX2(write);
6178 INS01 : ISS;
6179 ALU : EX2;
6180 %}
6181
6182 // Conditional 2 operand
6183 // EG. CSEL X0, X1, X2, <cond>
6184 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6185 %{
6186 single_instruction;
6187 cr : EX1(read);
6188 src1 : EX1(read);
6189 src2 : EX1(read);
6190 dst : EX2(write);
6191 INS01 : ISS;
6192 ALU : EX2;
6193 %}
6194
6195 // Conditional 2 operand
6196 // EG. CSEL X0, X1, X2, <cond>
6197 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6198 %{
6199 single_instruction;
6200 cr : EX1(read);
6201 src : EX1(read);
6202 dst : EX2(write);
6203 INS01 : ISS;
6204 ALU : EX2;
6205 %}
6206
6207 //------- Multiply pipeline operations --------------------
6208
6209 // Multiply reg-reg
6210 // Eg. MUL w0, w1, w2
6211 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6212 %{
6213 single_instruction;
6214 dst : WR(write);
6215 src1 : ISS(read);
6216 src2 : ISS(read);
6217 INS01 : ISS;
6218 MAC : WR;
6219 %}
6220
6221 // Multiply accumulate
6222 // Eg. MADD w0, w1, w2, w3
6223 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6224 %{
6225 single_instruction;
6226 dst : WR(write);
6227 src1 : ISS(read);
6228 src2 : ISS(read);
6229 src3 : ISS(read);
6230 INS01 : ISS;
6231 MAC : WR;
6232 %}
6233
6234 // Eg. MUL w0, w1, w2
6235 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6236 %{
6237 single_instruction;
6238 fixed_latency(3); // Maximum latency for 64 bit mul
6239 dst : WR(write);
6240 src1 : ISS(read);
6241 src2 : ISS(read);
6242 INS01 : ISS;
6243 MAC : WR;
6244 %}
6245
6246 // Multiply accumulate
6247 // Eg. MADD w0, w1, w2, w3
6248 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6249 %{
6250 single_instruction;
6251 fixed_latency(3); // Maximum latency for 64 bit mul
6252 dst : WR(write);
6253 src1 : ISS(read);
6254 src2 : ISS(read);
6255 src3 : ISS(read);
6256 INS01 : ISS;
6257 MAC : WR;
6258 %}
6259
6260 //------- Divide pipeline operations --------------------
6261
6262 // Eg. SDIV w0, w1, w2
6263 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6264 %{
6265 single_instruction;
6266 fixed_latency(8); // Maximum latency for 32 bit divide
6267 dst : WR(write);
6268 src1 : ISS(read);
6269 src2 : ISS(read);
6270 INS0 : ISS; // Can only dual issue as instruction 0
6271 DIV : WR;
6272 %}
6273
6274 // Eg. SDIV x0, x1, x2
6275 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6276 %{
6277 single_instruction;
6278 fixed_latency(16); // Maximum latency for 64 bit divide
6279 dst : WR(write);
6280 src1 : ISS(read);
6281 src2 : ISS(read);
6282 INS0 : ISS; // Can only dual issue as instruction 0
6283 DIV : WR;
6284 %}
6285
6286 //------- Load pipeline operations ------------------------
6287
6288 // Load - prefetch
6289 // Eg. PFRM <mem>
6290 pipe_class iload_prefetch(memory mem)
6291 %{
6292 single_instruction;
6293 mem : ISS(read);
6294 INS01 : ISS;
6295 LDST : WR;
6296 %}
6297
6298 // Load - reg, mem
6299 // Eg. LDR x0, <mem>
6300 pipe_class iload_reg_mem(iRegI dst, memory mem)
6301 %{
6302 single_instruction;
6303 dst : WR(write);
6304 mem : ISS(read);
6305 INS01 : ISS;
6306 LDST : WR;
6307 %}
6308
6309 // Load - reg, reg
6310 // Eg. LDR x0, [sp, x1]
6311 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6312 %{
6313 single_instruction;
6314 dst : WR(write);
6315 src : ISS(read);
6316 INS01 : ISS;
6317 LDST : WR;
6318 %}
6319
6320 //------- Store pipeline operations -----------------------
6321
6322 // Store - zr, mem
6323 // Eg. STR zr, <mem>
6324 pipe_class istore_mem(memory mem)
6325 %{
6326 single_instruction;
6327 mem : ISS(read);
6328 INS01 : ISS;
6329 LDST : WR;
6330 %}
6331
6332 // Store - reg, mem
6333 // Eg. STR x0, <mem>
6334 pipe_class istore_reg_mem(iRegI src, memory mem)
6335 %{
6336 single_instruction;
6337 mem : ISS(read);
6338 src : EX2(read);
6339 INS01 : ISS;
6340 LDST : WR;
6341 %}
6342
6343 // Store - reg, reg
6344 // Eg. STR x0, [sp, x1]
6345 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6346 %{
6347 single_instruction;
6348 dst : ISS(read);
6349 src : EX2(read);
6350 INS01 : ISS;
6351 LDST : WR;
6352 %}
6353
6354 //------- Store pipeline operations -----------------------
6355
6356 // Branch
6357 pipe_class pipe_branch()
6358 %{
6359 single_instruction;
6360 INS01 : ISS;
6361 BRANCH : EX1;
6362 %}
6363
6364 // Conditional branch
6365 pipe_class pipe_branch_cond(rFlagsReg cr)
6366 %{
6367 single_instruction;
6368 cr : EX1(read);
6369 INS01 : ISS;
6370 BRANCH : EX1;
6371 %}
6372
6373 // Compare & Branch
6374 // EG. CBZ/CBNZ
6375 pipe_class pipe_cmp_branch(iRegI op1)
6376 %{
6377 single_instruction;
6378 op1 : EX1(read);
6379 INS01 : ISS;
6380 BRANCH : EX1;
6381 %}
6382
6383 //------- Synchronisation operations ----------------------
6384
6385 // Any operation requiring serialization.
6386 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6387 pipe_class pipe_serial()
6388 %{
6389 single_instruction;
6390 force_serialization;
6391 fixed_latency(16);
6392 INS01 : ISS(2); // Cannot dual issue with any other instruction
6393 LDST : WR;
6394 %}
6395
6396 // Generic big/slow expanded idiom - also serialized
6397 pipe_class pipe_slow()
6398 %{
6399 instruction_count(10);
6400 multiple_bundles;
6401 force_serialization;
6402 fixed_latency(16);
6403 INS01 : ISS(2); // Cannot dual issue with any other instruction
6404 LDST : WR;
6405 %}
6406
6407 // Empty pipeline class
6408 pipe_class pipe_class_empty()
6409 %{
6410 single_instruction;
6411 fixed_latency(0);
6412 %}
6413
6414 // Default pipeline class.
6415 pipe_class pipe_class_default()
6416 %{
6417 single_instruction;
6418 fixed_latency(2);
6419 %}
6420
6421 // Pipeline class for compares.
6422 pipe_class pipe_class_compare()
6423 %{
6424 single_instruction;
6425 fixed_latency(16);
6426 %}
6427
6428 // Pipeline class for memory operations.
6429 pipe_class pipe_class_memory()
6430 %{
6431 single_instruction;
6432 fixed_latency(16);
6433 %}
6434
6435 // Pipeline class for call.
6436 pipe_class pipe_class_call()
6437 %{
6438 single_instruction;
6439 fixed_latency(100);
6440 %}
6441
6442 // Define the class for the Nop node.
6443 define %{
6444 MachNop = pipe_class_empty;
6445 %}
6446
6447 %}
6448 //----------INSTRUCTIONS-------------------------------------------------------
6449 //
6450 // match -- States which machine-independent subtree may be replaced
6451 // by this instruction.
6452 // ins_cost -- The estimated cost of this instruction is used by instruction
6453 // selection to identify a minimum cost tree of machine
6454 // instructions that matches a tree of machine-independent
6455 // instructions.
6456 // format -- A string providing the disassembly for this instruction.
6457 // The value of an instruction's operand may be inserted
6458 // by referring to it with a '$' prefix.
6459 // opcode -- Three instruction opcodes may be provided. These are referred
6460 // to within an encode class as $primary, $secondary, and $tertiary
6461 // rrspectively. The primary opcode is commonly used to
6462 // indicate the type of machine instruction, while secondary
6463 // and tertiary are often used for prefix options or addressing
6464 // modes.
6465 // ins_encode -- A list of encode classes with parameters. The encode class
6466 // name must have been defined in an 'enc_class' specification
6467 // in the encode section of the architecture description.
6468
6469 // ============================================================================
6470 // Memory (Load/Store) Instructions
6471
6472 // Load Instructions
6473
6474 // Load Byte (8 bit signed)
6475 instruct loadB(iRegINoSp dst, memory1 mem)
6476 %{
6477 match(Set dst (LoadB mem));
6478 predicate(!needs_acquiring_load(n));
6479
6480 ins_cost(4 * INSN_COST);
6481 format %{ "ldrsbw $dst, $mem\t# byte" %}
6482
6483 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6484
6485 ins_pipe(iload_reg_mem);
6486 %}
6487
6488 // Load Byte (8 bit signed) into long
6489 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6490 %{
6491 match(Set dst (ConvI2L (LoadB mem)));
6492 predicate(!needs_acquiring_load(n->in(1)));
6493
6494 ins_cost(4 * INSN_COST);
6495 format %{ "ldrsb $dst, $mem\t# byte" %}
6496
6497 ins_encode(aarch64_enc_ldrsb(dst, mem));
6498
6499 ins_pipe(iload_reg_mem);
6500 %}
6501
6502 // Load Byte (8 bit unsigned)
6503 instruct loadUB(iRegINoSp dst, memory1 mem)
6504 %{
6505 match(Set dst (LoadUB mem));
6506 predicate(!needs_acquiring_load(n));
6507
6508 ins_cost(4 * INSN_COST);
6509 format %{ "ldrbw $dst, $mem\t# byte" %}
6510
6511 ins_encode(aarch64_enc_ldrb(dst, mem));
6512
6513 ins_pipe(iload_reg_mem);
6514 %}
6515
6516 // Load Byte (8 bit unsigned) into long
6517 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6518 %{
6519 match(Set dst (ConvI2L (LoadUB mem)));
6520 predicate(!needs_acquiring_load(n->in(1)));
6521
6522 ins_cost(4 * INSN_COST);
6523 format %{ "ldrb $dst, $mem\t# byte" %}
6524
6525 ins_encode(aarch64_enc_ldrb(dst, mem));
6526
6527 ins_pipe(iload_reg_mem);
6528 %}
6529
6530 // Load Short (16 bit signed)
6531 instruct loadS(iRegINoSp dst, memory2 mem)
6532 %{
6533 match(Set dst (LoadS mem));
6534 predicate(!needs_acquiring_load(n));
6535
6536 ins_cost(4 * INSN_COST);
6537 format %{ "ldrshw $dst, $mem\t# short" %}
6538
6539 ins_encode(aarch64_enc_ldrshw(dst, mem));
6540
6541 ins_pipe(iload_reg_mem);
6542 %}
6543
6544 // Load Short (16 bit signed) into long
6545 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6546 %{
6547 match(Set dst (ConvI2L (LoadS mem)));
6548 predicate(!needs_acquiring_load(n->in(1)));
6549
6550 ins_cost(4 * INSN_COST);
6551 format %{ "ldrsh $dst, $mem\t# short" %}
6552
6553 ins_encode(aarch64_enc_ldrsh(dst, mem));
6554
6555 ins_pipe(iload_reg_mem);
6556 %}
6557
6558 // Load Char (16 bit unsigned)
6559 instruct loadUS(iRegINoSp dst, memory2 mem)
6560 %{
6561 match(Set dst (LoadUS mem));
6562 predicate(!needs_acquiring_load(n));
6563
6564 ins_cost(4 * INSN_COST);
6565 format %{ "ldrh $dst, $mem\t# short" %}
6566
6567 ins_encode(aarch64_enc_ldrh(dst, mem));
6568
6569 ins_pipe(iload_reg_mem);
6570 %}
6571
6572 // Load Short/Char (16 bit unsigned) into long
6573 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6574 %{
6575 match(Set dst (ConvI2L (LoadUS mem)));
6576 predicate(!needs_acquiring_load(n->in(1)));
6577
6578 ins_cost(4 * INSN_COST);
6579 format %{ "ldrh $dst, $mem\t# short" %}
6580
6581 ins_encode(aarch64_enc_ldrh(dst, mem));
6582
6583 ins_pipe(iload_reg_mem);
6584 %}
6585
6586 // Load Integer (32 bit signed)
6587 instruct loadI(iRegINoSp dst, memory4 mem)
6588 %{
6589 match(Set dst (LoadI mem));
6590 predicate(!needs_acquiring_load(n));
6591
6592 ins_cost(4 * INSN_COST);
6593 format %{ "ldrw $dst, $mem\t# int" %}
6594
6595 ins_encode(aarch64_enc_ldrw(dst, mem));
6596
6597 ins_pipe(iload_reg_mem);
6598 %}
6599
6600 // Load Integer (32 bit signed) into long
6601 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6602 %{
6603 match(Set dst (ConvI2L (LoadI mem)));
6604 predicate(!needs_acquiring_load(n->in(1)));
6605
6606 ins_cost(4 * INSN_COST);
6607 format %{ "ldrsw $dst, $mem\t# int" %}
6608
6609 ins_encode(aarch64_enc_ldrsw(dst, mem));
6610
6611 ins_pipe(iload_reg_mem);
6612 %}
6613
6614 // Load Integer (32 bit unsigned) into long
6615 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6616 %{
6617 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6618 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6619
6620 ins_cost(4 * INSN_COST);
6621 format %{ "ldrw $dst, $mem\t# int" %}
6622
6623 ins_encode(aarch64_enc_ldrw(dst, mem));
6624
6625 ins_pipe(iload_reg_mem);
6626 %}
6627
6628 // Load Long (64 bit signed)
6629 instruct loadL(iRegLNoSp dst, memory8 mem)
6630 %{
6631 match(Set dst (LoadL mem));
6632 predicate(!needs_acquiring_load(n));
6633
6634 ins_cost(4 * INSN_COST);
6635 format %{ "ldr $dst, $mem\t# int" %}
6636
6637 ins_encode(aarch64_enc_ldr(dst, mem));
6638
6639 ins_pipe(iload_reg_mem);
6640 %}
6641
6642 // Load Range
6643 instruct loadRange(iRegINoSp dst, memory4 mem)
6644 %{
6645 match(Set dst (LoadRange mem));
6646
6647 ins_cost(4 * INSN_COST);
6648 format %{ "ldrw $dst, $mem\t# range" %}
6649
6650 ins_encode(aarch64_enc_ldrw(dst, mem));
6651
6652 ins_pipe(iload_reg_mem);
6653 %}
6654
6655 // Load Pointer
6656 instruct loadP(iRegPNoSp dst, memory8 mem)
6657 %{
6658 match(Set dst (LoadP mem));
6659 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6660
6661 ins_cost(4 * INSN_COST);
6662 format %{ "ldr $dst, $mem\t# ptr" %}
6663
6664 ins_encode(aarch64_enc_ldr(dst, mem));
6665
6666 ins_pipe(iload_reg_mem);
6667 %}
6668
6669 // Load Compressed Pointer
6670 instruct loadN(iRegNNoSp dst, memory4 mem)
6671 %{
6672 match(Set dst (LoadN mem));
6673 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6674
6675 ins_cost(4 * INSN_COST);
6676 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6677
6678 ins_encode(aarch64_enc_ldrw(dst, mem));
6679
6680 ins_pipe(iload_reg_mem);
6681 %}
6682
6683 // Load Klass Pointer
6684 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6685 %{
6686 match(Set dst (LoadKlass mem));
6687 predicate(!needs_acquiring_load(n));
6688
6689 ins_cost(4 * INSN_COST);
6690 format %{ "ldr $dst, $mem\t# class" %}
6691
6692 ins_encode(aarch64_enc_ldr(dst, mem));
6693
6694 ins_pipe(iload_reg_mem);
6695 %}
6696
6697 // Load Narrow Klass Pointer
6698 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6699 %{
6700 match(Set dst (LoadNKlass mem));
6701 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6702
6703 ins_cost(4 * INSN_COST);
6704 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6705
6706 ins_encode(aarch64_enc_ldrw(dst, mem));
6707
6708 ins_pipe(iload_reg_mem);
6709 %}
6710
6711 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6712 %{
6713 match(Set dst (LoadNKlass mem));
6714 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6715
6716 ins_cost(4 * INSN_COST);
6717 format %{
6718 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6719 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6720 %}
6721 ins_encode %{
6722 // inlined aarch64_enc_ldrw
6723 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6724 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6725 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6726 %}
6727 ins_pipe(iload_reg_mem);
6728 %}
6729
6730 // Load Float
6731 instruct loadF(vRegF dst, memory4 mem)
6732 %{
6733 match(Set dst (LoadF mem));
6734 predicate(!needs_acquiring_load(n));
6735
6736 ins_cost(4 * INSN_COST);
6737 format %{ "ldrs $dst, $mem\t# float" %}
6738
6739 ins_encode( aarch64_enc_ldrs(dst, mem) );
6740
6741 ins_pipe(pipe_class_memory);
6742 %}
6743
6744 // Load Double
6745 instruct loadD(vRegD dst, memory8 mem)
6746 %{
6747 match(Set dst (LoadD mem));
6748 predicate(!needs_acquiring_load(n));
6749
6750 ins_cost(4 * INSN_COST);
6751 format %{ "ldrd $dst, $mem\t# double" %}
6752
6753 ins_encode( aarch64_enc_ldrd(dst, mem) );
6754
6755 ins_pipe(pipe_class_memory);
6756 %}
6757
6758
6759 // Load Int Constant
6760 instruct loadConI(iRegINoSp dst, immI src)
6761 %{
6762 match(Set dst src);
6763
6764 ins_cost(INSN_COST);
6765 format %{ "mov $dst, $src\t# int" %}
6766
6767 ins_encode( aarch64_enc_movw_imm(dst, src) );
6768
6769 ins_pipe(ialu_imm);
6770 %}
6771
6772 // Load Long Constant
6773 instruct loadConL(iRegLNoSp dst, immL src)
6774 %{
6775 match(Set dst src);
6776
6777 ins_cost(INSN_COST);
6778 format %{ "mov $dst, $src\t# long" %}
6779
6780 ins_encode( aarch64_enc_mov_imm(dst, src) );
6781
6782 ins_pipe(ialu_imm);
6783 %}
6784
6785 // Load Pointer Constant
6786
6787 instruct loadConP(iRegPNoSp dst, immP con)
6788 %{
6789 match(Set dst con);
6790
6791 ins_cost(INSN_COST * 4);
6792 format %{
6793 "mov $dst, $con\t# ptr\n\t"
6794 %}
6795
6796 ins_encode(aarch64_enc_mov_p(dst, con));
6797
6798 ins_pipe(ialu_imm);
6799 %}
6800
6801 // Load Null Pointer Constant
6802
6803 instruct loadConP0(iRegPNoSp dst, immP0 con)
6804 %{
6805 match(Set dst con);
6806
6807 ins_cost(INSN_COST);
6808 format %{ "mov $dst, $con\t# nullptr ptr" %}
6809
6810 ins_encode(aarch64_enc_mov_p0(dst, con));
6811
6812 ins_pipe(ialu_imm);
6813 %}
6814
6815 // Load Pointer Constant One
6816
6817 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6818 %{
6819 match(Set dst con);
6820
6821 ins_cost(INSN_COST);
6822 format %{ "mov $dst, $con\t# nullptr ptr" %}
6823
6824 ins_encode(aarch64_enc_mov_p1(dst, con));
6825
6826 ins_pipe(ialu_imm);
6827 %}
6828
6829 // Load Byte Map Base Constant
6830
6831 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6832 %{
6833 match(Set dst con);
6834
6835 ins_cost(INSN_COST);
6836 format %{ "adr $dst, $con\t# Byte Map Base" %}
6837
6838 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6839
6840 ins_pipe(ialu_imm);
6841 %}
6842
6843 // Load Narrow Pointer Constant
6844
6845 instruct loadConN(iRegNNoSp dst, immN con)
6846 %{
6847 match(Set dst con);
6848
6849 ins_cost(INSN_COST * 4);
6850 format %{ "mov $dst, $con\t# compressed ptr" %}
6851
6852 ins_encode(aarch64_enc_mov_n(dst, con));
6853
6854 ins_pipe(ialu_imm);
6855 %}
6856
6857 // Load Narrow Null Pointer Constant
6858
6859 instruct loadConN0(iRegNNoSp dst, immN0 con)
6860 %{
6861 match(Set dst con);
6862
6863 ins_cost(INSN_COST);
6864 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6865
6866 ins_encode(aarch64_enc_mov_n0(dst, con));
6867
6868 ins_pipe(ialu_imm);
6869 %}
6870
6871 // Load Narrow Klass Constant
6872
6873 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6874 %{
6875 match(Set dst con);
6876
6877 ins_cost(INSN_COST);
6878 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6879
6880 ins_encode(aarch64_enc_mov_nk(dst, con));
6881
6882 ins_pipe(ialu_imm);
6883 %}
6884
6885 // Load Packed Float Constant
6886
6887 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6888 match(Set dst con);
6889 ins_cost(INSN_COST * 4);
6890 format %{ "fmovs $dst, $con"%}
6891 ins_encode %{
6892 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6893 %}
6894
6895 ins_pipe(fp_imm_s);
6896 %}
6897
6898 // Load Float Constant
6899
6900 instruct loadConF(vRegF dst, immF con) %{
6901 match(Set dst con);
6902
6903 ins_cost(INSN_COST * 4);
6904
6905 format %{
6906 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6907 %}
6908
6909 ins_encode %{
6910 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6911 %}
6912
6913 ins_pipe(fp_load_constant_s);
6914 %}
6915
6916 // Load Packed Double Constant
6917
6918 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6919 match(Set dst con);
6920 ins_cost(INSN_COST);
6921 format %{ "fmovd $dst, $con"%}
6922 ins_encode %{
6923 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6924 %}
6925
6926 ins_pipe(fp_imm_d);
6927 %}
6928
6929 // Load Double Constant
6930
6931 instruct loadConD(vRegD dst, immD con) %{
6932 match(Set dst con);
6933
6934 ins_cost(INSN_COST * 5);
6935 format %{
6936 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6937 %}
6938
6939 ins_encode %{
6940 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6941 %}
6942
6943 ins_pipe(fp_load_constant_d);
6944 %}
6945
6946 // Store Instructions
6947
6948 // Store Byte
6949 instruct storeB(iRegIorL2I src, memory1 mem)
6950 %{
6951 match(Set mem (StoreB mem src));
6952 predicate(!needs_releasing_store(n));
6953
6954 ins_cost(INSN_COST);
6955 format %{ "strb $src, $mem\t# byte" %}
6956
6957 ins_encode(aarch64_enc_strb(src, mem));
6958
6959 ins_pipe(istore_reg_mem);
6960 %}
6961
6962
6963 instruct storeimmB0(immI0 zero, memory1 mem)
6964 %{
6965 match(Set mem (StoreB mem zero));
6966 predicate(!needs_releasing_store(n));
6967
6968 ins_cost(INSN_COST);
6969 format %{ "strb rscractch2, $mem\t# byte" %}
6970
6971 ins_encode(aarch64_enc_strb0(mem));
6972
6973 ins_pipe(istore_mem);
6974 %}
6975
6976 // Store Char/Short
6977 instruct storeC(iRegIorL2I src, memory2 mem)
6978 %{
6979 match(Set mem (StoreC mem src));
6980 predicate(!needs_releasing_store(n));
6981
6982 ins_cost(INSN_COST);
6983 format %{ "strh $src, $mem\t# short" %}
6984
6985 ins_encode(aarch64_enc_strh(src, mem));
6986
6987 ins_pipe(istore_reg_mem);
6988 %}
6989
6990 instruct storeimmC0(immI0 zero, memory2 mem)
6991 %{
6992 match(Set mem (StoreC mem zero));
6993 predicate(!needs_releasing_store(n));
6994
6995 ins_cost(INSN_COST);
6996 format %{ "strh zr, $mem\t# short" %}
6997
6998 ins_encode(aarch64_enc_strh0(mem));
6999
7000 ins_pipe(istore_mem);
7001 %}
7002
7003 // Store Integer
7004
7005 instruct storeI(iRegIorL2I src, memory4 mem)
7006 %{
7007 match(Set mem(StoreI mem src));
7008 predicate(!needs_releasing_store(n));
7009
7010 ins_cost(INSN_COST);
7011 format %{ "strw $src, $mem\t# int" %}
7012
7013 ins_encode(aarch64_enc_strw(src, mem));
7014
7015 ins_pipe(istore_reg_mem);
7016 %}
7017
7018 instruct storeimmI0(immI0 zero, memory4 mem)
7019 %{
7020 match(Set mem(StoreI mem zero));
7021 predicate(!needs_releasing_store(n));
7022
7023 ins_cost(INSN_COST);
7024 format %{ "strw zr, $mem\t# int" %}
7025
7026 ins_encode(aarch64_enc_strw0(mem));
7027
7028 ins_pipe(istore_mem);
7029 %}
7030
7031 // Store Long (64 bit signed)
7032 instruct storeL(iRegL src, memory8 mem)
7033 %{
7034 match(Set mem (StoreL mem src));
7035 predicate(!needs_releasing_store(n));
7036
7037 ins_cost(INSN_COST);
7038 format %{ "str $src, $mem\t# int" %}
7039
7040 ins_encode(aarch64_enc_str(src, mem));
7041
7042 ins_pipe(istore_reg_mem);
7043 %}
7044
7045 // Store Long (64 bit signed)
7046 instruct storeimmL0(immL0 zero, memory8 mem)
7047 %{
7048 match(Set mem (StoreL mem zero));
7049 predicate(!needs_releasing_store(n));
7050
7051 ins_cost(INSN_COST);
7052 format %{ "str zr, $mem\t# int" %}
7053
7054 ins_encode(aarch64_enc_str0(mem));
7055
7056 ins_pipe(istore_mem);
7057 %}
7058
7059 // Store Pointer
7060 instruct storeP(iRegP src, memory8 mem)
7061 %{
7062 match(Set mem (StoreP mem src));
7063 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7064
7065 ins_cost(INSN_COST);
7066 format %{ "str $src, $mem\t# ptr" %}
7067
7068 ins_encode(aarch64_enc_str(src, mem));
7069
7070 ins_pipe(istore_reg_mem);
7071 %}
7072
7073 // Store Pointer
7074 instruct storeimmP0(immP0 zero, memory8 mem)
7075 %{
7076 match(Set mem (StoreP mem zero));
7077 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7078
7079 ins_cost(INSN_COST);
7080 format %{ "str zr, $mem\t# ptr" %}
7081
7082 ins_encode(aarch64_enc_str0(mem));
7083
7084 ins_pipe(istore_mem);
7085 %}
7086
7087 // Store Compressed Pointer
7088 instruct storeN(iRegN src, memory4 mem)
7089 %{
7090 match(Set mem (StoreN mem src));
7091 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7092
7093 ins_cost(INSN_COST);
7094 format %{ "strw $src, $mem\t# compressed ptr" %}
7095
7096 ins_encode(aarch64_enc_strw(src, mem));
7097
7098 ins_pipe(istore_reg_mem);
7099 %}
7100
7101 instruct storeImmN0(immN0 zero, memory4 mem)
7102 %{
7103 match(Set mem (StoreN mem zero));
7104 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7105
7106 ins_cost(INSN_COST);
7107 format %{ "strw zr, $mem\t# compressed ptr" %}
7108
7109 ins_encode(aarch64_enc_strw0(mem));
7110
7111 ins_pipe(istore_mem);
7112 %}
7113
7114 // Store Float
7115 instruct storeF(vRegF src, memory4 mem)
7116 %{
7117 match(Set mem (StoreF mem src));
7118 predicate(!needs_releasing_store(n));
7119
7120 ins_cost(INSN_COST);
7121 format %{ "strs $src, $mem\t# float" %}
7122
7123 ins_encode( aarch64_enc_strs(src, mem) );
7124
7125 ins_pipe(pipe_class_memory);
7126 %}
7127
7128 // TODO
7129 // implement storeImmF0 and storeFImmPacked
7130
7131 // Store Double
7132 instruct storeD(vRegD src, memory8 mem)
7133 %{
7134 match(Set mem (StoreD mem src));
7135 predicate(!needs_releasing_store(n));
7136
7137 ins_cost(INSN_COST);
7138 format %{ "strd $src, $mem\t# double" %}
7139
7140 ins_encode( aarch64_enc_strd(src, mem) );
7141
7142 ins_pipe(pipe_class_memory);
7143 %}
7144
7145 // Store Compressed Klass Pointer
7146 instruct storeNKlass(iRegN src, memory4 mem)
7147 %{
7148 predicate(!needs_releasing_store(n));
7149 match(Set mem (StoreNKlass mem src));
7150
7151 ins_cost(INSN_COST);
7152 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7153
7154 ins_encode(aarch64_enc_strw(src, mem));
7155
7156 ins_pipe(istore_reg_mem);
7157 %}
7158
7159 // TODO
7160 // implement storeImmD0 and storeDImmPacked
7161
7162 // prefetch instructions
7163 // Must be safe to execute with invalid address (cannot fault).
7164
7165 instruct prefetchalloc( memory8 mem ) %{
7166 match(PrefetchAllocation mem);
7167
7168 ins_cost(INSN_COST);
7169 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7170
7171 ins_encode( aarch64_enc_prefetchw(mem) );
7172
7173 ins_pipe(iload_prefetch);
7174 %}
7175
7176 // ---------------- volatile loads and stores ----------------
7177
7178 // Load Byte (8 bit signed)
7179 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7180 %{
7181 match(Set dst (LoadB mem));
7182
7183 ins_cost(VOLATILE_REF_COST);
7184 format %{ "ldarsb $dst, $mem\t# byte" %}
7185
7186 ins_encode(aarch64_enc_ldarsb(dst, mem));
7187
7188 ins_pipe(pipe_serial);
7189 %}
7190
7191 // Load Byte (8 bit signed) into long
7192 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7193 %{
7194 match(Set dst (ConvI2L (LoadB mem)));
7195
7196 ins_cost(VOLATILE_REF_COST);
7197 format %{ "ldarsb $dst, $mem\t# byte" %}
7198
7199 ins_encode(aarch64_enc_ldarsb(dst, mem));
7200
7201 ins_pipe(pipe_serial);
7202 %}
7203
7204 // Load Byte (8 bit unsigned)
7205 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7206 %{
7207 match(Set dst (LoadUB mem));
7208
7209 ins_cost(VOLATILE_REF_COST);
7210 format %{ "ldarb $dst, $mem\t# byte" %}
7211
7212 ins_encode(aarch64_enc_ldarb(dst, mem));
7213
7214 ins_pipe(pipe_serial);
7215 %}
7216
7217 // Load Byte (8 bit unsigned) into long
7218 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7219 %{
7220 match(Set dst (ConvI2L (LoadUB mem)));
7221
7222 ins_cost(VOLATILE_REF_COST);
7223 format %{ "ldarb $dst, $mem\t# byte" %}
7224
7225 ins_encode(aarch64_enc_ldarb(dst, mem));
7226
7227 ins_pipe(pipe_serial);
7228 %}
7229
7230 // Load Short (16 bit signed)
7231 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7232 %{
7233 match(Set dst (LoadS mem));
7234
7235 ins_cost(VOLATILE_REF_COST);
7236 format %{ "ldarshw $dst, $mem\t# short" %}
7237
7238 ins_encode(aarch64_enc_ldarshw(dst, mem));
7239
7240 ins_pipe(pipe_serial);
7241 %}
7242
7243 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7244 %{
7245 match(Set dst (LoadUS mem));
7246
7247 ins_cost(VOLATILE_REF_COST);
7248 format %{ "ldarhw $dst, $mem\t# short" %}
7249
7250 ins_encode(aarch64_enc_ldarhw(dst, mem));
7251
7252 ins_pipe(pipe_serial);
7253 %}
7254
7255 // Load Short/Char (16 bit unsigned) into long
7256 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7257 %{
7258 match(Set dst (ConvI2L (LoadUS mem)));
7259
7260 ins_cost(VOLATILE_REF_COST);
7261 format %{ "ldarh $dst, $mem\t# short" %}
7262
7263 ins_encode(aarch64_enc_ldarh(dst, mem));
7264
7265 ins_pipe(pipe_serial);
7266 %}
7267
7268 // Load Short/Char (16 bit signed) into long
7269 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7270 %{
7271 match(Set dst (ConvI2L (LoadS mem)));
7272
7273 ins_cost(VOLATILE_REF_COST);
7274 format %{ "ldarh $dst, $mem\t# short" %}
7275
7276 ins_encode(aarch64_enc_ldarsh(dst, mem));
7277
7278 ins_pipe(pipe_serial);
7279 %}
7280
7281 // Load Integer (32 bit signed)
7282 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7283 %{
7284 match(Set dst (LoadI mem));
7285
7286 ins_cost(VOLATILE_REF_COST);
7287 format %{ "ldarw $dst, $mem\t# int" %}
7288
7289 ins_encode(aarch64_enc_ldarw(dst, mem));
7290
7291 ins_pipe(pipe_serial);
7292 %}
7293
7294 // Load Integer (32 bit unsigned) into long
7295 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7296 %{
7297 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7298
7299 ins_cost(VOLATILE_REF_COST);
7300 format %{ "ldarw $dst, $mem\t# int" %}
7301
7302 ins_encode(aarch64_enc_ldarw(dst, mem));
7303
7304 ins_pipe(pipe_serial);
7305 %}
7306
7307 // Load Long (64 bit signed)
7308 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadL mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldar $dst, $mem\t# int" %}
7314
7315 ins_encode(aarch64_enc_ldar(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 // Load Pointer
7321 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (LoadP mem));
7324 predicate(n->as_Load()->barrier_data() == 0);
7325
7326 ins_cost(VOLATILE_REF_COST);
7327 format %{ "ldar $dst, $mem\t# ptr" %}
7328
7329 ins_encode(aarch64_enc_ldar(dst, mem));
7330
7331 ins_pipe(pipe_serial);
7332 %}
7333
7334 // Load Compressed Pointer
7335 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7336 %{
7337 match(Set dst (LoadN mem));
7338 predicate(n->as_Load()->barrier_data() == 0);
7339
7340 ins_cost(VOLATILE_REF_COST);
7341 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7342
7343 ins_encode(aarch64_enc_ldarw(dst, mem));
7344
7345 ins_pipe(pipe_serial);
7346 %}
7347
7348 // Load Float
7349 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7350 %{
7351 match(Set dst (LoadF mem));
7352
7353 ins_cost(VOLATILE_REF_COST);
7354 format %{ "ldars $dst, $mem\t# float" %}
7355
7356 ins_encode( aarch64_enc_fldars(dst, mem) );
7357
7358 ins_pipe(pipe_serial);
7359 %}
7360
7361 // Load Double
7362 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7363 %{
7364 match(Set dst (LoadD mem));
7365
7366 ins_cost(VOLATILE_REF_COST);
7367 format %{ "ldard $dst, $mem\t# double" %}
7368
7369 ins_encode( aarch64_enc_fldard(dst, mem) );
7370
7371 ins_pipe(pipe_serial);
7372 %}
7373
7374 // Store Byte
7375 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7376 %{
7377 match(Set mem (StoreB mem src));
7378
7379 ins_cost(VOLATILE_REF_COST);
7380 format %{ "stlrb $src, $mem\t# byte" %}
7381
7382 ins_encode(aarch64_enc_stlrb(src, mem));
7383
7384 ins_pipe(pipe_class_memory);
7385 %}
7386
7387 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set mem (StoreB mem zero));
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "stlrb zr, $mem\t# byte" %}
7393
7394 ins_encode(aarch64_enc_stlrb0(mem));
7395
7396 ins_pipe(pipe_class_memory);
7397 %}
7398
7399 // Store Char/Short
7400 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set mem (StoreC mem src));
7403
7404 ins_cost(VOLATILE_REF_COST);
7405 format %{ "stlrh $src, $mem\t# short" %}
7406
7407 ins_encode(aarch64_enc_stlrh(src, mem));
7408
7409 ins_pipe(pipe_class_memory);
7410 %}
7411
7412 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7413 %{
7414 match(Set mem (StoreC mem zero));
7415
7416 ins_cost(VOLATILE_REF_COST);
7417 format %{ "stlrh zr, $mem\t# short" %}
7418
7419 ins_encode(aarch64_enc_stlrh0(mem));
7420
7421 ins_pipe(pipe_class_memory);
7422 %}
7423
7424 // Store Integer
7425
7426 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7427 %{
7428 match(Set mem(StoreI mem src));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "stlrw $src, $mem\t# int" %}
7432
7433 ins_encode(aarch64_enc_stlrw(src, mem));
7434
7435 ins_pipe(pipe_class_memory);
7436 %}
7437
7438 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7439 %{
7440 match(Set mem(StoreI mem zero));
7441
7442 ins_cost(VOLATILE_REF_COST);
7443 format %{ "stlrw zr, $mem\t# int" %}
7444
7445 ins_encode(aarch64_enc_stlrw0(mem));
7446
7447 ins_pipe(pipe_class_memory);
7448 %}
7449
7450 // Store Long (64 bit signed)
7451 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7452 %{
7453 match(Set mem (StoreL mem src));
7454
7455 ins_cost(VOLATILE_REF_COST);
7456 format %{ "stlr $src, $mem\t# int" %}
7457
7458 ins_encode(aarch64_enc_stlr(src, mem));
7459
7460 ins_pipe(pipe_class_memory);
7461 %}
7462
7463 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7464 %{
7465 match(Set mem (StoreL mem zero));
7466
7467 ins_cost(VOLATILE_REF_COST);
7468 format %{ "stlr zr, $mem\t# int" %}
7469
7470 ins_encode(aarch64_enc_stlr0(mem));
7471
7472 ins_pipe(pipe_class_memory);
7473 %}
7474
7475 // Store Pointer
7476 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7477 %{
7478 match(Set mem (StoreP mem src));
7479 predicate(n->as_Store()->barrier_data() == 0);
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlr $src, $mem\t# ptr" %}
7483
7484 ins_encode(aarch64_enc_stlr(src, mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7490 %{
7491 match(Set mem (StoreP mem zero));
7492 predicate(n->as_Store()->barrier_data() == 0);
7493
7494 ins_cost(VOLATILE_REF_COST);
7495 format %{ "stlr zr, $mem\t# ptr" %}
7496
7497 ins_encode(aarch64_enc_stlr0(mem));
7498
7499 ins_pipe(pipe_class_memory);
7500 %}
7501
7502 // Store Compressed Pointer
7503 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem (StoreN mem src));
7506 predicate(n->as_Store()->barrier_data() == 0);
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7510
7511 ins_encode(aarch64_enc_stlrw(src, mem));
7512
7513 ins_pipe(pipe_class_memory);
7514 %}
7515
7516 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7517 %{
7518 match(Set mem (StoreN mem zero));
7519 predicate(n->as_Store()->barrier_data() == 0);
7520
7521 ins_cost(VOLATILE_REF_COST);
7522 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7523
7524 ins_encode(aarch64_enc_stlrw0(mem));
7525
7526 ins_pipe(pipe_class_memory);
7527 %}
7528
7529 // Store Float
7530 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7531 %{
7532 match(Set mem (StoreF mem src));
7533
7534 ins_cost(VOLATILE_REF_COST);
7535 format %{ "stlrs $src, $mem\t# float" %}
7536
7537 ins_encode( aarch64_enc_fstlrs(src, mem) );
7538
7539 ins_pipe(pipe_class_memory);
7540 %}
7541
7542 // TODO
7543 // implement storeImmF0 and storeFImmPacked
7544
7545 // Store Double
7546 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7547 %{
7548 match(Set mem (StoreD mem src));
7549
7550 ins_cost(VOLATILE_REF_COST);
7551 format %{ "stlrd $src, $mem\t# double" %}
7552
7553 ins_encode( aarch64_enc_fstlrd(src, mem) );
7554
7555 ins_pipe(pipe_class_memory);
7556 %}
7557
7558 // ---------------- end of volatile loads and stores ----------------
7559
7560 instruct cacheWB(indirect addr)
7561 %{
7562 predicate(VM_Version::supports_data_cache_line_flush());
7563 match(CacheWB addr);
7564
7565 ins_cost(100);
7566 format %{"cache wb $addr" %}
7567 ins_encode %{
7568 assert($addr->index_position() < 0, "should be");
7569 assert($addr$$disp == 0, "should be");
7570 __ cache_wb(Address($addr$$base$$Register, 0));
7571 %}
7572 ins_pipe(pipe_slow); // XXX
7573 %}
7574
7575 instruct cacheWBPreSync()
7576 %{
7577 predicate(VM_Version::supports_data_cache_line_flush());
7578 match(CacheWBPreSync);
7579
7580 ins_cost(100);
7581 format %{"cache wb presync" %}
7582 ins_encode %{
7583 __ cache_wbsync(true);
7584 %}
7585 ins_pipe(pipe_slow); // XXX
7586 %}
7587
7588 instruct cacheWBPostSync()
7589 %{
7590 predicate(VM_Version::supports_data_cache_line_flush());
7591 match(CacheWBPostSync);
7592
7593 ins_cost(100);
7594 format %{"cache wb postsync" %}
7595 ins_encode %{
7596 __ cache_wbsync(false);
7597 %}
7598 ins_pipe(pipe_slow); // XXX
7599 %}
7600
7601 // ============================================================================
7602 // BSWAP Instructions
7603
7604 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7605 match(Set dst (ReverseBytesI src));
7606
7607 ins_cost(INSN_COST);
7608 format %{ "revw $dst, $src" %}
7609
7610 ins_encode %{
7611 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7612 %}
7613
7614 ins_pipe(ialu_reg);
7615 %}
7616
7617 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7618 match(Set dst (ReverseBytesL src));
7619
7620 ins_cost(INSN_COST);
7621 format %{ "rev $dst, $src" %}
7622
7623 ins_encode %{
7624 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7625 %}
7626
7627 ins_pipe(ialu_reg);
7628 %}
7629
7630 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7631 match(Set dst (ReverseBytesUS src));
7632
7633 ins_cost(INSN_COST);
7634 format %{ "rev16w $dst, $src" %}
7635
7636 ins_encode %{
7637 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7638 %}
7639
7640 ins_pipe(ialu_reg);
7641 %}
7642
7643 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7644 match(Set dst (ReverseBytesS src));
7645
7646 ins_cost(INSN_COST);
7647 format %{ "rev16w $dst, $src\n\t"
7648 "sbfmw $dst, $dst, #0, #15" %}
7649
7650 ins_encode %{
7651 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7652 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7653 %}
7654
7655 ins_pipe(ialu_reg);
7656 %}
7657
7658 // ============================================================================
7659 // Zero Count Instructions
7660
7661 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7662 match(Set dst (CountLeadingZerosI src));
7663
7664 ins_cost(INSN_COST);
7665 format %{ "clzw $dst, $src" %}
7666 ins_encode %{
7667 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7668 %}
7669
7670 ins_pipe(ialu_reg);
7671 %}
7672
7673 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7674 match(Set dst (CountLeadingZerosL src));
7675
7676 ins_cost(INSN_COST);
7677 format %{ "clz $dst, $src" %}
7678 ins_encode %{
7679 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7680 %}
7681
7682 ins_pipe(ialu_reg);
7683 %}
7684
7685 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7686 match(Set dst (CountTrailingZerosI src));
7687
7688 ins_cost(INSN_COST * 2);
7689 format %{ "rbitw $dst, $src\n\t"
7690 "clzw $dst, $dst" %}
7691 ins_encode %{
7692 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7693 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7694 %}
7695
7696 ins_pipe(ialu_reg);
7697 %}
7698
7699 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7700 match(Set dst (CountTrailingZerosL src));
7701
7702 ins_cost(INSN_COST * 2);
7703 format %{ "rbit $dst, $src\n\t"
7704 "clz $dst, $dst" %}
7705 ins_encode %{
7706 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7707 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7708 %}
7709
7710 ins_pipe(ialu_reg);
7711 %}
7712
7713 //---------- Population Count Instructions -------------------------------------
7714 //
7715
7716 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7717 match(Set dst (PopCountI src));
7718 effect(TEMP tmp);
7719 ins_cost(INSN_COST * 13);
7720
7721 format %{ "movw $src, $src\n\t"
7722 "mov $tmp, $src\t# vector (1D)\n\t"
7723 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7724 "addv $tmp, $tmp\t# vector (8B)\n\t"
7725 "mov $dst, $tmp\t# vector (1D)" %}
7726 ins_encode %{
7727 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7728 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7729 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7730 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7731 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7732 %}
7733
7734 ins_pipe(pipe_class_default);
7735 %}
7736
7737 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7738 match(Set dst (PopCountI (LoadI mem)));
7739 effect(TEMP tmp);
7740 ins_cost(INSN_COST * 13);
7741
7742 format %{ "ldrs $tmp, $mem\n\t"
7743 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7744 "addv $tmp, $tmp\t# vector (8B)\n\t"
7745 "mov $dst, $tmp\t# vector (1D)" %}
7746 ins_encode %{
7747 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7748 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7749 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7750 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7751 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7752 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7753 %}
7754
7755 ins_pipe(pipe_class_default);
7756 %}
7757
7758 // Note: Long.bitCount(long) returns an int.
7759 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7760 match(Set dst (PopCountL src));
7761 effect(TEMP tmp);
7762 ins_cost(INSN_COST * 13);
7763
7764 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7765 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7766 "addv $tmp, $tmp\t# vector (8B)\n\t"
7767 "mov $dst, $tmp\t# vector (1D)" %}
7768 ins_encode %{
7769 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7770 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7771 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7772 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7773 %}
7774
7775 ins_pipe(pipe_class_default);
7776 %}
7777
7778 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7779 match(Set dst (PopCountL (LoadL mem)));
7780 effect(TEMP tmp);
7781 ins_cost(INSN_COST * 13);
7782
7783 format %{ "ldrd $tmp, $mem\n\t"
7784 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7785 "addv $tmp, $tmp\t# vector (8B)\n\t"
7786 "mov $dst, $tmp\t# vector (1D)" %}
7787 ins_encode %{
7788 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7789 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7790 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7791 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7792 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7793 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7794 %}
7795
7796 ins_pipe(pipe_class_default);
7797 %}
7798
7799 // ============================================================================
7800 // VerifyVectorAlignment Instruction
7801
7802 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7803 match(Set addr (VerifyVectorAlignment addr mask));
7804 effect(KILL cr);
7805 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7806 ins_encode %{
7807 Label Lskip;
7808 // check if masked bits of addr are zero
7809 __ tst($addr$$Register, $mask$$constant);
7810 __ br(Assembler::EQ, Lskip);
7811 __ stop("verify_vector_alignment found a misaligned vector memory access");
7812 __ bind(Lskip);
7813 %}
7814 ins_pipe(pipe_slow);
7815 %}
7816
7817 // ============================================================================
7818 // MemBar Instruction
7819
7820 instruct load_fence() %{
7821 match(LoadFence);
7822 ins_cost(VOLATILE_REF_COST);
7823
7824 format %{ "load_fence" %}
7825
7826 ins_encode %{
7827 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7828 %}
7829 ins_pipe(pipe_serial);
7830 %}
7831
7832 instruct unnecessary_membar_acquire() %{
7833 predicate(unnecessary_acquire(n));
7834 match(MemBarAcquire);
7835 ins_cost(0);
7836
7837 format %{ "membar_acquire (elided)" %}
7838
7839 ins_encode %{
7840 __ block_comment("membar_acquire (elided)");
7841 %}
7842
7843 ins_pipe(pipe_class_empty);
7844 %}
7845
7846 instruct membar_acquire() %{
7847 match(MemBarAcquire);
7848 ins_cost(VOLATILE_REF_COST);
7849
7850 format %{ "membar_acquire\n\t"
7851 "dmb ishld" %}
7852
7853 ins_encode %{
7854 __ block_comment("membar_acquire");
7855 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7856 %}
7857
7858 ins_pipe(pipe_serial);
7859 %}
7860
7861
7862 instruct membar_acquire_lock() %{
7863 match(MemBarAcquireLock);
7864 ins_cost(VOLATILE_REF_COST);
7865
7866 format %{ "membar_acquire_lock (elided)" %}
7867
7868 ins_encode %{
7869 __ block_comment("membar_acquire_lock (elided)");
7870 %}
7871
7872 ins_pipe(pipe_serial);
7873 %}
7874
7875 instruct store_fence() %{
7876 match(StoreFence);
7877 ins_cost(VOLATILE_REF_COST);
7878
7879 format %{ "store_fence" %}
7880
7881 ins_encode %{
7882 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7883 %}
7884 ins_pipe(pipe_serial);
7885 %}
7886
7887 instruct unnecessary_membar_release() %{
7888 predicate(unnecessary_release(n));
7889 match(MemBarRelease);
7890 ins_cost(0);
7891
7892 format %{ "membar_release (elided)" %}
7893
7894 ins_encode %{
7895 __ block_comment("membar_release (elided)");
7896 %}
7897 ins_pipe(pipe_serial);
7898 %}
7899
7900 instruct membar_release() %{
7901 match(MemBarRelease);
7902 ins_cost(VOLATILE_REF_COST);
7903
7904 format %{ "membar_release\n\t"
7905 "dmb ishst\n\tdmb ishld" %}
7906
7907 ins_encode %{
7908 __ block_comment("membar_release");
7909 // These will be merged if AlwaysMergeDMB is enabled.
7910 __ membar(Assembler::StoreStore);
7911 __ membar(Assembler::LoadStore);
7912 %}
7913 ins_pipe(pipe_serial);
7914 %}
7915
7916 instruct membar_storestore() %{
7917 match(MemBarStoreStore);
7918 match(StoreStoreFence);
7919 ins_cost(VOLATILE_REF_COST);
7920
7921 format %{ "MEMBAR-store-store" %}
7922
7923 ins_encode %{
7924 __ membar(Assembler::StoreStore);
7925 %}
7926 ins_pipe(pipe_serial);
7927 %}
7928
7929 instruct membar_release_lock() %{
7930 match(MemBarReleaseLock);
7931 ins_cost(VOLATILE_REF_COST);
7932
7933 format %{ "membar_release_lock (elided)" %}
7934
7935 ins_encode %{
7936 __ block_comment("membar_release_lock (elided)");
7937 %}
7938
7939 ins_pipe(pipe_serial);
7940 %}
7941
7942 instruct unnecessary_membar_volatile() %{
7943 predicate(unnecessary_volatile(n));
7944 match(MemBarVolatile);
7945 ins_cost(0);
7946
7947 format %{ "membar_volatile (elided)" %}
7948
7949 ins_encode %{
7950 __ block_comment("membar_volatile (elided)");
7951 %}
7952
7953 ins_pipe(pipe_serial);
7954 %}
7955
7956 instruct membar_volatile() %{
7957 match(MemBarVolatile);
7958 ins_cost(VOLATILE_REF_COST*100);
7959
7960 format %{ "membar_volatile\n\t"
7961 "dmb ish"%}
7962
7963 ins_encode %{
7964 __ block_comment("membar_volatile");
7965 __ membar(Assembler::StoreLoad);
7966 %}
7967
7968 ins_pipe(pipe_serial);
7969 %}
7970
7971 // ============================================================================
7972 // Cast/Convert Instructions
7973
7974 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7975 match(Set dst (CastX2P src));
7976
7977 ins_cost(INSN_COST);
7978 format %{ "mov $dst, $src\t# long -> ptr" %}
7979
7980 ins_encode %{
7981 if ($dst$$reg != $src$$reg) {
7982 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7983 }
7984 %}
7985
7986 ins_pipe(ialu_reg);
7987 %}
7988
7989 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7990 match(Set dst (CastP2X src));
7991
7992 ins_cost(INSN_COST);
7993 format %{ "mov $dst, $src\t# ptr -> long" %}
7994
7995 ins_encode %{
7996 if ($dst$$reg != $src$$reg) {
7997 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7998 }
7999 %}
8000
8001 ins_pipe(ialu_reg);
8002 %}
8003
8004 // Convert oop into int for vectors alignment masking
8005 instruct convP2I(iRegINoSp dst, iRegP src) %{
8006 match(Set dst (ConvL2I (CastP2X src)));
8007
8008 ins_cost(INSN_COST);
8009 format %{ "movw $dst, $src\t# ptr -> int" %}
8010 ins_encode %{
8011 __ movw($dst$$Register, $src$$Register);
8012 %}
8013
8014 ins_pipe(ialu_reg);
8015 %}
8016
8017 // Convert compressed oop into int for vectors alignment masking
8018 // in case of 32bit oops (heap < 4Gb).
8019 instruct convN2I(iRegINoSp dst, iRegN src)
8020 %{
8021 predicate(CompressedOops::shift() == 0);
8022 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8023
8024 ins_cost(INSN_COST);
8025 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8026 ins_encode %{
8027 __ movw($dst$$Register, $src$$Register);
8028 %}
8029
8030 ins_pipe(ialu_reg);
8031 %}
8032
8033
8034 // Convert oop pointer into compressed form
8035 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8036 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8037 match(Set dst (EncodeP src));
8038 effect(KILL cr);
8039 ins_cost(INSN_COST * 3);
8040 format %{ "encode_heap_oop $dst, $src" %}
8041 ins_encode %{
8042 Register s = $src$$Register;
8043 Register d = $dst$$Register;
8044 __ encode_heap_oop(d, s);
8045 %}
8046 ins_pipe(ialu_reg);
8047 %}
8048
8049 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8050 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8051 match(Set dst (EncodeP src));
8052 ins_cost(INSN_COST * 3);
8053 format %{ "encode_heap_oop_not_null $dst, $src" %}
8054 ins_encode %{
8055 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8056 %}
8057 ins_pipe(ialu_reg);
8058 %}
8059
8060 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8061 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8062 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8063 match(Set dst (DecodeN src));
8064 ins_cost(INSN_COST * 3);
8065 format %{ "decode_heap_oop $dst, $src" %}
8066 ins_encode %{
8067 Register s = $src$$Register;
8068 Register d = $dst$$Register;
8069 __ decode_heap_oop(d, s);
8070 %}
8071 ins_pipe(ialu_reg);
8072 %}
8073
8074 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8075 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8076 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8077 match(Set dst (DecodeN src));
8078 ins_cost(INSN_COST * 3);
8079 format %{ "decode_heap_oop_not_null $dst, $src" %}
8080 ins_encode %{
8081 Register s = $src$$Register;
8082 Register d = $dst$$Register;
8083 __ decode_heap_oop_not_null(d, s);
8084 %}
8085 ins_pipe(ialu_reg);
8086 %}
8087
8088 // n.b. AArch64 implementations of encode_klass_not_null and
8089 // decode_klass_not_null do not modify the flags register so, unlike
8090 // Intel, we don't kill CR as a side effect here
8091
8092 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8093 match(Set dst (EncodePKlass src));
8094
8095 ins_cost(INSN_COST * 3);
8096 format %{ "encode_klass_not_null $dst,$src" %}
8097
8098 ins_encode %{
8099 Register src_reg = as_Register($src$$reg);
8100 Register dst_reg = as_Register($dst$$reg);
8101 __ encode_klass_not_null(dst_reg, src_reg);
8102 %}
8103
8104 ins_pipe(ialu_reg);
8105 %}
8106
8107 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8108 match(Set dst (DecodeNKlass src));
8109
8110 ins_cost(INSN_COST * 3);
8111 format %{ "decode_klass_not_null $dst,$src" %}
8112
8113 ins_encode %{
8114 Register src_reg = as_Register($src$$reg);
8115 Register dst_reg = as_Register($dst$$reg);
8116 if (dst_reg != src_reg) {
8117 __ decode_klass_not_null(dst_reg, src_reg);
8118 } else {
8119 __ decode_klass_not_null(dst_reg);
8120 }
8121 %}
8122
8123 ins_pipe(ialu_reg);
8124 %}
8125
8126 instruct checkCastPP(iRegPNoSp dst)
8127 %{
8128 match(Set dst (CheckCastPP dst));
8129
8130 size(0);
8131 format %{ "# checkcastPP of $dst" %}
8132 ins_encode(/* empty encoding */);
8133 ins_pipe(pipe_class_empty);
8134 %}
8135
8136 instruct castPP(iRegPNoSp dst)
8137 %{
8138 match(Set dst (CastPP dst));
8139
8140 size(0);
8141 format %{ "# castPP of $dst" %}
8142 ins_encode(/* empty encoding */);
8143 ins_pipe(pipe_class_empty);
8144 %}
8145
8146 instruct castII(iRegI dst)
8147 %{
8148 match(Set dst (CastII dst));
8149
8150 size(0);
8151 format %{ "# castII of $dst" %}
8152 ins_encode(/* empty encoding */);
8153 ins_cost(0);
8154 ins_pipe(pipe_class_empty);
8155 %}
8156
8157 instruct castLL(iRegL dst)
8158 %{
8159 match(Set dst (CastLL dst));
8160
8161 size(0);
8162 format %{ "# castLL of $dst" %}
8163 ins_encode(/* empty encoding */);
8164 ins_cost(0);
8165 ins_pipe(pipe_class_empty);
8166 %}
8167
8168 instruct castFF(vRegF dst)
8169 %{
8170 match(Set dst (CastFF dst));
8171
8172 size(0);
8173 format %{ "# castFF of $dst" %}
8174 ins_encode(/* empty encoding */);
8175 ins_cost(0);
8176 ins_pipe(pipe_class_empty);
8177 %}
8178
8179 instruct castDD(vRegD dst)
8180 %{
8181 match(Set dst (CastDD dst));
8182
8183 size(0);
8184 format %{ "# castDD of $dst" %}
8185 ins_encode(/* empty encoding */);
8186 ins_cost(0);
8187 ins_pipe(pipe_class_empty);
8188 %}
8189
8190 instruct castVV(vReg dst)
8191 %{
8192 match(Set dst (CastVV dst));
8193
8194 size(0);
8195 format %{ "# castVV of $dst" %}
8196 ins_encode(/* empty encoding */);
8197 ins_cost(0);
8198 ins_pipe(pipe_class_empty);
8199 %}
8200
8201 instruct castVVMask(pRegGov dst)
8202 %{
8203 match(Set dst (CastVV dst));
8204
8205 size(0);
8206 format %{ "# castVV of $dst" %}
8207 ins_encode(/* empty encoding */);
8208 ins_cost(0);
8209 ins_pipe(pipe_class_empty);
8210 %}
8211
8212 // ============================================================================
8213 // Atomic operation instructions
8214 //
8215
8216 // standard CompareAndSwapX when we are using barriers
8217 // these have higher priority than the rules selected by a predicate
8218
8219 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8220 // can't match them
8221
8222 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8223
8224 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8225 ins_cost(2 * VOLATILE_REF_COST);
8226
8227 effect(KILL cr);
8228
8229 format %{
8230 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8231 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8232 %}
8233
8234 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8235 aarch64_enc_cset_eq(res));
8236
8237 ins_pipe(pipe_slow);
8238 %}
8239
8240 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8241
8242 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8243 ins_cost(2 * VOLATILE_REF_COST);
8244
8245 effect(KILL cr);
8246
8247 format %{
8248 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8249 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8250 %}
8251
8252 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8253 aarch64_enc_cset_eq(res));
8254
8255 ins_pipe(pipe_slow);
8256 %}
8257
8258 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8259
8260 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8261 ins_cost(2 * VOLATILE_REF_COST);
8262
8263 effect(KILL cr);
8264
8265 format %{
8266 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8267 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8268 %}
8269
8270 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8271 aarch64_enc_cset_eq(res));
8272
8273 ins_pipe(pipe_slow);
8274 %}
8275
8276 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8277
8278 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8279 ins_cost(2 * VOLATILE_REF_COST);
8280
8281 effect(KILL cr);
8282
8283 format %{
8284 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8285 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8286 %}
8287
8288 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8289 aarch64_enc_cset_eq(res));
8290
8291 ins_pipe(pipe_slow);
8292 %}
8293
8294 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8295
8296 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8297 predicate(n->as_LoadStore()->barrier_data() == 0);
8298 ins_cost(2 * VOLATILE_REF_COST);
8299
8300 effect(KILL cr);
8301
8302 format %{
8303 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8304 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8305 %}
8306
8307 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8308 aarch64_enc_cset_eq(res));
8309
8310 ins_pipe(pipe_slow);
8311 %}
8312
8313 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8314
8315 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8316 predicate(n->as_LoadStore()->barrier_data() == 0);
8317 ins_cost(2 * VOLATILE_REF_COST);
8318
8319 effect(KILL cr);
8320
8321 format %{
8322 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8323 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8324 %}
8325
8326 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8327 aarch64_enc_cset_eq(res));
8328
8329 ins_pipe(pipe_slow);
8330 %}
8331
8332 // alternative CompareAndSwapX when we are eliding barriers
8333
8334 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8335
8336 predicate(needs_acquiring_load_exclusive(n));
8337 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8338 ins_cost(VOLATILE_REF_COST);
8339
8340 effect(KILL cr);
8341
8342 format %{
8343 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8344 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8345 %}
8346
8347 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8348 aarch64_enc_cset_eq(res));
8349
8350 ins_pipe(pipe_slow);
8351 %}
8352
8353 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8354
8355 predicate(needs_acquiring_load_exclusive(n));
8356 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8357 ins_cost(VOLATILE_REF_COST);
8358
8359 effect(KILL cr);
8360
8361 format %{
8362 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8363 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8364 %}
8365
8366 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8367 aarch64_enc_cset_eq(res));
8368
8369 ins_pipe(pipe_slow);
8370 %}
8371
8372 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8373
8374 predicate(needs_acquiring_load_exclusive(n));
8375 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8376 ins_cost(VOLATILE_REF_COST);
8377
8378 effect(KILL cr);
8379
8380 format %{
8381 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8382 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8383 %}
8384
8385 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8386 aarch64_enc_cset_eq(res));
8387
8388 ins_pipe(pipe_slow);
8389 %}
8390
8391 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8392
8393 predicate(needs_acquiring_load_exclusive(n));
8394 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8395 ins_cost(VOLATILE_REF_COST);
8396
8397 effect(KILL cr);
8398
8399 format %{
8400 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8401 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8402 %}
8403
8404 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8405 aarch64_enc_cset_eq(res));
8406
8407 ins_pipe(pipe_slow);
8408 %}
8409
8410 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8411
8412 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8413 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8414 ins_cost(VOLATILE_REF_COST);
8415
8416 effect(KILL cr);
8417
8418 format %{
8419 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8420 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8421 %}
8422
8423 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8424 aarch64_enc_cset_eq(res));
8425
8426 ins_pipe(pipe_slow);
8427 %}
8428
8429 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8430
8431 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8432 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8433 ins_cost(VOLATILE_REF_COST);
8434
8435 effect(KILL cr);
8436
8437 format %{
8438 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8439 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8440 %}
8441
8442 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8443 aarch64_enc_cset_eq(res));
8444
8445 ins_pipe(pipe_slow);
8446 %}
8447
8448
8449 // ---------------------------------------------------------------------
8450
8451 // BEGIN This section of the file is automatically generated. Do not edit --------------
8452
8453 // Sundry CAS operations. Note that release is always true,
8454 // regardless of the memory ordering of the CAS. This is because we
8455 // need the volatile case to be sequentially consistent but there is
8456 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8457 // can't check the type of memory ordering here, so we always emit a
8458 // STLXR.
8459
8460 // This section is generated from cas.m4
8461
8462
8463 // This pattern is generated automatically from cas.m4.
8464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8465 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8466 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8467 ins_cost(2 * VOLATILE_REF_COST);
8468 effect(TEMP_DEF res, KILL cr);
8469 format %{
8470 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8471 %}
8472 ins_encode %{
8473 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8474 Assembler::byte, /*acquire*/ false, /*release*/ true,
8475 /*weak*/ false, $res$$Register);
8476 __ sxtbw($res$$Register, $res$$Register);
8477 %}
8478 ins_pipe(pipe_slow);
8479 %}
8480
8481 // This pattern is generated automatically from cas.m4.
8482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8483 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8484 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8485 ins_cost(2 * VOLATILE_REF_COST);
8486 effect(TEMP_DEF res, KILL cr);
8487 format %{
8488 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8489 %}
8490 ins_encode %{
8491 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8492 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8493 /*weak*/ false, $res$$Register);
8494 __ sxthw($res$$Register, $res$$Register);
8495 %}
8496 ins_pipe(pipe_slow);
8497 %}
8498
8499 // This pattern is generated automatically from cas.m4.
8500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8501 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8502 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8503 ins_cost(2 * VOLATILE_REF_COST);
8504 effect(TEMP_DEF res, KILL cr);
8505 format %{
8506 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8507 %}
8508 ins_encode %{
8509 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8510 Assembler::word, /*acquire*/ false, /*release*/ true,
8511 /*weak*/ false, $res$$Register);
8512 %}
8513 ins_pipe(pipe_slow);
8514 %}
8515
8516 // This pattern is generated automatically from cas.m4.
8517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8518 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8519 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8520 ins_cost(2 * VOLATILE_REF_COST);
8521 effect(TEMP_DEF res, KILL cr);
8522 format %{
8523 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8524 %}
8525 ins_encode %{
8526 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8527 Assembler::xword, /*acquire*/ false, /*release*/ true,
8528 /*weak*/ false, $res$$Register);
8529 %}
8530 ins_pipe(pipe_slow);
8531 %}
8532
8533 // This pattern is generated automatically from cas.m4.
8534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8535 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8536 predicate(n->as_LoadStore()->barrier_data() == 0);
8537 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8538 ins_cost(2 * VOLATILE_REF_COST);
8539 effect(TEMP_DEF res, KILL cr);
8540 format %{
8541 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8542 %}
8543 ins_encode %{
8544 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8545 Assembler::word, /*acquire*/ false, /*release*/ true,
8546 /*weak*/ false, $res$$Register);
8547 %}
8548 ins_pipe(pipe_slow);
8549 %}
8550
8551 // This pattern is generated automatically from cas.m4.
8552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8553 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8554 predicate(n->as_LoadStore()->barrier_data() == 0);
8555 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8556 ins_cost(2 * VOLATILE_REF_COST);
8557 effect(TEMP_DEF res, KILL cr);
8558 format %{
8559 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8560 %}
8561 ins_encode %{
8562 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8563 Assembler::xword, /*acquire*/ false, /*release*/ true,
8564 /*weak*/ false, $res$$Register);
8565 %}
8566 ins_pipe(pipe_slow);
8567 %}
8568
8569 // This pattern is generated automatically from cas.m4.
8570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8571 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8572 predicate(needs_acquiring_load_exclusive(n));
8573 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8574 ins_cost(VOLATILE_REF_COST);
8575 effect(TEMP_DEF res, KILL cr);
8576 format %{
8577 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8578 %}
8579 ins_encode %{
8580 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8581 Assembler::byte, /*acquire*/ true, /*release*/ true,
8582 /*weak*/ false, $res$$Register);
8583 __ sxtbw($res$$Register, $res$$Register);
8584 %}
8585 ins_pipe(pipe_slow);
8586 %}
8587
8588 // This pattern is generated automatically from cas.m4.
8589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8590 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8591 predicate(needs_acquiring_load_exclusive(n));
8592 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8593 ins_cost(VOLATILE_REF_COST);
8594 effect(TEMP_DEF res, KILL cr);
8595 format %{
8596 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8597 %}
8598 ins_encode %{
8599 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8600 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8601 /*weak*/ false, $res$$Register);
8602 __ sxthw($res$$Register, $res$$Register);
8603 %}
8604 ins_pipe(pipe_slow);
8605 %}
8606
8607 // This pattern is generated automatically from cas.m4.
8608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8609 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8610 predicate(needs_acquiring_load_exclusive(n));
8611 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8612 ins_cost(VOLATILE_REF_COST);
8613 effect(TEMP_DEF res, KILL cr);
8614 format %{
8615 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8616 %}
8617 ins_encode %{
8618 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8619 Assembler::word, /*acquire*/ true, /*release*/ true,
8620 /*weak*/ false, $res$$Register);
8621 %}
8622 ins_pipe(pipe_slow);
8623 %}
8624
8625 // This pattern is generated automatically from cas.m4.
8626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8627 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8628 predicate(needs_acquiring_load_exclusive(n));
8629 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8630 ins_cost(VOLATILE_REF_COST);
8631 effect(TEMP_DEF res, KILL cr);
8632 format %{
8633 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8634 %}
8635 ins_encode %{
8636 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8637 Assembler::xword, /*acquire*/ true, /*release*/ true,
8638 /*weak*/ false, $res$$Register);
8639 %}
8640 ins_pipe(pipe_slow);
8641 %}
8642
8643 // This pattern is generated automatically from cas.m4.
8644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8645 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8646 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8647 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8648 ins_cost(VOLATILE_REF_COST);
8649 effect(TEMP_DEF res, KILL cr);
8650 format %{
8651 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8652 %}
8653 ins_encode %{
8654 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8655 Assembler::word, /*acquire*/ true, /*release*/ true,
8656 /*weak*/ false, $res$$Register);
8657 %}
8658 ins_pipe(pipe_slow);
8659 %}
8660
8661 // This pattern is generated automatically from cas.m4.
8662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8663 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8664 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8665 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8666 ins_cost(VOLATILE_REF_COST);
8667 effect(TEMP_DEF res, KILL cr);
8668 format %{
8669 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8670 %}
8671 ins_encode %{
8672 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8673 Assembler::xword, /*acquire*/ true, /*release*/ true,
8674 /*weak*/ false, $res$$Register);
8675 %}
8676 ins_pipe(pipe_slow);
8677 %}
8678
8679 // This pattern is generated automatically from cas.m4.
8680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8681 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8682 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8683 ins_cost(2 * VOLATILE_REF_COST);
8684 effect(KILL cr);
8685 format %{
8686 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8687 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8688 %}
8689 ins_encode %{
8690 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8691 Assembler::byte, /*acquire*/ false, /*release*/ true,
8692 /*weak*/ true, noreg);
8693 __ csetw($res$$Register, Assembler::EQ);
8694 %}
8695 ins_pipe(pipe_slow);
8696 %}
8697
8698 // This pattern is generated automatically from cas.m4.
8699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8700 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8701 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8702 ins_cost(2 * VOLATILE_REF_COST);
8703 effect(KILL cr);
8704 format %{
8705 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8706 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8707 %}
8708 ins_encode %{
8709 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8710 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8711 /*weak*/ true, noreg);
8712 __ csetw($res$$Register, Assembler::EQ);
8713 %}
8714 ins_pipe(pipe_slow);
8715 %}
8716
8717 // This pattern is generated automatically from cas.m4.
8718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8719 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8720 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8721 ins_cost(2 * VOLATILE_REF_COST);
8722 effect(KILL cr);
8723 format %{
8724 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8725 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8726 %}
8727 ins_encode %{
8728 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8729 Assembler::word, /*acquire*/ false, /*release*/ true,
8730 /*weak*/ true, noreg);
8731 __ csetw($res$$Register, Assembler::EQ);
8732 %}
8733 ins_pipe(pipe_slow);
8734 %}
8735
8736 // This pattern is generated automatically from cas.m4.
8737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8738 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8739 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8740 ins_cost(2 * VOLATILE_REF_COST);
8741 effect(KILL cr);
8742 format %{
8743 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8744 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8745 %}
8746 ins_encode %{
8747 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8748 Assembler::xword, /*acquire*/ false, /*release*/ true,
8749 /*weak*/ true, noreg);
8750 __ csetw($res$$Register, Assembler::EQ);
8751 %}
8752 ins_pipe(pipe_slow);
8753 %}
8754
8755 // This pattern is generated automatically from cas.m4.
8756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8757 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8758 predicate(n->as_LoadStore()->barrier_data() == 0);
8759 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8760 ins_cost(2 * VOLATILE_REF_COST);
8761 effect(KILL cr);
8762 format %{
8763 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8764 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8765 %}
8766 ins_encode %{
8767 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8768 Assembler::word, /*acquire*/ false, /*release*/ true,
8769 /*weak*/ true, noreg);
8770 __ csetw($res$$Register, Assembler::EQ);
8771 %}
8772 ins_pipe(pipe_slow);
8773 %}
8774
8775 // This pattern is generated automatically from cas.m4.
8776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8777 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8778 predicate(n->as_LoadStore()->barrier_data() == 0);
8779 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8780 ins_cost(2 * VOLATILE_REF_COST);
8781 effect(KILL cr);
8782 format %{
8783 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8784 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8785 %}
8786 ins_encode %{
8787 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8788 Assembler::xword, /*acquire*/ false, /*release*/ true,
8789 /*weak*/ true, noreg);
8790 __ csetw($res$$Register, Assembler::EQ);
8791 %}
8792 ins_pipe(pipe_slow);
8793 %}
8794
8795 // This pattern is generated automatically from cas.m4.
8796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8797 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8798 predicate(needs_acquiring_load_exclusive(n));
8799 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8800 ins_cost(VOLATILE_REF_COST);
8801 effect(KILL cr);
8802 format %{
8803 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8804 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8805 %}
8806 ins_encode %{
8807 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8808 Assembler::byte, /*acquire*/ true, /*release*/ true,
8809 /*weak*/ true, noreg);
8810 __ csetw($res$$Register, Assembler::EQ);
8811 %}
8812 ins_pipe(pipe_slow);
8813 %}
8814
8815 // This pattern is generated automatically from cas.m4.
8816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8817 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8818 predicate(needs_acquiring_load_exclusive(n));
8819 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8820 ins_cost(VOLATILE_REF_COST);
8821 effect(KILL cr);
8822 format %{
8823 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8824 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8825 %}
8826 ins_encode %{
8827 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8828 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8829 /*weak*/ true, noreg);
8830 __ csetw($res$$Register, Assembler::EQ);
8831 %}
8832 ins_pipe(pipe_slow);
8833 %}
8834
8835 // This pattern is generated automatically from cas.m4.
8836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8837 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8838 predicate(needs_acquiring_load_exclusive(n));
8839 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8840 ins_cost(VOLATILE_REF_COST);
8841 effect(KILL cr);
8842 format %{
8843 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8844 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8845 %}
8846 ins_encode %{
8847 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8848 Assembler::word, /*acquire*/ true, /*release*/ true,
8849 /*weak*/ true, noreg);
8850 __ csetw($res$$Register, Assembler::EQ);
8851 %}
8852 ins_pipe(pipe_slow);
8853 %}
8854
8855 // This pattern is generated automatically from cas.m4.
8856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8857 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8858 predicate(needs_acquiring_load_exclusive(n));
8859 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8860 ins_cost(VOLATILE_REF_COST);
8861 effect(KILL cr);
8862 format %{
8863 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8864 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8865 %}
8866 ins_encode %{
8867 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8868 Assembler::xword, /*acquire*/ true, /*release*/ true,
8869 /*weak*/ true, noreg);
8870 __ csetw($res$$Register, Assembler::EQ);
8871 %}
8872 ins_pipe(pipe_slow);
8873 %}
8874
8875 // This pattern is generated automatically from cas.m4.
8876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8877 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8878 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8879 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8880 ins_cost(VOLATILE_REF_COST);
8881 effect(KILL cr);
8882 format %{
8883 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8884 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8885 %}
8886 ins_encode %{
8887 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8888 Assembler::word, /*acquire*/ true, /*release*/ true,
8889 /*weak*/ true, noreg);
8890 __ csetw($res$$Register, Assembler::EQ);
8891 %}
8892 ins_pipe(pipe_slow);
8893 %}
8894
8895 // This pattern is generated automatically from cas.m4.
8896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8897 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8898 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8899 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8900 ins_cost(VOLATILE_REF_COST);
8901 effect(KILL cr);
8902 format %{
8903 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8904 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8905 %}
8906 ins_encode %{
8907 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8908 Assembler::xword, /*acquire*/ true, /*release*/ true,
8909 /*weak*/ true, noreg);
8910 __ csetw($res$$Register, Assembler::EQ);
8911 %}
8912 ins_pipe(pipe_slow);
8913 %}
8914
8915 // END This section of the file is automatically generated. Do not edit --------------
8916 // ---------------------------------------------------------------------
8917
8918 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8919 match(Set prev (GetAndSetI mem newv));
8920 ins_cost(2 * VOLATILE_REF_COST);
8921 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8922 ins_encode %{
8923 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8924 %}
8925 ins_pipe(pipe_serial);
8926 %}
8927
8928 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8929 match(Set prev (GetAndSetL mem newv));
8930 ins_cost(2 * VOLATILE_REF_COST);
8931 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8932 ins_encode %{
8933 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8934 %}
8935 ins_pipe(pipe_serial);
8936 %}
8937
8938 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8939 predicate(n->as_LoadStore()->barrier_data() == 0);
8940 match(Set prev (GetAndSetN mem newv));
8941 ins_cost(2 * VOLATILE_REF_COST);
8942 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8943 ins_encode %{
8944 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8945 %}
8946 ins_pipe(pipe_serial);
8947 %}
8948
8949 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8950 predicate(n->as_LoadStore()->barrier_data() == 0);
8951 match(Set prev (GetAndSetP mem newv));
8952 ins_cost(2 * VOLATILE_REF_COST);
8953 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8954 ins_encode %{
8955 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8956 %}
8957 ins_pipe(pipe_serial);
8958 %}
8959
8960 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8961 predicate(needs_acquiring_load_exclusive(n));
8962 match(Set prev (GetAndSetI mem newv));
8963 ins_cost(VOLATILE_REF_COST);
8964 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8965 ins_encode %{
8966 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8967 %}
8968 ins_pipe(pipe_serial);
8969 %}
8970
8971 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8972 predicate(needs_acquiring_load_exclusive(n));
8973 match(Set prev (GetAndSetL mem newv));
8974 ins_cost(VOLATILE_REF_COST);
8975 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8976 ins_encode %{
8977 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8978 %}
8979 ins_pipe(pipe_serial);
8980 %}
8981
8982 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8983 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8984 match(Set prev (GetAndSetN mem newv));
8985 ins_cost(VOLATILE_REF_COST);
8986 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8987 ins_encode %{
8988 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8989 %}
8990 ins_pipe(pipe_serial);
8991 %}
8992
8993 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8994 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8995 match(Set prev (GetAndSetP mem newv));
8996 ins_cost(VOLATILE_REF_COST);
8997 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8998 ins_encode %{
8999 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9000 %}
9001 ins_pipe(pipe_serial);
9002 %}
9003
9004
9005 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9006 match(Set newval (GetAndAddL mem incr));
9007 ins_cost(2 * VOLATILE_REF_COST + 1);
9008 format %{ "get_and_addL $newval, [$mem], $incr" %}
9009 ins_encode %{
9010 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9011 %}
9012 ins_pipe(pipe_serial);
9013 %}
9014
9015 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9016 predicate(n->as_LoadStore()->result_not_used());
9017 match(Set dummy (GetAndAddL mem incr));
9018 ins_cost(2 * VOLATILE_REF_COST);
9019 format %{ "get_and_addL [$mem], $incr" %}
9020 ins_encode %{
9021 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9022 %}
9023 ins_pipe(pipe_serial);
9024 %}
9025
9026 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9027 match(Set newval (GetAndAddL mem incr));
9028 ins_cost(2 * VOLATILE_REF_COST + 1);
9029 format %{ "get_and_addL $newval, [$mem], $incr" %}
9030 ins_encode %{
9031 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9032 %}
9033 ins_pipe(pipe_serial);
9034 %}
9035
9036 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9037 predicate(n->as_LoadStore()->result_not_used());
9038 match(Set dummy (GetAndAddL mem incr));
9039 ins_cost(2 * VOLATILE_REF_COST);
9040 format %{ "get_and_addL [$mem], $incr" %}
9041 ins_encode %{
9042 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9043 %}
9044 ins_pipe(pipe_serial);
9045 %}
9046
9047 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9048 match(Set newval (GetAndAddI mem incr));
9049 ins_cost(2 * VOLATILE_REF_COST + 1);
9050 format %{ "get_and_addI $newval, [$mem], $incr" %}
9051 ins_encode %{
9052 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9053 %}
9054 ins_pipe(pipe_serial);
9055 %}
9056
9057 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9058 predicate(n->as_LoadStore()->result_not_used());
9059 match(Set dummy (GetAndAddI mem incr));
9060 ins_cost(2 * VOLATILE_REF_COST);
9061 format %{ "get_and_addI [$mem], $incr" %}
9062 ins_encode %{
9063 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9064 %}
9065 ins_pipe(pipe_serial);
9066 %}
9067
9068 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9069 match(Set newval (GetAndAddI mem incr));
9070 ins_cost(2 * VOLATILE_REF_COST + 1);
9071 format %{ "get_and_addI $newval, [$mem], $incr" %}
9072 ins_encode %{
9073 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9074 %}
9075 ins_pipe(pipe_serial);
9076 %}
9077
9078 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9079 predicate(n->as_LoadStore()->result_not_used());
9080 match(Set dummy (GetAndAddI mem incr));
9081 ins_cost(2 * VOLATILE_REF_COST);
9082 format %{ "get_and_addI [$mem], $incr" %}
9083 ins_encode %{
9084 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9085 %}
9086 ins_pipe(pipe_serial);
9087 %}
9088
9089 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9090 predicate(needs_acquiring_load_exclusive(n));
9091 match(Set newval (GetAndAddL mem incr));
9092 ins_cost(VOLATILE_REF_COST + 1);
9093 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9094 ins_encode %{
9095 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9096 %}
9097 ins_pipe(pipe_serial);
9098 %}
9099
9100 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9101 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9102 match(Set dummy (GetAndAddL mem incr));
9103 ins_cost(VOLATILE_REF_COST);
9104 format %{ "get_and_addL_acq [$mem], $incr" %}
9105 ins_encode %{
9106 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9107 %}
9108 ins_pipe(pipe_serial);
9109 %}
9110
9111 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9112 predicate(needs_acquiring_load_exclusive(n));
9113 match(Set newval (GetAndAddL mem incr));
9114 ins_cost(VOLATILE_REF_COST + 1);
9115 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9116 ins_encode %{
9117 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9118 %}
9119 ins_pipe(pipe_serial);
9120 %}
9121
9122 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9123 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9124 match(Set dummy (GetAndAddL mem incr));
9125 ins_cost(VOLATILE_REF_COST);
9126 format %{ "get_and_addL_acq [$mem], $incr" %}
9127 ins_encode %{
9128 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9129 %}
9130 ins_pipe(pipe_serial);
9131 %}
9132
9133 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9134 predicate(needs_acquiring_load_exclusive(n));
9135 match(Set newval (GetAndAddI mem incr));
9136 ins_cost(VOLATILE_REF_COST + 1);
9137 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9138 ins_encode %{
9139 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9140 %}
9141 ins_pipe(pipe_serial);
9142 %}
9143
9144 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9145 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9146 match(Set dummy (GetAndAddI mem incr));
9147 ins_cost(VOLATILE_REF_COST);
9148 format %{ "get_and_addI_acq [$mem], $incr" %}
9149 ins_encode %{
9150 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9151 %}
9152 ins_pipe(pipe_serial);
9153 %}
9154
9155 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9156 predicate(needs_acquiring_load_exclusive(n));
9157 match(Set newval (GetAndAddI mem incr));
9158 ins_cost(VOLATILE_REF_COST + 1);
9159 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9160 ins_encode %{
9161 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9162 %}
9163 ins_pipe(pipe_serial);
9164 %}
9165
9166 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9167 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9168 match(Set dummy (GetAndAddI mem incr));
9169 ins_cost(VOLATILE_REF_COST);
9170 format %{ "get_and_addI_acq [$mem], $incr" %}
9171 ins_encode %{
9172 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9173 %}
9174 ins_pipe(pipe_serial);
9175 %}
9176
9177 // Manifest a CmpU result in an integer register.
9178 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9179 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9180 %{
9181 match(Set dst (CmpU3 src1 src2));
9182 effect(KILL flags);
9183
9184 ins_cost(INSN_COST * 3);
9185 format %{
9186 "cmpw $src1, $src2\n\t"
9187 "csetw $dst, ne\n\t"
9188 "cnegw $dst, lo\t# CmpU3(reg)"
9189 %}
9190 ins_encode %{
9191 __ cmpw($src1$$Register, $src2$$Register);
9192 __ csetw($dst$$Register, Assembler::NE);
9193 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9194 %}
9195
9196 ins_pipe(pipe_class_default);
9197 %}
9198
9199 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9200 %{
9201 match(Set dst (CmpU3 src1 src2));
9202 effect(KILL flags);
9203
9204 ins_cost(INSN_COST * 3);
9205 format %{
9206 "subsw zr, $src1, $src2\n\t"
9207 "csetw $dst, ne\n\t"
9208 "cnegw $dst, lo\t# CmpU3(imm)"
9209 %}
9210 ins_encode %{
9211 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9212 __ csetw($dst$$Register, Assembler::NE);
9213 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9214 %}
9215
9216 ins_pipe(pipe_class_default);
9217 %}
9218
9219 // Manifest a CmpUL result in an integer register.
9220 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9221 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9222 %{
9223 match(Set dst (CmpUL3 src1 src2));
9224 effect(KILL flags);
9225
9226 ins_cost(INSN_COST * 3);
9227 format %{
9228 "cmp $src1, $src2\n\t"
9229 "csetw $dst, ne\n\t"
9230 "cnegw $dst, lo\t# CmpUL3(reg)"
9231 %}
9232 ins_encode %{
9233 __ cmp($src1$$Register, $src2$$Register);
9234 __ csetw($dst$$Register, Assembler::NE);
9235 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9236 %}
9237
9238 ins_pipe(pipe_class_default);
9239 %}
9240
9241 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9242 %{
9243 match(Set dst (CmpUL3 src1 src2));
9244 effect(KILL flags);
9245
9246 ins_cost(INSN_COST * 3);
9247 format %{
9248 "subs zr, $src1, $src2\n\t"
9249 "csetw $dst, ne\n\t"
9250 "cnegw $dst, lo\t# CmpUL3(imm)"
9251 %}
9252 ins_encode %{
9253 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9254 __ csetw($dst$$Register, Assembler::NE);
9255 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9256 %}
9257
9258 ins_pipe(pipe_class_default);
9259 %}
9260
9261 // Manifest a CmpL result in an integer register.
9262 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9263 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9264 %{
9265 match(Set dst (CmpL3 src1 src2));
9266 effect(KILL flags);
9267
9268 ins_cost(INSN_COST * 3);
9269 format %{
9270 "cmp $src1, $src2\n\t"
9271 "csetw $dst, ne\n\t"
9272 "cnegw $dst, lt\t# CmpL3(reg)"
9273 %}
9274 ins_encode %{
9275 __ cmp($src1$$Register, $src2$$Register);
9276 __ csetw($dst$$Register, Assembler::NE);
9277 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9278 %}
9279
9280 ins_pipe(pipe_class_default);
9281 %}
9282
9283 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9284 %{
9285 match(Set dst (CmpL3 src1 src2));
9286 effect(KILL flags);
9287
9288 ins_cost(INSN_COST * 3);
9289 format %{
9290 "subs zr, $src1, $src2\n\t"
9291 "csetw $dst, ne\n\t"
9292 "cnegw $dst, lt\t# CmpL3(imm)"
9293 %}
9294 ins_encode %{
9295 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9296 __ csetw($dst$$Register, Assembler::NE);
9297 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9298 %}
9299
9300 ins_pipe(pipe_class_default);
9301 %}
9302
9303 // ============================================================================
9304 // Conditional Move Instructions
9305
9306 // n.b. we have identical rules for both a signed compare op (cmpOp)
9307 // and an unsigned compare op (cmpOpU). it would be nice if we could
9308 // define an op class which merged both inputs and use it to type the
9309 // argument to a single rule. unfortunatelyt his fails because the
9310 // opclass does not live up to the COND_INTER interface of its
9311 // component operands. When the generic code tries to negate the
9312 // operand it ends up running the generci Machoper::negate method
9313 // which throws a ShouldNotHappen. So, we have to provide two flavours
9314 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9315
9316 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9317 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9318
9319 ins_cost(INSN_COST * 2);
9320 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9321
9322 ins_encode %{
9323 __ cselw(as_Register($dst$$reg),
9324 as_Register($src2$$reg),
9325 as_Register($src1$$reg),
9326 (Assembler::Condition)$cmp$$cmpcode);
9327 %}
9328
9329 ins_pipe(icond_reg_reg);
9330 %}
9331
9332 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9333 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9334
9335 ins_cost(INSN_COST * 2);
9336 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9337
9338 ins_encode %{
9339 __ cselw(as_Register($dst$$reg),
9340 as_Register($src2$$reg),
9341 as_Register($src1$$reg),
9342 (Assembler::Condition)$cmp$$cmpcode);
9343 %}
9344
9345 ins_pipe(icond_reg_reg);
9346 %}
9347
9348 // special cases where one arg is zero
9349
9350 // n.b. this is selected in preference to the rule above because it
9351 // avoids loading constant 0 into a source register
9352
9353 // TODO
9354 // we ought only to be able to cull one of these variants as the ideal
9355 // transforms ought always to order the zero consistently (to left/right?)
9356
9357 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9358 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9359
9360 ins_cost(INSN_COST * 2);
9361 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9362
9363 ins_encode %{
9364 __ cselw(as_Register($dst$$reg),
9365 as_Register($src$$reg),
9366 zr,
9367 (Assembler::Condition)$cmp$$cmpcode);
9368 %}
9369
9370 ins_pipe(icond_reg);
9371 %}
9372
9373 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9374 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9375
9376 ins_cost(INSN_COST * 2);
9377 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9378
9379 ins_encode %{
9380 __ cselw(as_Register($dst$$reg),
9381 as_Register($src$$reg),
9382 zr,
9383 (Assembler::Condition)$cmp$$cmpcode);
9384 %}
9385
9386 ins_pipe(icond_reg);
9387 %}
9388
9389 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9390 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9391
9392 ins_cost(INSN_COST * 2);
9393 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9394
9395 ins_encode %{
9396 __ cselw(as_Register($dst$$reg),
9397 zr,
9398 as_Register($src$$reg),
9399 (Assembler::Condition)$cmp$$cmpcode);
9400 %}
9401
9402 ins_pipe(icond_reg);
9403 %}
9404
9405 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9406 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9407
9408 ins_cost(INSN_COST * 2);
9409 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9410
9411 ins_encode %{
9412 __ cselw(as_Register($dst$$reg),
9413 zr,
9414 as_Register($src$$reg),
9415 (Assembler::Condition)$cmp$$cmpcode);
9416 %}
9417
9418 ins_pipe(icond_reg);
9419 %}
9420
9421 // special case for creating a boolean 0 or 1
9422
9423 // n.b. this is selected in preference to the rule above because it
9424 // avoids loading constants 0 and 1 into a source register
9425
9426 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9427 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9428
9429 ins_cost(INSN_COST * 2);
9430 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9431
9432 ins_encode %{
9433 // equivalently
9434 // cset(as_Register($dst$$reg),
9435 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9436 __ csincw(as_Register($dst$$reg),
9437 zr,
9438 zr,
9439 (Assembler::Condition)$cmp$$cmpcode);
9440 %}
9441
9442 ins_pipe(icond_none);
9443 %}
9444
9445 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9446 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9447
9448 ins_cost(INSN_COST * 2);
9449 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9450
9451 ins_encode %{
9452 // equivalently
9453 // cset(as_Register($dst$$reg),
9454 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9455 __ csincw(as_Register($dst$$reg),
9456 zr,
9457 zr,
9458 (Assembler::Condition)$cmp$$cmpcode);
9459 %}
9460
9461 ins_pipe(icond_none);
9462 %}
9463
9464 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9465 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9466
9467 ins_cost(INSN_COST * 2);
9468 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9469
9470 ins_encode %{
9471 __ csel(as_Register($dst$$reg),
9472 as_Register($src2$$reg),
9473 as_Register($src1$$reg),
9474 (Assembler::Condition)$cmp$$cmpcode);
9475 %}
9476
9477 ins_pipe(icond_reg_reg);
9478 %}
9479
9480 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9481 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9482
9483 ins_cost(INSN_COST * 2);
9484 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9485
9486 ins_encode %{
9487 __ csel(as_Register($dst$$reg),
9488 as_Register($src2$$reg),
9489 as_Register($src1$$reg),
9490 (Assembler::Condition)$cmp$$cmpcode);
9491 %}
9492
9493 ins_pipe(icond_reg_reg);
9494 %}
9495
9496 // special cases where one arg is zero
9497
9498 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9499 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9500
9501 ins_cost(INSN_COST * 2);
9502 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9503
9504 ins_encode %{
9505 __ csel(as_Register($dst$$reg),
9506 zr,
9507 as_Register($src$$reg),
9508 (Assembler::Condition)$cmp$$cmpcode);
9509 %}
9510
9511 ins_pipe(icond_reg);
9512 %}
9513
9514 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9515 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9516
9517 ins_cost(INSN_COST * 2);
9518 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9519
9520 ins_encode %{
9521 __ csel(as_Register($dst$$reg),
9522 zr,
9523 as_Register($src$$reg),
9524 (Assembler::Condition)$cmp$$cmpcode);
9525 %}
9526
9527 ins_pipe(icond_reg);
9528 %}
9529
9530 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9531 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9532
9533 ins_cost(INSN_COST * 2);
9534 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9535
9536 ins_encode %{
9537 __ csel(as_Register($dst$$reg),
9538 as_Register($src$$reg),
9539 zr,
9540 (Assembler::Condition)$cmp$$cmpcode);
9541 %}
9542
9543 ins_pipe(icond_reg);
9544 %}
9545
9546 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9547 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9548
9549 ins_cost(INSN_COST * 2);
9550 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9551
9552 ins_encode %{
9553 __ csel(as_Register($dst$$reg),
9554 as_Register($src$$reg),
9555 zr,
9556 (Assembler::Condition)$cmp$$cmpcode);
9557 %}
9558
9559 ins_pipe(icond_reg);
9560 %}
9561
9562 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9563 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9564
9565 ins_cost(INSN_COST * 2);
9566 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9567
9568 ins_encode %{
9569 __ csel(as_Register($dst$$reg),
9570 as_Register($src2$$reg),
9571 as_Register($src1$$reg),
9572 (Assembler::Condition)$cmp$$cmpcode);
9573 %}
9574
9575 ins_pipe(icond_reg_reg);
9576 %}
9577
9578 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9579 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9580
9581 ins_cost(INSN_COST * 2);
9582 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9583
9584 ins_encode %{
9585 __ csel(as_Register($dst$$reg),
9586 as_Register($src2$$reg),
9587 as_Register($src1$$reg),
9588 (Assembler::Condition)$cmp$$cmpcode);
9589 %}
9590
9591 ins_pipe(icond_reg_reg);
9592 %}
9593
9594 // special cases where one arg is zero
9595
9596 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9597 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9598
9599 ins_cost(INSN_COST * 2);
9600 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9601
9602 ins_encode %{
9603 __ csel(as_Register($dst$$reg),
9604 zr,
9605 as_Register($src$$reg),
9606 (Assembler::Condition)$cmp$$cmpcode);
9607 %}
9608
9609 ins_pipe(icond_reg);
9610 %}
9611
9612 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9613 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9614
9615 ins_cost(INSN_COST * 2);
9616 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9617
9618 ins_encode %{
9619 __ csel(as_Register($dst$$reg),
9620 zr,
9621 as_Register($src$$reg),
9622 (Assembler::Condition)$cmp$$cmpcode);
9623 %}
9624
9625 ins_pipe(icond_reg);
9626 %}
9627
9628 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9629 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9630
9631 ins_cost(INSN_COST * 2);
9632 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9633
9634 ins_encode %{
9635 __ csel(as_Register($dst$$reg),
9636 as_Register($src$$reg),
9637 zr,
9638 (Assembler::Condition)$cmp$$cmpcode);
9639 %}
9640
9641 ins_pipe(icond_reg);
9642 %}
9643
9644 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9645 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9646
9647 ins_cost(INSN_COST * 2);
9648 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9649
9650 ins_encode %{
9651 __ csel(as_Register($dst$$reg),
9652 as_Register($src$$reg),
9653 zr,
9654 (Assembler::Condition)$cmp$$cmpcode);
9655 %}
9656
9657 ins_pipe(icond_reg);
9658 %}
9659
9660 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9661 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9662
9663 ins_cost(INSN_COST * 2);
9664 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9665
9666 ins_encode %{
9667 __ cselw(as_Register($dst$$reg),
9668 as_Register($src2$$reg),
9669 as_Register($src1$$reg),
9670 (Assembler::Condition)$cmp$$cmpcode);
9671 %}
9672
9673 ins_pipe(icond_reg_reg);
9674 %}
9675
9676 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9677 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9678
9679 ins_cost(INSN_COST * 2);
9680 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9681
9682 ins_encode %{
9683 __ cselw(as_Register($dst$$reg),
9684 as_Register($src2$$reg),
9685 as_Register($src1$$reg),
9686 (Assembler::Condition)$cmp$$cmpcode);
9687 %}
9688
9689 ins_pipe(icond_reg_reg);
9690 %}
9691
9692 // special cases where one arg is zero
9693
9694 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9695 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9696
9697 ins_cost(INSN_COST * 2);
9698 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9699
9700 ins_encode %{
9701 __ cselw(as_Register($dst$$reg),
9702 zr,
9703 as_Register($src$$reg),
9704 (Assembler::Condition)$cmp$$cmpcode);
9705 %}
9706
9707 ins_pipe(icond_reg);
9708 %}
9709
9710 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9711 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9712
9713 ins_cost(INSN_COST * 2);
9714 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9715
9716 ins_encode %{
9717 __ cselw(as_Register($dst$$reg),
9718 zr,
9719 as_Register($src$$reg),
9720 (Assembler::Condition)$cmp$$cmpcode);
9721 %}
9722
9723 ins_pipe(icond_reg);
9724 %}
9725
9726 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9727 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9728
9729 ins_cost(INSN_COST * 2);
9730 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9731
9732 ins_encode %{
9733 __ cselw(as_Register($dst$$reg),
9734 as_Register($src$$reg),
9735 zr,
9736 (Assembler::Condition)$cmp$$cmpcode);
9737 %}
9738
9739 ins_pipe(icond_reg);
9740 %}
9741
9742 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9743 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9744
9745 ins_cost(INSN_COST * 2);
9746 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9747
9748 ins_encode %{
9749 __ cselw(as_Register($dst$$reg),
9750 as_Register($src$$reg),
9751 zr,
9752 (Assembler::Condition)$cmp$$cmpcode);
9753 %}
9754
9755 ins_pipe(icond_reg);
9756 %}
9757
9758 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9759 %{
9760 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9761
9762 ins_cost(INSN_COST * 3);
9763
9764 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9765 ins_encode %{
9766 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9767 __ fcsels(as_FloatRegister($dst$$reg),
9768 as_FloatRegister($src2$$reg),
9769 as_FloatRegister($src1$$reg),
9770 cond);
9771 %}
9772
9773 ins_pipe(fp_cond_reg_reg_s);
9774 %}
9775
9776 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9777 %{
9778 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9779
9780 ins_cost(INSN_COST * 3);
9781
9782 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9783 ins_encode %{
9784 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9785 __ fcsels(as_FloatRegister($dst$$reg),
9786 as_FloatRegister($src2$$reg),
9787 as_FloatRegister($src1$$reg),
9788 cond);
9789 %}
9790
9791 ins_pipe(fp_cond_reg_reg_s);
9792 %}
9793
9794 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9795 %{
9796 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9797
9798 ins_cost(INSN_COST * 3);
9799
9800 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9801 ins_encode %{
9802 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9803 __ fcseld(as_FloatRegister($dst$$reg),
9804 as_FloatRegister($src2$$reg),
9805 as_FloatRegister($src1$$reg),
9806 cond);
9807 %}
9808
9809 ins_pipe(fp_cond_reg_reg_d);
9810 %}
9811
9812 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9813 %{
9814 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9815
9816 ins_cost(INSN_COST * 3);
9817
9818 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9819 ins_encode %{
9820 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9821 __ fcseld(as_FloatRegister($dst$$reg),
9822 as_FloatRegister($src2$$reg),
9823 as_FloatRegister($src1$$reg),
9824 cond);
9825 %}
9826
9827 ins_pipe(fp_cond_reg_reg_d);
9828 %}
9829
9830 // ============================================================================
9831 // Arithmetic Instructions
9832 //
9833
9834 // Integer Addition
9835
9836 // TODO
9837 // these currently employ operations which do not set CR and hence are
9838 // not flagged as killing CR but we would like to isolate the cases
9839 // where we want to set flags from those where we don't. need to work
9840 // out how to do that.
9841
9842 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9843 match(Set dst (AddI src1 src2));
9844
9845 ins_cost(INSN_COST);
9846 format %{ "addw $dst, $src1, $src2" %}
9847
9848 ins_encode %{
9849 __ addw(as_Register($dst$$reg),
9850 as_Register($src1$$reg),
9851 as_Register($src2$$reg));
9852 %}
9853
9854 ins_pipe(ialu_reg_reg);
9855 %}
9856
9857 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9858 match(Set dst (AddI src1 src2));
9859
9860 ins_cost(INSN_COST);
9861 format %{ "addw $dst, $src1, $src2" %}
9862
9863 // use opcode to indicate that this is an add not a sub
9864 opcode(0x0);
9865
9866 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9867
9868 ins_pipe(ialu_reg_imm);
9869 %}
9870
9871 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9872 match(Set dst (AddI (ConvL2I src1) src2));
9873
9874 ins_cost(INSN_COST);
9875 format %{ "addw $dst, $src1, $src2" %}
9876
9877 // use opcode to indicate that this is an add not a sub
9878 opcode(0x0);
9879
9880 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9881
9882 ins_pipe(ialu_reg_imm);
9883 %}
9884
9885 // Pointer Addition
9886 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9887 match(Set dst (AddP src1 src2));
9888
9889 ins_cost(INSN_COST);
9890 format %{ "add $dst, $src1, $src2\t# ptr" %}
9891
9892 ins_encode %{
9893 __ add(as_Register($dst$$reg),
9894 as_Register($src1$$reg),
9895 as_Register($src2$$reg));
9896 %}
9897
9898 ins_pipe(ialu_reg_reg);
9899 %}
9900
9901 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9902 match(Set dst (AddP src1 (ConvI2L src2)));
9903
9904 ins_cost(1.9 * INSN_COST);
9905 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9906
9907 ins_encode %{
9908 __ add(as_Register($dst$$reg),
9909 as_Register($src1$$reg),
9910 as_Register($src2$$reg), ext::sxtw);
9911 %}
9912
9913 ins_pipe(ialu_reg_reg);
9914 %}
9915
9916 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9917 match(Set dst (AddP src1 (LShiftL src2 scale)));
9918
9919 ins_cost(1.9 * INSN_COST);
9920 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9921
9922 ins_encode %{
9923 __ lea(as_Register($dst$$reg),
9924 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9925 Address::lsl($scale$$constant)));
9926 %}
9927
9928 ins_pipe(ialu_reg_reg_shift);
9929 %}
9930
9931 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9932 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9933
9934 ins_cost(1.9 * INSN_COST);
9935 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9936
9937 ins_encode %{
9938 __ lea(as_Register($dst$$reg),
9939 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9940 Address::sxtw($scale$$constant)));
9941 %}
9942
9943 ins_pipe(ialu_reg_reg_shift);
9944 %}
9945
9946 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9947 match(Set dst (LShiftL (ConvI2L src) scale));
9948
9949 ins_cost(INSN_COST);
9950 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9951
9952 ins_encode %{
9953 __ sbfiz(as_Register($dst$$reg),
9954 as_Register($src$$reg),
9955 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9956 %}
9957
9958 ins_pipe(ialu_reg_shift);
9959 %}
9960
9961 // Pointer Immediate Addition
9962 // n.b. this needs to be more expensive than using an indirect memory
9963 // operand
9964 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9965 match(Set dst (AddP src1 src2));
9966
9967 ins_cost(INSN_COST);
9968 format %{ "add $dst, $src1, $src2\t# ptr" %}
9969
9970 // use opcode to indicate that this is an add not a sub
9971 opcode(0x0);
9972
9973 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9974
9975 ins_pipe(ialu_reg_imm);
9976 %}
9977
9978 // Long Addition
9979 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9980
9981 match(Set dst (AddL src1 src2));
9982
9983 ins_cost(INSN_COST);
9984 format %{ "add $dst, $src1, $src2" %}
9985
9986 ins_encode %{
9987 __ add(as_Register($dst$$reg),
9988 as_Register($src1$$reg),
9989 as_Register($src2$$reg));
9990 %}
9991
9992 ins_pipe(ialu_reg_reg);
9993 %}
9994
9995 // No constant pool entries requiredLong Immediate Addition.
9996 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9997 match(Set dst (AddL src1 src2));
9998
9999 ins_cost(INSN_COST);
10000 format %{ "add $dst, $src1, $src2" %}
10001
10002 // use opcode to indicate that this is an add not a sub
10003 opcode(0x0);
10004
10005 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10006
10007 ins_pipe(ialu_reg_imm);
10008 %}
10009
10010 // Integer Subtraction
10011 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10012 match(Set dst (SubI src1 src2));
10013
10014 ins_cost(INSN_COST);
10015 format %{ "subw $dst, $src1, $src2" %}
10016
10017 ins_encode %{
10018 __ subw(as_Register($dst$$reg),
10019 as_Register($src1$$reg),
10020 as_Register($src2$$reg));
10021 %}
10022
10023 ins_pipe(ialu_reg_reg);
10024 %}
10025
10026 // Immediate Subtraction
10027 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10028 match(Set dst (SubI src1 src2));
10029
10030 ins_cost(INSN_COST);
10031 format %{ "subw $dst, $src1, $src2" %}
10032
10033 // use opcode to indicate that this is a sub not an add
10034 opcode(0x1);
10035
10036 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10037
10038 ins_pipe(ialu_reg_imm);
10039 %}
10040
10041 // Long Subtraction
10042 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10043
10044 match(Set dst (SubL src1 src2));
10045
10046 ins_cost(INSN_COST);
10047 format %{ "sub $dst, $src1, $src2" %}
10048
10049 ins_encode %{
10050 __ sub(as_Register($dst$$reg),
10051 as_Register($src1$$reg),
10052 as_Register($src2$$reg));
10053 %}
10054
10055 ins_pipe(ialu_reg_reg);
10056 %}
10057
10058 // No constant pool entries requiredLong Immediate Subtraction.
10059 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10060 match(Set dst (SubL src1 src2));
10061
10062 ins_cost(INSN_COST);
10063 format %{ "sub$dst, $src1, $src2" %}
10064
10065 // use opcode to indicate that this is a sub not an add
10066 opcode(0x1);
10067
10068 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10069
10070 ins_pipe(ialu_reg_imm);
10071 %}
10072
10073 // Integer Negation (special case for sub)
10074
10075 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10076 match(Set dst (SubI zero src));
10077
10078 ins_cost(INSN_COST);
10079 format %{ "negw $dst, $src\t# int" %}
10080
10081 ins_encode %{
10082 __ negw(as_Register($dst$$reg),
10083 as_Register($src$$reg));
10084 %}
10085
10086 ins_pipe(ialu_reg);
10087 %}
10088
10089 // Long Negation
10090
10091 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10092 match(Set dst (SubL zero src));
10093
10094 ins_cost(INSN_COST);
10095 format %{ "neg $dst, $src\t# long" %}
10096
10097 ins_encode %{
10098 __ neg(as_Register($dst$$reg),
10099 as_Register($src$$reg));
10100 %}
10101
10102 ins_pipe(ialu_reg);
10103 %}
10104
10105 // Integer Multiply
10106
10107 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10108 match(Set dst (MulI src1 src2));
10109
10110 ins_cost(INSN_COST * 3);
10111 format %{ "mulw $dst, $src1, $src2" %}
10112
10113 ins_encode %{
10114 __ mulw(as_Register($dst$$reg),
10115 as_Register($src1$$reg),
10116 as_Register($src2$$reg));
10117 %}
10118
10119 ins_pipe(imul_reg_reg);
10120 %}
10121
10122 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10123 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10124
10125 ins_cost(INSN_COST * 3);
10126 format %{ "smull $dst, $src1, $src2" %}
10127
10128 ins_encode %{
10129 __ smull(as_Register($dst$$reg),
10130 as_Register($src1$$reg),
10131 as_Register($src2$$reg));
10132 %}
10133
10134 ins_pipe(imul_reg_reg);
10135 %}
10136
10137 // Long Multiply
10138
10139 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10140 match(Set dst (MulL src1 src2));
10141
10142 ins_cost(INSN_COST * 5);
10143 format %{ "mul $dst, $src1, $src2" %}
10144
10145 ins_encode %{
10146 __ mul(as_Register($dst$$reg),
10147 as_Register($src1$$reg),
10148 as_Register($src2$$reg));
10149 %}
10150
10151 ins_pipe(lmul_reg_reg);
10152 %}
10153
10154 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10155 %{
10156 match(Set dst (MulHiL src1 src2));
10157
10158 ins_cost(INSN_COST * 7);
10159 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10160
10161 ins_encode %{
10162 __ smulh(as_Register($dst$$reg),
10163 as_Register($src1$$reg),
10164 as_Register($src2$$reg));
10165 %}
10166
10167 ins_pipe(lmul_reg_reg);
10168 %}
10169
10170 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10171 %{
10172 match(Set dst (UMulHiL src1 src2));
10173
10174 ins_cost(INSN_COST * 7);
10175 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10176
10177 ins_encode %{
10178 __ umulh(as_Register($dst$$reg),
10179 as_Register($src1$$reg),
10180 as_Register($src2$$reg));
10181 %}
10182
10183 ins_pipe(lmul_reg_reg);
10184 %}
10185
10186 // Combined Integer Multiply & Add/Sub
10187
10188 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10189 match(Set dst (AddI src3 (MulI src1 src2)));
10190
10191 ins_cost(INSN_COST * 3);
10192 format %{ "madd $dst, $src1, $src2, $src3" %}
10193
10194 ins_encode %{
10195 __ maddw(as_Register($dst$$reg),
10196 as_Register($src1$$reg),
10197 as_Register($src2$$reg),
10198 as_Register($src3$$reg));
10199 %}
10200
10201 ins_pipe(imac_reg_reg);
10202 %}
10203
10204 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10205 match(Set dst (SubI src3 (MulI src1 src2)));
10206
10207 ins_cost(INSN_COST * 3);
10208 format %{ "msub $dst, $src1, $src2, $src3" %}
10209
10210 ins_encode %{
10211 __ msubw(as_Register($dst$$reg),
10212 as_Register($src1$$reg),
10213 as_Register($src2$$reg),
10214 as_Register($src3$$reg));
10215 %}
10216
10217 ins_pipe(imac_reg_reg);
10218 %}
10219
10220 // Combined Integer Multiply & Neg
10221
10222 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10223 match(Set dst (MulI (SubI zero src1) src2));
10224
10225 ins_cost(INSN_COST * 3);
10226 format %{ "mneg $dst, $src1, $src2" %}
10227
10228 ins_encode %{
10229 __ mnegw(as_Register($dst$$reg),
10230 as_Register($src1$$reg),
10231 as_Register($src2$$reg));
10232 %}
10233
10234 ins_pipe(imac_reg_reg);
10235 %}
10236
10237 // Combined Long Multiply & Add/Sub
10238
10239 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10240 match(Set dst (AddL src3 (MulL src1 src2)));
10241
10242 ins_cost(INSN_COST * 5);
10243 format %{ "madd $dst, $src1, $src2, $src3" %}
10244
10245 ins_encode %{
10246 __ madd(as_Register($dst$$reg),
10247 as_Register($src1$$reg),
10248 as_Register($src2$$reg),
10249 as_Register($src3$$reg));
10250 %}
10251
10252 ins_pipe(lmac_reg_reg);
10253 %}
10254
10255 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10256 match(Set dst (SubL src3 (MulL src1 src2)));
10257
10258 ins_cost(INSN_COST * 5);
10259 format %{ "msub $dst, $src1, $src2, $src3" %}
10260
10261 ins_encode %{
10262 __ msub(as_Register($dst$$reg),
10263 as_Register($src1$$reg),
10264 as_Register($src2$$reg),
10265 as_Register($src3$$reg));
10266 %}
10267
10268 ins_pipe(lmac_reg_reg);
10269 %}
10270
10271 // Combined Long Multiply & Neg
10272
10273 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10274 match(Set dst (MulL (SubL zero src1) src2));
10275
10276 ins_cost(INSN_COST * 5);
10277 format %{ "mneg $dst, $src1, $src2" %}
10278
10279 ins_encode %{
10280 __ mneg(as_Register($dst$$reg),
10281 as_Register($src1$$reg),
10282 as_Register($src2$$reg));
10283 %}
10284
10285 ins_pipe(lmac_reg_reg);
10286 %}
10287
10288 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10289
10290 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10291 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10292
10293 ins_cost(INSN_COST * 3);
10294 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10295
10296 ins_encode %{
10297 __ smaddl(as_Register($dst$$reg),
10298 as_Register($src1$$reg),
10299 as_Register($src2$$reg),
10300 as_Register($src3$$reg));
10301 %}
10302
10303 ins_pipe(imac_reg_reg);
10304 %}
10305
10306 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10307 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10308
10309 ins_cost(INSN_COST * 3);
10310 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10311
10312 ins_encode %{
10313 __ smsubl(as_Register($dst$$reg),
10314 as_Register($src1$$reg),
10315 as_Register($src2$$reg),
10316 as_Register($src3$$reg));
10317 %}
10318
10319 ins_pipe(imac_reg_reg);
10320 %}
10321
10322 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10323 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10324
10325 ins_cost(INSN_COST * 3);
10326 format %{ "smnegl $dst, $src1, $src2" %}
10327
10328 ins_encode %{
10329 __ smnegl(as_Register($dst$$reg),
10330 as_Register($src1$$reg),
10331 as_Register($src2$$reg));
10332 %}
10333
10334 ins_pipe(imac_reg_reg);
10335 %}
10336
10337 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10338
10339 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10340 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10341
10342 ins_cost(INSN_COST * 5);
10343 format %{ "mulw rscratch1, $src1, $src2\n\t"
10344 "maddw $dst, $src3, $src4, rscratch1" %}
10345
10346 ins_encode %{
10347 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10348 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10349
10350 ins_pipe(imac_reg_reg);
10351 %}
10352
10353 // Integer Divide
10354
10355 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10356 match(Set dst (DivI src1 src2));
10357
10358 ins_cost(INSN_COST * 19);
10359 format %{ "sdivw $dst, $src1, $src2" %}
10360
10361 ins_encode(aarch64_enc_divw(dst, src1, src2));
10362 ins_pipe(idiv_reg_reg);
10363 %}
10364
10365 // Long Divide
10366
10367 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10368 match(Set dst (DivL src1 src2));
10369
10370 ins_cost(INSN_COST * 35);
10371 format %{ "sdiv $dst, $src1, $src2" %}
10372
10373 ins_encode(aarch64_enc_div(dst, src1, src2));
10374 ins_pipe(ldiv_reg_reg);
10375 %}
10376
10377 // Integer Remainder
10378
10379 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10380 match(Set dst (ModI src1 src2));
10381
10382 ins_cost(INSN_COST * 22);
10383 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10384 "msubw $dst, rscratch1, $src2, $src1" %}
10385
10386 ins_encode(aarch64_enc_modw(dst, src1, src2));
10387 ins_pipe(idiv_reg_reg);
10388 %}
10389
10390 // Long Remainder
10391
10392 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10393 match(Set dst (ModL src1 src2));
10394
10395 ins_cost(INSN_COST * 38);
10396 format %{ "sdiv rscratch1, $src1, $src2\n"
10397 "msub $dst, rscratch1, $src2, $src1" %}
10398
10399 ins_encode(aarch64_enc_mod(dst, src1, src2));
10400 ins_pipe(ldiv_reg_reg);
10401 %}
10402
10403 // Unsigned Integer Divide
10404
10405 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10406 match(Set dst (UDivI src1 src2));
10407
10408 ins_cost(INSN_COST * 19);
10409 format %{ "udivw $dst, $src1, $src2" %}
10410
10411 ins_encode %{
10412 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10413 %}
10414
10415 ins_pipe(idiv_reg_reg);
10416 %}
10417
10418 // Unsigned Long Divide
10419
10420 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10421 match(Set dst (UDivL src1 src2));
10422
10423 ins_cost(INSN_COST * 35);
10424 format %{ "udiv $dst, $src1, $src2" %}
10425
10426 ins_encode %{
10427 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10428 %}
10429
10430 ins_pipe(ldiv_reg_reg);
10431 %}
10432
10433 // Unsigned Integer Remainder
10434
10435 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10436 match(Set dst (UModI src1 src2));
10437
10438 ins_cost(INSN_COST * 22);
10439 format %{ "udivw rscratch1, $src1, $src2\n\t"
10440 "msubw $dst, rscratch1, $src2, $src1" %}
10441
10442 ins_encode %{
10443 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10444 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10445 %}
10446
10447 ins_pipe(idiv_reg_reg);
10448 %}
10449
10450 // Unsigned Long Remainder
10451
10452 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10453 match(Set dst (UModL src1 src2));
10454
10455 ins_cost(INSN_COST * 38);
10456 format %{ "udiv rscratch1, $src1, $src2\n"
10457 "msub $dst, rscratch1, $src2, $src1" %}
10458
10459 ins_encode %{
10460 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10461 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10462 %}
10463
10464 ins_pipe(ldiv_reg_reg);
10465 %}
10466
10467 // Integer Shifts
10468
10469 // Shift Left Register
10470 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10471 match(Set dst (LShiftI src1 src2));
10472
10473 ins_cost(INSN_COST * 2);
10474 format %{ "lslvw $dst, $src1, $src2" %}
10475
10476 ins_encode %{
10477 __ lslvw(as_Register($dst$$reg),
10478 as_Register($src1$$reg),
10479 as_Register($src2$$reg));
10480 %}
10481
10482 ins_pipe(ialu_reg_reg_vshift);
10483 %}
10484
10485 // Shift Left Immediate
10486 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10487 match(Set dst (LShiftI src1 src2));
10488
10489 ins_cost(INSN_COST);
10490 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10491
10492 ins_encode %{
10493 __ lslw(as_Register($dst$$reg),
10494 as_Register($src1$$reg),
10495 $src2$$constant & 0x1f);
10496 %}
10497
10498 ins_pipe(ialu_reg_shift);
10499 %}
10500
10501 // Shift Right Logical Register
10502 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10503 match(Set dst (URShiftI src1 src2));
10504
10505 ins_cost(INSN_COST * 2);
10506 format %{ "lsrvw $dst, $src1, $src2" %}
10507
10508 ins_encode %{
10509 __ lsrvw(as_Register($dst$$reg),
10510 as_Register($src1$$reg),
10511 as_Register($src2$$reg));
10512 %}
10513
10514 ins_pipe(ialu_reg_reg_vshift);
10515 %}
10516
10517 // Shift Right Logical Immediate
10518 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10519 match(Set dst (URShiftI src1 src2));
10520
10521 ins_cost(INSN_COST);
10522 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10523
10524 ins_encode %{
10525 __ lsrw(as_Register($dst$$reg),
10526 as_Register($src1$$reg),
10527 $src2$$constant & 0x1f);
10528 %}
10529
10530 ins_pipe(ialu_reg_shift);
10531 %}
10532
10533 // Shift Right Arithmetic Register
10534 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10535 match(Set dst (RShiftI src1 src2));
10536
10537 ins_cost(INSN_COST * 2);
10538 format %{ "asrvw $dst, $src1, $src2" %}
10539
10540 ins_encode %{
10541 __ asrvw(as_Register($dst$$reg),
10542 as_Register($src1$$reg),
10543 as_Register($src2$$reg));
10544 %}
10545
10546 ins_pipe(ialu_reg_reg_vshift);
10547 %}
10548
10549 // Shift Right Arithmetic Immediate
10550 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10551 match(Set dst (RShiftI src1 src2));
10552
10553 ins_cost(INSN_COST);
10554 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10555
10556 ins_encode %{
10557 __ asrw(as_Register($dst$$reg),
10558 as_Register($src1$$reg),
10559 $src2$$constant & 0x1f);
10560 %}
10561
10562 ins_pipe(ialu_reg_shift);
10563 %}
10564
10565 // Combined Int Mask and Right Shift (using UBFM)
10566 // TODO
10567
10568 // Long Shifts
10569
10570 // Shift Left Register
10571 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10572 match(Set dst (LShiftL src1 src2));
10573
10574 ins_cost(INSN_COST * 2);
10575 format %{ "lslv $dst, $src1, $src2" %}
10576
10577 ins_encode %{
10578 __ lslv(as_Register($dst$$reg),
10579 as_Register($src1$$reg),
10580 as_Register($src2$$reg));
10581 %}
10582
10583 ins_pipe(ialu_reg_reg_vshift);
10584 %}
10585
10586 // Shift Left Immediate
10587 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10588 match(Set dst (LShiftL src1 src2));
10589
10590 ins_cost(INSN_COST);
10591 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10592
10593 ins_encode %{
10594 __ lsl(as_Register($dst$$reg),
10595 as_Register($src1$$reg),
10596 $src2$$constant & 0x3f);
10597 %}
10598
10599 ins_pipe(ialu_reg_shift);
10600 %}
10601
10602 // Shift Right Logical Register
10603 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10604 match(Set dst (URShiftL src1 src2));
10605
10606 ins_cost(INSN_COST * 2);
10607 format %{ "lsrv $dst, $src1, $src2" %}
10608
10609 ins_encode %{
10610 __ lsrv(as_Register($dst$$reg),
10611 as_Register($src1$$reg),
10612 as_Register($src2$$reg));
10613 %}
10614
10615 ins_pipe(ialu_reg_reg_vshift);
10616 %}
10617
10618 // Shift Right Logical Immediate
10619 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10620 match(Set dst (URShiftL src1 src2));
10621
10622 ins_cost(INSN_COST);
10623 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10624
10625 ins_encode %{
10626 __ lsr(as_Register($dst$$reg),
10627 as_Register($src1$$reg),
10628 $src2$$constant & 0x3f);
10629 %}
10630
10631 ins_pipe(ialu_reg_shift);
10632 %}
10633
10634 // A special-case pattern for card table stores.
10635 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10636 match(Set dst (URShiftL (CastP2X src1) src2));
10637
10638 ins_cost(INSN_COST);
10639 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10640
10641 ins_encode %{
10642 __ lsr(as_Register($dst$$reg),
10643 as_Register($src1$$reg),
10644 $src2$$constant & 0x3f);
10645 %}
10646
10647 ins_pipe(ialu_reg_shift);
10648 %}
10649
10650 // Shift Right Arithmetic Register
10651 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10652 match(Set dst (RShiftL src1 src2));
10653
10654 ins_cost(INSN_COST * 2);
10655 format %{ "asrv $dst, $src1, $src2" %}
10656
10657 ins_encode %{
10658 __ asrv(as_Register($dst$$reg),
10659 as_Register($src1$$reg),
10660 as_Register($src2$$reg));
10661 %}
10662
10663 ins_pipe(ialu_reg_reg_vshift);
10664 %}
10665
10666 // Shift Right Arithmetic Immediate
10667 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10668 match(Set dst (RShiftL src1 src2));
10669
10670 ins_cost(INSN_COST);
10671 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10672
10673 ins_encode %{
10674 __ asr(as_Register($dst$$reg),
10675 as_Register($src1$$reg),
10676 $src2$$constant & 0x3f);
10677 %}
10678
10679 ins_pipe(ialu_reg_shift);
10680 %}
10681
10682 // BEGIN This section of the file is automatically generated. Do not edit --------------
10683 // This section is generated from aarch64_ad.m4
10684
10685 // This pattern is automatically generated from aarch64_ad.m4
10686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10687 instruct regL_not_reg(iRegLNoSp dst,
10688 iRegL src1, immL_M1 m1,
10689 rFlagsReg cr) %{
10690 match(Set dst (XorL src1 m1));
10691 ins_cost(INSN_COST);
10692 format %{ "eon $dst, $src1, zr" %}
10693
10694 ins_encode %{
10695 __ eon(as_Register($dst$$reg),
10696 as_Register($src1$$reg),
10697 zr,
10698 Assembler::LSL, 0);
10699 %}
10700
10701 ins_pipe(ialu_reg);
10702 %}
10703
10704 // This pattern is automatically generated from aarch64_ad.m4
10705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10706 instruct regI_not_reg(iRegINoSp dst,
10707 iRegIorL2I src1, immI_M1 m1,
10708 rFlagsReg cr) %{
10709 match(Set dst (XorI src1 m1));
10710 ins_cost(INSN_COST);
10711 format %{ "eonw $dst, $src1, zr" %}
10712
10713 ins_encode %{
10714 __ eonw(as_Register($dst$$reg),
10715 as_Register($src1$$reg),
10716 zr,
10717 Assembler::LSL, 0);
10718 %}
10719
10720 ins_pipe(ialu_reg);
10721 %}
10722
10723 // This pattern is automatically generated from aarch64_ad.m4
10724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10725 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10726 immI0 zero, iRegIorL2I src1, immI src2) %{
10727 match(Set dst (SubI zero (URShiftI src1 src2)));
10728
10729 ins_cost(1.9 * INSN_COST);
10730 format %{ "negw $dst, $src1, LSR $src2" %}
10731
10732 ins_encode %{
10733 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10734 Assembler::LSR, $src2$$constant & 0x1f);
10735 %}
10736
10737 ins_pipe(ialu_reg_shift);
10738 %}
10739
10740 // This pattern is automatically generated from aarch64_ad.m4
10741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10742 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10743 immI0 zero, iRegIorL2I src1, immI src2) %{
10744 match(Set dst (SubI zero (RShiftI src1 src2)));
10745
10746 ins_cost(1.9 * INSN_COST);
10747 format %{ "negw $dst, $src1, ASR $src2" %}
10748
10749 ins_encode %{
10750 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10751 Assembler::ASR, $src2$$constant & 0x1f);
10752 %}
10753
10754 ins_pipe(ialu_reg_shift);
10755 %}
10756
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10760 immI0 zero, iRegIorL2I src1, immI src2) %{
10761 match(Set dst (SubI zero (LShiftI src1 src2)));
10762
10763 ins_cost(1.9 * INSN_COST);
10764 format %{ "negw $dst, $src1, LSL $src2" %}
10765
10766 ins_encode %{
10767 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10768 Assembler::LSL, $src2$$constant & 0x1f);
10769 %}
10770
10771 ins_pipe(ialu_reg_shift);
10772 %}
10773
10774 // This pattern is automatically generated from aarch64_ad.m4
10775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10776 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10777 immL0 zero, iRegL src1, immI src2) %{
10778 match(Set dst (SubL zero (URShiftL src1 src2)));
10779
10780 ins_cost(1.9 * INSN_COST);
10781 format %{ "neg $dst, $src1, LSR $src2" %}
10782
10783 ins_encode %{
10784 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10785 Assembler::LSR, $src2$$constant & 0x3f);
10786 %}
10787
10788 ins_pipe(ialu_reg_shift);
10789 %}
10790
10791 // This pattern is automatically generated from aarch64_ad.m4
10792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10793 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10794 immL0 zero, iRegL src1, immI src2) %{
10795 match(Set dst (SubL zero (RShiftL src1 src2)));
10796
10797 ins_cost(1.9 * INSN_COST);
10798 format %{ "neg $dst, $src1, ASR $src2" %}
10799
10800 ins_encode %{
10801 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10802 Assembler::ASR, $src2$$constant & 0x3f);
10803 %}
10804
10805 ins_pipe(ialu_reg_shift);
10806 %}
10807
10808 // This pattern is automatically generated from aarch64_ad.m4
10809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10810 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10811 immL0 zero, iRegL src1, immI src2) %{
10812 match(Set dst (SubL zero (LShiftL src1 src2)));
10813
10814 ins_cost(1.9 * INSN_COST);
10815 format %{ "neg $dst, $src1, LSL $src2" %}
10816
10817 ins_encode %{
10818 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10819 Assembler::LSL, $src2$$constant & 0x3f);
10820 %}
10821
10822 ins_pipe(ialu_reg_shift);
10823 %}
10824
10825 // This pattern is automatically generated from aarch64_ad.m4
10826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10827 instruct AndI_reg_not_reg(iRegINoSp dst,
10828 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10829 match(Set dst (AndI src1 (XorI src2 m1)));
10830 ins_cost(INSN_COST);
10831 format %{ "bicw $dst, $src1, $src2" %}
10832
10833 ins_encode %{
10834 __ bicw(as_Register($dst$$reg),
10835 as_Register($src1$$reg),
10836 as_Register($src2$$reg),
10837 Assembler::LSL, 0);
10838 %}
10839
10840 ins_pipe(ialu_reg_reg);
10841 %}
10842
10843 // This pattern is automatically generated from aarch64_ad.m4
10844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10845 instruct AndL_reg_not_reg(iRegLNoSp dst,
10846 iRegL src1, iRegL src2, immL_M1 m1) %{
10847 match(Set dst (AndL src1 (XorL src2 m1)));
10848 ins_cost(INSN_COST);
10849 format %{ "bic $dst, $src1, $src2" %}
10850
10851 ins_encode %{
10852 __ bic(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg),
10855 Assembler::LSL, 0);
10856 %}
10857
10858 ins_pipe(ialu_reg_reg);
10859 %}
10860
10861 // This pattern is automatically generated from aarch64_ad.m4
10862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10863 instruct OrI_reg_not_reg(iRegINoSp dst,
10864 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10865 match(Set dst (OrI src1 (XorI src2 m1)));
10866 ins_cost(INSN_COST);
10867 format %{ "ornw $dst, $src1, $src2" %}
10868
10869 ins_encode %{
10870 __ ornw(as_Register($dst$$reg),
10871 as_Register($src1$$reg),
10872 as_Register($src2$$reg),
10873 Assembler::LSL, 0);
10874 %}
10875
10876 ins_pipe(ialu_reg_reg);
10877 %}
10878
10879 // This pattern is automatically generated from aarch64_ad.m4
10880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10881 instruct OrL_reg_not_reg(iRegLNoSp dst,
10882 iRegL src1, iRegL src2, immL_M1 m1) %{
10883 match(Set dst (OrL src1 (XorL src2 m1)));
10884 ins_cost(INSN_COST);
10885 format %{ "orn $dst, $src1, $src2" %}
10886
10887 ins_encode %{
10888 __ orn(as_Register($dst$$reg),
10889 as_Register($src1$$reg),
10890 as_Register($src2$$reg),
10891 Assembler::LSL, 0);
10892 %}
10893
10894 ins_pipe(ialu_reg_reg);
10895 %}
10896
10897 // This pattern is automatically generated from aarch64_ad.m4
10898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10899 instruct XorI_reg_not_reg(iRegINoSp dst,
10900 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10901 match(Set dst (XorI m1 (XorI src2 src1)));
10902 ins_cost(INSN_COST);
10903 format %{ "eonw $dst, $src1, $src2" %}
10904
10905 ins_encode %{
10906 __ eonw(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::LSL, 0);
10910 %}
10911
10912 ins_pipe(ialu_reg_reg);
10913 %}
10914
10915 // This pattern is automatically generated from aarch64_ad.m4
10916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10917 instruct XorL_reg_not_reg(iRegLNoSp dst,
10918 iRegL src1, iRegL src2, immL_M1 m1) %{
10919 match(Set dst (XorL m1 (XorL src2 src1)));
10920 ins_cost(INSN_COST);
10921 format %{ "eon $dst, $src1, $src2" %}
10922
10923 ins_encode %{
10924 __ eon(as_Register($dst$$reg),
10925 as_Register($src1$$reg),
10926 as_Register($src2$$reg),
10927 Assembler::LSL, 0);
10928 %}
10929
10930 ins_pipe(ialu_reg_reg);
10931 %}
10932
10933 // This pattern is automatically generated from aarch64_ad.m4
10934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10935 // val & (-1 ^ (val >>> shift)) ==> bicw
10936 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10937 iRegIorL2I src1, iRegIorL2I src2,
10938 immI src3, immI_M1 src4) %{
10939 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10940 ins_cost(1.9 * INSN_COST);
10941 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10942
10943 ins_encode %{
10944 __ bicw(as_Register($dst$$reg),
10945 as_Register($src1$$reg),
10946 as_Register($src2$$reg),
10947 Assembler::LSR,
10948 $src3$$constant & 0x1f);
10949 %}
10950
10951 ins_pipe(ialu_reg_reg_shift);
10952 %}
10953
10954 // This pattern is automatically generated from aarch64_ad.m4
10955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10956 // val & (-1 ^ (val >>> shift)) ==> bic
10957 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10958 iRegL src1, iRegL src2,
10959 immI src3, immL_M1 src4) %{
10960 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10961 ins_cost(1.9 * INSN_COST);
10962 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10963
10964 ins_encode %{
10965 __ bic(as_Register($dst$$reg),
10966 as_Register($src1$$reg),
10967 as_Register($src2$$reg),
10968 Assembler::LSR,
10969 $src3$$constant & 0x3f);
10970 %}
10971
10972 ins_pipe(ialu_reg_reg_shift);
10973 %}
10974
10975 // This pattern is automatically generated from aarch64_ad.m4
10976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10977 // val & (-1 ^ (val >> shift)) ==> bicw
10978 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10979 iRegIorL2I src1, iRegIorL2I src2,
10980 immI src3, immI_M1 src4) %{
10981 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10982 ins_cost(1.9 * INSN_COST);
10983 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10984
10985 ins_encode %{
10986 __ bicw(as_Register($dst$$reg),
10987 as_Register($src1$$reg),
10988 as_Register($src2$$reg),
10989 Assembler::ASR,
10990 $src3$$constant & 0x1f);
10991 %}
10992
10993 ins_pipe(ialu_reg_reg_shift);
10994 %}
10995
10996 // This pattern is automatically generated from aarch64_ad.m4
10997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10998 // val & (-1 ^ (val >> shift)) ==> bic
10999 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11000 iRegL src1, iRegL src2,
11001 immI src3, immL_M1 src4) %{
11002 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11003 ins_cost(1.9 * INSN_COST);
11004 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11005
11006 ins_encode %{
11007 __ bic(as_Register($dst$$reg),
11008 as_Register($src1$$reg),
11009 as_Register($src2$$reg),
11010 Assembler::ASR,
11011 $src3$$constant & 0x3f);
11012 %}
11013
11014 ins_pipe(ialu_reg_reg_shift);
11015 %}
11016
11017 // This pattern is automatically generated from aarch64_ad.m4
11018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11019 // val & (-1 ^ (val ror shift)) ==> bicw
11020 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11021 iRegIorL2I src1, iRegIorL2I src2,
11022 immI src3, immI_M1 src4) %{
11023 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11024 ins_cost(1.9 * INSN_COST);
11025 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11026
11027 ins_encode %{
11028 __ bicw(as_Register($dst$$reg),
11029 as_Register($src1$$reg),
11030 as_Register($src2$$reg),
11031 Assembler::ROR,
11032 $src3$$constant & 0x1f);
11033 %}
11034
11035 ins_pipe(ialu_reg_reg_shift);
11036 %}
11037
11038 // This pattern is automatically generated from aarch64_ad.m4
11039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11040 // val & (-1 ^ (val ror shift)) ==> bic
11041 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11042 iRegL src1, iRegL src2,
11043 immI src3, immL_M1 src4) %{
11044 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11045 ins_cost(1.9 * INSN_COST);
11046 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11047
11048 ins_encode %{
11049 __ bic(as_Register($dst$$reg),
11050 as_Register($src1$$reg),
11051 as_Register($src2$$reg),
11052 Assembler::ROR,
11053 $src3$$constant & 0x3f);
11054 %}
11055
11056 ins_pipe(ialu_reg_reg_shift);
11057 %}
11058
11059 // This pattern is automatically generated from aarch64_ad.m4
11060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11061 // val & (-1 ^ (val << shift)) ==> bicw
11062 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11063 iRegIorL2I src1, iRegIorL2I src2,
11064 immI src3, immI_M1 src4) %{
11065 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11066 ins_cost(1.9 * INSN_COST);
11067 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11068
11069 ins_encode %{
11070 __ bicw(as_Register($dst$$reg),
11071 as_Register($src1$$reg),
11072 as_Register($src2$$reg),
11073 Assembler::LSL,
11074 $src3$$constant & 0x1f);
11075 %}
11076
11077 ins_pipe(ialu_reg_reg_shift);
11078 %}
11079
11080 // This pattern is automatically generated from aarch64_ad.m4
11081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11082 // val & (-1 ^ (val << shift)) ==> bic
11083 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11084 iRegL src1, iRegL src2,
11085 immI src3, immL_M1 src4) %{
11086 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11087 ins_cost(1.9 * INSN_COST);
11088 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11089
11090 ins_encode %{
11091 __ bic(as_Register($dst$$reg),
11092 as_Register($src1$$reg),
11093 as_Register($src2$$reg),
11094 Assembler::LSL,
11095 $src3$$constant & 0x3f);
11096 %}
11097
11098 ins_pipe(ialu_reg_reg_shift);
11099 %}
11100
11101 // This pattern is automatically generated from aarch64_ad.m4
11102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11103 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11104 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11105 iRegIorL2I src1, iRegIorL2I src2,
11106 immI src3, immI_M1 src4) %{
11107 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11108 ins_cost(1.9 * INSN_COST);
11109 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11110
11111 ins_encode %{
11112 __ eonw(as_Register($dst$$reg),
11113 as_Register($src1$$reg),
11114 as_Register($src2$$reg),
11115 Assembler::LSR,
11116 $src3$$constant & 0x1f);
11117 %}
11118
11119 ins_pipe(ialu_reg_reg_shift);
11120 %}
11121
11122 // This pattern is automatically generated from aarch64_ad.m4
11123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11124 // val ^ (-1 ^ (val >>> shift)) ==> eon
11125 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11126 iRegL src1, iRegL src2,
11127 immI src3, immL_M1 src4) %{
11128 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11129 ins_cost(1.9 * INSN_COST);
11130 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11131
11132 ins_encode %{
11133 __ eon(as_Register($dst$$reg),
11134 as_Register($src1$$reg),
11135 as_Register($src2$$reg),
11136 Assembler::LSR,
11137 $src3$$constant & 0x3f);
11138 %}
11139
11140 ins_pipe(ialu_reg_reg_shift);
11141 %}
11142
11143 // This pattern is automatically generated from aarch64_ad.m4
11144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11145 // val ^ (-1 ^ (val >> shift)) ==> eonw
11146 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11147 iRegIorL2I src1, iRegIorL2I src2,
11148 immI src3, immI_M1 src4) %{
11149 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11150 ins_cost(1.9 * INSN_COST);
11151 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11152
11153 ins_encode %{
11154 __ eonw(as_Register($dst$$reg),
11155 as_Register($src1$$reg),
11156 as_Register($src2$$reg),
11157 Assembler::ASR,
11158 $src3$$constant & 0x1f);
11159 %}
11160
11161 ins_pipe(ialu_reg_reg_shift);
11162 %}
11163
11164 // This pattern is automatically generated from aarch64_ad.m4
11165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11166 // val ^ (-1 ^ (val >> shift)) ==> eon
11167 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11168 iRegL src1, iRegL src2,
11169 immI src3, immL_M1 src4) %{
11170 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11171 ins_cost(1.9 * INSN_COST);
11172 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11173
11174 ins_encode %{
11175 __ eon(as_Register($dst$$reg),
11176 as_Register($src1$$reg),
11177 as_Register($src2$$reg),
11178 Assembler::ASR,
11179 $src3$$constant & 0x3f);
11180 %}
11181
11182 ins_pipe(ialu_reg_reg_shift);
11183 %}
11184
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 // val ^ (-1 ^ (val ror shift)) ==> eonw
11188 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11189 iRegIorL2I src1, iRegIorL2I src2,
11190 immI src3, immI_M1 src4) %{
11191 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11192 ins_cost(1.9 * INSN_COST);
11193 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11194
11195 ins_encode %{
11196 __ eonw(as_Register($dst$$reg),
11197 as_Register($src1$$reg),
11198 as_Register($src2$$reg),
11199 Assembler::ROR,
11200 $src3$$constant & 0x1f);
11201 %}
11202
11203 ins_pipe(ialu_reg_reg_shift);
11204 %}
11205
11206 // This pattern is automatically generated from aarch64_ad.m4
11207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11208 // val ^ (-1 ^ (val ror shift)) ==> eon
11209 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11210 iRegL src1, iRegL src2,
11211 immI src3, immL_M1 src4) %{
11212 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11213 ins_cost(1.9 * INSN_COST);
11214 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11215
11216 ins_encode %{
11217 __ eon(as_Register($dst$$reg),
11218 as_Register($src1$$reg),
11219 as_Register($src2$$reg),
11220 Assembler::ROR,
11221 $src3$$constant & 0x3f);
11222 %}
11223
11224 ins_pipe(ialu_reg_reg_shift);
11225 %}
11226
11227 // This pattern is automatically generated from aarch64_ad.m4
11228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11229 // val ^ (-1 ^ (val << shift)) ==> eonw
11230 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11231 iRegIorL2I src1, iRegIorL2I src2,
11232 immI src3, immI_M1 src4) %{
11233 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11234 ins_cost(1.9 * INSN_COST);
11235 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11236
11237 ins_encode %{
11238 __ eonw(as_Register($dst$$reg),
11239 as_Register($src1$$reg),
11240 as_Register($src2$$reg),
11241 Assembler::LSL,
11242 $src3$$constant & 0x1f);
11243 %}
11244
11245 ins_pipe(ialu_reg_reg_shift);
11246 %}
11247
11248 // This pattern is automatically generated from aarch64_ad.m4
11249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11250 // val ^ (-1 ^ (val << shift)) ==> eon
11251 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11252 iRegL src1, iRegL src2,
11253 immI src3, immL_M1 src4) %{
11254 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11255 ins_cost(1.9 * INSN_COST);
11256 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11257
11258 ins_encode %{
11259 __ eon(as_Register($dst$$reg),
11260 as_Register($src1$$reg),
11261 as_Register($src2$$reg),
11262 Assembler::LSL,
11263 $src3$$constant & 0x3f);
11264 %}
11265
11266 ins_pipe(ialu_reg_reg_shift);
11267 %}
11268
11269 // This pattern is automatically generated from aarch64_ad.m4
11270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11271 // val | (-1 ^ (val >>> shift)) ==> ornw
11272 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11273 iRegIorL2I src1, iRegIorL2I src2,
11274 immI src3, immI_M1 src4) %{
11275 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11276 ins_cost(1.9 * INSN_COST);
11277 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11278
11279 ins_encode %{
11280 __ ornw(as_Register($dst$$reg),
11281 as_Register($src1$$reg),
11282 as_Register($src2$$reg),
11283 Assembler::LSR,
11284 $src3$$constant & 0x1f);
11285 %}
11286
11287 ins_pipe(ialu_reg_reg_shift);
11288 %}
11289
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 // val | (-1 ^ (val >>> shift)) ==> orn
11293 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11294 iRegL src1, iRegL src2,
11295 immI src3, immL_M1 src4) %{
11296 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11297 ins_cost(1.9 * INSN_COST);
11298 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11299
11300 ins_encode %{
11301 __ orn(as_Register($dst$$reg),
11302 as_Register($src1$$reg),
11303 as_Register($src2$$reg),
11304 Assembler::LSR,
11305 $src3$$constant & 0x3f);
11306 %}
11307
11308 ins_pipe(ialu_reg_reg_shift);
11309 %}
11310
11311 // This pattern is automatically generated from aarch64_ad.m4
11312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11313 // val | (-1 ^ (val >> shift)) ==> ornw
11314 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11315 iRegIorL2I src1, iRegIorL2I src2,
11316 immI src3, immI_M1 src4) %{
11317 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11318 ins_cost(1.9 * INSN_COST);
11319 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11320
11321 ins_encode %{
11322 __ ornw(as_Register($dst$$reg),
11323 as_Register($src1$$reg),
11324 as_Register($src2$$reg),
11325 Assembler::ASR,
11326 $src3$$constant & 0x1f);
11327 %}
11328
11329 ins_pipe(ialu_reg_reg_shift);
11330 %}
11331
11332 // This pattern is automatically generated from aarch64_ad.m4
11333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11334 // val | (-1 ^ (val >> shift)) ==> orn
11335 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11336 iRegL src1, iRegL src2,
11337 immI src3, immL_M1 src4) %{
11338 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11339 ins_cost(1.9 * INSN_COST);
11340 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11341
11342 ins_encode %{
11343 __ orn(as_Register($dst$$reg),
11344 as_Register($src1$$reg),
11345 as_Register($src2$$reg),
11346 Assembler::ASR,
11347 $src3$$constant & 0x3f);
11348 %}
11349
11350 ins_pipe(ialu_reg_reg_shift);
11351 %}
11352
11353 // This pattern is automatically generated from aarch64_ad.m4
11354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11355 // val | (-1 ^ (val ror shift)) ==> ornw
11356 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11357 iRegIorL2I src1, iRegIorL2I src2,
11358 immI src3, immI_M1 src4) %{
11359 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11360 ins_cost(1.9 * INSN_COST);
11361 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11362
11363 ins_encode %{
11364 __ ornw(as_Register($dst$$reg),
11365 as_Register($src1$$reg),
11366 as_Register($src2$$reg),
11367 Assembler::ROR,
11368 $src3$$constant & 0x1f);
11369 %}
11370
11371 ins_pipe(ialu_reg_reg_shift);
11372 %}
11373
11374 // This pattern is automatically generated from aarch64_ad.m4
11375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11376 // val | (-1 ^ (val ror shift)) ==> orn
11377 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11378 iRegL src1, iRegL src2,
11379 immI src3, immL_M1 src4) %{
11380 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11381 ins_cost(1.9 * INSN_COST);
11382 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11383
11384 ins_encode %{
11385 __ orn(as_Register($dst$$reg),
11386 as_Register($src1$$reg),
11387 as_Register($src2$$reg),
11388 Assembler::ROR,
11389 $src3$$constant & 0x3f);
11390 %}
11391
11392 ins_pipe(ialu_reg_reg_shift);
11393 %}
11394
11395 // This pattern is automatically generated from aarch64_ad.m4
11396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11397 // val | (-1 ^ (val << shift)) ==> ornw
11398 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11399 iRegIorL2I src1, iRegIorL2I src2,
11400 immI src3, immI_M1 src4) %{
11401 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11402 ins_cost(1.9 * INSN_COST);
11403 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11404
11405 ins_encode %{
11406 __ ornw(as_Register($dst$$reg),
11407 as_Register($src1$$reg),
11408 as_Register($src2$$reg),
11409 Assembler::LSL,
11410 $src3$$constant & 0x1f);
11411 %}
11412
11413 ins_pipe(ialu_reg_reg_shift);
11414 %}
11415
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418 // val | (-1 ^ (val << shift)) ==> orn
11419 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11420 iRegL src1, iRegL src2,
11421 immI src3, immL_M1 src4) %{
11422 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11423 ins_cost(1.9 * INSN_COST);
11424 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11425
11426 ins_encode %{
11427 __ orn(as_Register($dst$$reg),
11428 as_Register($src1$$reg),
11429 as_Register($src2$$reg),
11430 Assembler::LSL,
11431 $src3$$constant & 0x3f);
11432 %}
11433
11434 ins_pipe(ialu_reg_reg_shift);
11435 %}
11436
11437 // This pattern is automatically generated from aarch64_ad.m4
11438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11439 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11440 iRegIorL2I src1, iRegIorL2I src2,
11441 immI src3) %{
11442 match(Set dst (AndI src1 (URShiftI src2 src3)));
11443
11444 ins_cost(1.9 * INSN_COST);
11445 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11446
11447 ins_encode %{
11448 __ andw(as_Register($dst$$reg),
11449 as_Register($src1$$reg),
11450 as_Register($src2$$reg),
11451 Assembler::LSR,
11452 $src3$$constant & 0x1f);
11453 %}
11454
11455 ins_pipe(ialu_reg_reg_shift);
11456 %}
11457
11458 // This pattern is automatically generated from aarch64_ad.m4
11459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11460 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11461 iRegL src1, iRegL src2,
11462 immI src3) %{
11463 match(Set dst (AndL src1 (URShiftL src2 src3)));
11464
11465 ins_cost(1.9 * INSN_COST);
11466 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11467
11468 ins_encode %{
11469 __ andr(as_Register($dst$$reg),
11470 as_Register($src1$$reg),
11471 as_Register($src2$$reg),
11472 Assembler::LSR,
11473 $src3$$constant & 0x3f);
11474 %}
11475
11476 ins_pipe(ialu_reg_reg_shift);
11477 %}
11478
11479 // This pattern is automatically generated from aarch64_ad.m4
11480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11481 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11482 iRegIorL2I src1, iRegIorL2I src2,
11483 immI src3) %{
11484 match(Set dst (AndI src1 (RShiftI src2 src3)));
11485
11486 ins_cost(1.9 * INSN_COST);
11487 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11488
11489 ins_encode %{
11490 __ andw(as_Register($dst$$reg),
11491 as_Register($src1$$reg),
11492 as_Register($src2$$reg),
11493 Assembler::ASR,
11494 $src3$$constant & 0x1f);
11495 %}
11496
11497 ins_pipe(ialu_reg_reg_shift);
11498 %}
11499
11500 // This pattern is automatically generated from aarch64_ad.m4
11501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11502 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11503 iRegL src1, iRegL src2,
11504 immI src3) %{
11505 match(Set dst (AndL src1 (RShiftL src2 src3)));
11506
11507 ins_cost(1.9 * INSN_COST);
11508 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11509
11510 ins_encode %{
11511 __ andr(as_Register($dst$$reg),
11512 as_Register($src1$$reg),
11513 as_Register($src2$$reg),
11514 Assembler::ASR,
11515 $src3$$constant & 0x3f);
11516 %}
11517
11518 ins_pipe(ialu_reg_reg_shift);
11519 %}
11520
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11524 iRegIorL2I src1, iRegIorL2I src2,
11525 immI src3) %{
11526 match(Set dst (AndI src1 (LShiftI src2 src3)));
11527
11528 ins_cost(1.9 * INSN_COST);
11529 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11530
11531 ins_encode %{
11532 __ andw(as_Register($dst$$reg),
11533 as_Register($src1$$reg),
11534 as_Register($src2$$reg),
11535 Assembler::LSL,
11536 $src3$$constant & 0x1f);
11537 %}
11538
11539 ins_pipe(ialu_reg_reg_shift);
11540 %}
11541
11542 // This pattern is automatically generated from aarch64_ad.m4
11543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11544 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11545 iRegL src1, iRegL src2,
11546 immI src3) %{
11547 match(Set dst (AndL src1 (LShiftL src2 src3)));
11548
11549 ins_cost(1.9 * INSN_COST);
11550 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11551
11552 ins_encode %{
11553 __ andr(as_Register($dst$$reg),
11554 as_Register($src1$$reg),
11555 as_Register($src2$$reg),
11556 Assembler::LSL,
11557 $src3$$constant & 0x3f);
11558 %}
11559
11560 ins_pipe(ialu_reg_reg_shift);
11561 %}
11562
11563 // This pattern is automatically generated from aarch64_ad.m4
11564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11565 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11566 iRegIorL2I src1, iRegIorL2I src2,
11567 immI src3) %{
11568 match(Set dst (AndI src1 (RotateRight src2 src3)));
11569
11570 ins_cost(1.9 * INSN_COST);
11571 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11572
11573 ins_encode %{
11574 __ andw(as_Register($dst$$reg),
11575 as_Register($src1$$reg),
11576 as_Register($src2$$reg),
11577 Assembler::ROR,
11578 $src3$$constant & 0x1f);
11579 %}
11580
11581 ins_pipe(ialu_reg_reg_shift);
11582 %}
11583
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11587 iRegL src1, iRegL src2,
11588 immI src3) %{
11589 match(Set dst (AndL src1 (RotateRight src2 src3)));
11590
11591 ins_cost(1.9 * INSN_COST);
11592 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11593
11594 ins_encode %{
11595 __ andr(as_Register($dst$$reg),
11596 as_Register($src1$$reg),
11597 as_Register($src2$$reg),
11598 Assembler::ROR,
11599 $src3$$constant & 0x3f);
11600 %}
11601
11602 ins_pipe(ialu_reg_reg_shift);
11603 %}
11604
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11608 iRegIorL2I src1, iRegIorL2I src2,
11609 immI src3) %{
11610 match(Set dst (XorI src1 (URShiftI src2 src3)));
11611
11612 ins_cost(1.9 * INSN_COST);
11613 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11614
11615 ins_encode %{
11616 __ eorw(as_Register($dst$$reg),
11617 as_Register($src1$$reg),
11618 as_Register($src2$$reg),
11619 Assembler::LSR,
11620 $src3$$constant & 0x1f);
11621 %}
11622
11623 ins_pipe(ialu_reg_reg_shift);
11624 %}
11625
11626 // This pattern is automatically generated from aarch64_ad.m4
11627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11628 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11629 iRegL src1, iRegL src2,
11630 immI src3) %{
11631 match(Set dst (XorL src1 (URShiftL src2 src3)));
11632
11633 ins_cost(1.9 * INSN_COST);
11634 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11635
11636 ins_encode %{
11637 __ eor(as_Register($dst$$reg),
11638 as_Register($src1$$reg),
11639 as_Register($src2$$reg),
11640 Assembler::LSR,
11641 $src3$$constant & 0x3f);
11642 %}
11643
11644 ins_pipe(ialu_reg_reg_shift);
11645 %}
11646
11647 // This pattern is automatically generated from aarch64_ad.m4
11648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11649 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11650 iRegIorL2I src1, iRegIorL2I src2,
11651 immI src3) %{
11652 match(Set dst (XorI src1 (RShiftI src2 src3)));
11653
11654 ins_cost(1.9 * INSN_COST);
11655 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11656
11657 ins_encode %{
11658 __ eorw(as_Register($dst$$reg),
11659 as_Register($src1$$reg),
11660 as_Register($src2$$reg),
11661 Assembler::ASR,
11662 $src3$$constant & 0x1f);
11663 %}
11664
11665 ins_pipe(ialu_reg_reg_shift);
11666 %}
11667
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11671 iRegL src1, iRegL src2,
11672 immI src3) %{
11673 match(Set dst (XorL src1 (RShiftL src2 src3)));
11674
11675 ins_cost(1.9 * INSN_COST);
11676 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11677
11678 ins_encode %{
11679 __ eor(as_Register($dst$$reg),
11680 as_Register($src1$$reg),
11681 as_Register($src2$$reg),
11682 Assembler::ASR,
11683 $src3$$constant & 0x3f);
11684 %}
11685
11686 ins_pipe(ialu_reg_reg_shift);
11687 %}
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11692 iRegIorL2I src1, iRegIorL2I src2,
11693 immI src3) %{
11694 match(Set dst (XorI src1 (LShiftI src2 src3)));
11695
11696 ins_cost(1.9 * INSN_COST);
11697 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11698
11699 ins_encode %{
11700 __ eorw(as_Register($dst$$reg),
11701 as_Register($src1$$reg),
11702 as_Register($src2$$reg),
11703 Assembler::LSL,
11704 $src3$$constant & 0x1f);
11705 %}
11706
11707 ins_pipe(ialu_reg_reg_shift);
11708 %}
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11713 iRegL src1, iRegL src2,
11714 immI src3) %{
11715 match(Set dst (XorL src1 (LShiftL src2 src3)));
11716
11717 ins_cost(1.9 * INSN_COST);
11718 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11719
11720 ins_encode %{
11721 __ eor(as_Register($dst$$reg),
11722 as_Register($src1$$reg),
11723 as_Register($src2$$reg),
11724 Assembler::LSL,
11725 $src3$$constant & 0x3f);
11726 %}
11727
11728 ins_pipe(ialu_reg_reg_shift);
11729 %}
11730
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11734 iRegIorL2I src1, iRegIorL2I src2,
11735 immI src3) %{
11736 match(Set dst (XorI src1 (RotateRight src2 src3)));
11737
11738 ins_cost(1.9 * INSN_COST);
11739 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11740
11741 ins_encode %{
11742 __ eorw(as_Register($dst$$reg),
11743 as_Register($src1$$reg),
11744 as_Register($src2$$reg),
11745 Assembler::ROR,
11746 $src3$$constant & 0x1f);
11747 %}
11748
11749 ins_pipe(ialu_reg_reg_shift);
11750 %}
11751
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11755 iRegL src1, iRegL src2,
11756 immI src3) %{
11757 match(Set dst (XorL src1 (RotateRight src2 src3)));
11758
11759 ins_cost(1.9 * INSN_COST);
11760 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11761
11762 ins_encode %{
11763 __ eor(as_Register($dst$$reg),
11764 as_Register($src1$$reg),
11765 as_Register($src2$$reg),
11766 Assembler::ROR,
11767 $src3$$constant & 0x3f);
11768 %}
11769
11770 ins_pipe(ialu_reg_reg_shift);
11771 %}
11772
11773 // This pattern is automatically generated from aarch64_ad.m4
11774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11775 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11776 iRegIorL2I src1, iRegIorL2I src2,
11777 immI src3) %{
11778 match(Set dst (OrI src1 (URShiftI src2 src3)));
11779
11780 ins_cost(1.9 * INSN_COST);
11781 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11782
11783 ins_encode %{
11784 __ orrw(as_Register($dst$$reg),
11785 as_Register($src1$$reg),
11786 as_Register($src2$$reg),
11787 Assembler::LSR,
11788 $src3$$constant & 0x1f);
11789 %}
11790
11791 ins_pipe(ialu_reg_reg_shift);
11792 %}
11793
11794 // This pattern is automatically generated from aarch64_ad.m4
11795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11796 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11797 iRegL src1, iRegL src2,
11798 immI src3) %{
11799 match(Set dst (OrL src1 (URShiftL src2 src3)));
11800
11801 ins_cost(1.9 * INSN_COST);
11802 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11803
11804 ins_encode %{
11805 __ orr(as_Register($dst$$reg),
11806 as_Register($src1$$reg),
11807 as_Register($src2$$reg),
11808 Assembler::LSR,
11809 $src3$$constant & 0x3f);
11810 %}
11811
11812 ins_pipe(ialu_reg_reg_shift);
11813 %}
11814
11815 // This pattern is automatically generated from aarch64_ad.m4
11816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11817 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11818 iRegIorL2I src1, iRegIorL2I src2,
11819 immI src3) %{
11820 match(Set dst (OrI src1 (RShiftI src2 src3)));
11821
11822 ins_cost(1.9 * INSN_COST);
11823 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11824
11825 ins_encode %{
11826 __ orrw(as_Register($dst$$reg),
11827 as_Register($src1$$reg),
11828 as_Register($src2$$reg),
11829 Assembler::ASR,
11830 $src3$$constant & 0x1f);
11831 %}
11832
11833 ins_pipe(ialu_reg_reg_shift);
11834 %}
11835
11836 // This pattern is automatically generated from aarch64_ad.m4
11837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11838 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11839 iRegL src1, iRegL src2,
11840 immI src3) %{
11841 match(Set dst (OrL src1 (RShiftL src2 src3)));
11842
11843 ins_cost(1.9 * INSN_COST);
11844 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11845
11846 ins_encode %{
11847 __ orr(as_Register($dst$$reg),
11848 as_Register($src1$$reg),
11849 as_Register($src2$$reg),
11850 Assembler::ASR,
11851 $src3$$constant & 0x3f);
11852 %}
11853
11854 ins_pipe(ialu_reg_reg_shift);
11855 %}
11856
11857 // This pattern is automatically generated from aarch64_ad.m4
11858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11859 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11860 iRegIorL2I src1, iRegIorL2I src2,
11861 immI src3) %{
11862 match(Set dst (OrI src1 (LShiftI src2 src3)));
11863
11864 ins_cost(1.9 * INSN_COST);
11865 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11866
11867 ins_encode %{
11868 __ orrw(as_Register($dst$$reg),
11869 as_Register($src1$$reg),
11870 as_Register($src2$$reg),
11871 Assembler::LSL,
11872 $src3$$constant & 0x1f);
11873 %}
11874
11875 ins_pipe(ialu_reg_reg_shift);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11881 iRegL src1, iRegL src2,
11882 immI src3) %{
11883 match(Set dst (OrL src1 (LShiftL src2 src3)));
11884
11885 ins_cost(1.9 * INSN_COST);
11886 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11887
11888 ins_encode %{
11889 __ orr(as_Register($dst$$reg),
11890 as_Register($src1$$reg),
11891 as_Register($src2$$reg),
11892 Assembler::LSL,
11893 $src3$$constant & 0x3f);
11894 %}
11895
11896 ins_pipe(ialu_reg_reg_shift);
11897 %}
11898
11899 // This pattern is automatically generated from aarch64_ad.m4
11900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11901 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11902 iRegIorL2I src1, iRegIorL2I src2,
11903 immI src3) %{
11904 match(Set dst (OrI src1 (RotateRight src2 src3)));
11905
11906 ins_cost(1.9 * INSN_COST);
11907 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11908
11909 ins_encode %{
11910 __ orrw(as_Register($dst$$reg),
11911 as_Register($src1$$reg),
11912 as_Register($src2$$reg),
11913 Assembler::ROR,
11914 $src3$$constant & 0x1f);
11915 %}
11916
11917 ins_pipe(ialu_reg_reg_shift);
11918 %}
11919
11920 // This pattern is automatically generated from aarch64_ad.m4
11921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11922 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11923 iRegL src1, iRegL src2,
11924 immI src3) %{
11925 match(Set dst (OrL src1 (RotateRight src2 src3)));
11926
11927 ins_cost(1.9 * INSN_COST);
11928 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11929
11930 ins_encode %{
11931 __ orr(as_Register($dst$$reg),
11932 as_Register($src1$$reg),
11933 as_Register($src2$$reg),
11934 Assembler::ROR,
11935 $src3$$constant & 0x3f);
11936 %}
11937
11938 ins_pipe(ialu_reg_reg_shift);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11944 iRegIorL2I src1, iRegIorL2I src2,
11945 immI src3) %{
11946 match(Set dst (AddI src1 (URShiftI src2 src3)));
11947
11948 ins_cost(1.9 * INSN_COST);
11949 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11950
11951 ins_encode %{
11952 __ addw(as_Register($dst$$reg),
11953 as_Register($src1$$reg),
11954 as_Register($src2$$reg),
11955 Assembler::LSR,
11956 $src3$$constant & 0x1f);
11957 %}
11958
11959 ins_pipe(ialu_reg_reg_shift);
11960 %}
11961
11962 // This pattern is automatically generated from aarch64_ad.m4
11963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11964 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11965 iRegL src1, iRegL src2,
11966 immI src3) %{
11967 match(Set dst (AddL src1 (URShiftL src2 src3)));
11968
11969 ins_cost(1.9 * INSN_COST);
11970 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11971
11972 ins_encode %{
11973 __ add(as_Register($dst$$reg),
11974 as_Register($src1$$reg),
11975 as_Register($src2$$reg),
11976 Assembler::LSR,
11977 $src3$$constant & 0x3f);
11978 %}
11979
11980 ins_pipe(ialu_reg_reg_shift);
11981 %}
11982
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11986 iRegIorL2I src1, iRegIorL2I src2,
11987 immI src3) %{
11988 match(Set dst (AddI src1 (RShiftI src2 src3)));
11989
11990 ins_cost(1.9 * INSN_COST);
11991 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11992
11993 ins_encode %{
11994 __ addw(as_Register($dst$$reg),
11995 as_Register($src1$$reg),
11996 as_Register($src2$$reg),
11997 Assembler::ASR,
11998 $src3$$constant & 0x1f);
11999 %}
12000
12001 ins_pipe(ialu_reg_reg_shift);
12002 %}
12003
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12007 iRegL src1, iRegL src2,
12008 immI src3) %{
12009 match(Set dst (AddL src1 (RShiftL src2 src3)));
12010
12011 ins_cost(1.9 * INSN_COST);
12012 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12013
12014 ins_encode %{
12015 __ add(as_Register($dst$$reg),
12016 as_Register($src1$$reg),
12017 as_Register($src2$$reg),
12018 Assembler::ASR,
12019 $src3$$constant & 0x3f);
12020 %}
12021
12022 ins_pipe(ialu_reg_reg_shift);
12023 %}
12024
12025 // This pattern is automatically generated from aarch64_ad.m4
12026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12027 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12028 iRegIorL2I src1, iRegIorL2I src2,
12029 immI src3) %{
12030 match(Set dst (AddI src1 (LShiftI src2 src3)));
12031
12032 ins_cost(1.9 * INSN_COST);
12033 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12034
12035 ins_encode %{
12036 __ addw(as_Register($dst$$reg),
12037 as_Register($src1$$reg),
12038 as_Register($src2$$reg),
12039 Assembler::LSL,
12040 $src3$$constant & 0x1f);
12041 %}
12042
12043 ins_pipe(ialu_reg_reg_shift);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12049 iRegL src1, iRegL src2,
12050 immI src3) %{
12051 match(Set dst (AddL src1 (LShiftL src2 src3)));
12052
12053 ins_cost(1.9 * INSN_COST);
12054 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12055
12056 ins_encode %{
12057 __ add(as_Register($dst$$reg),
12058 as_Register($src1$$reg),
12059 as_Register($src2$$reg),
12060 Assembler::LSL,
12061 $src3$$constant & 0x3f);
12062 %}
12063
12064 ins_pipe(ialu_reg_reg_shift);
12065 %}
12066
12067 // This pattern is automatically generated from aarch64_ad.m4
12068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12069 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12070 iRegIorL2I src1, iRegIorL2I src2,
12071 immI src3) %{
12072 match(Set dst (SubI src1 (URShiftI src2 src3)));
12073
12074 ins_cost(1.9 * INSN_COST);
12075 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12076
12077 ins_encode %{
12078 __ subw(as_Register($dst$$reg),
12079 as_Register($src1$$reg),
12080 as_Register($src2$$reg),
12081 Assembler::LSR,
12082 $src3$$constant & 0x1f);
12083 %}
12084
12085 ins_pipe(ialu_reg_reg_shift);
12086 %}
12087
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12091 iRegL src1, iRegL src2,
12092 immI src3) %{
12093 match(Set dst (SubL src1 (URShiftL src2 src3)));
12094
12095 ins_cost(1.9 * INSN_COST);
12096 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12097
12098 ins_encode %{
12099 __ sub(as_Register($dst$$reg),
12100 as_Register($src1$$reg),
12101 as_Register($src2$$reg),
12102 Assembler::LSR,
12103 $src3$$constant & 0x3f);
12104 %}
12105
12106 ins_pipe(ialu_reg_reg_shift);
12107 %}
12108
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12112 iRegIorL2I src1, iRegIorL2I src2,
12113 immI src3) %{
12114 match(Set dst (SubI src1 (RShiftI src2 src3)));
12115
12116 ins_cost(1.9 * INSN_COST);
12117 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12118
12119 ins_encode %{
12120 __ subw(as_Register($dst$$reg),
12121 as_Register($src1$$reg),
12122 as_Register($src2$$reg),
12123 Assembler::ASR,
12124 $src3$$constant & 0x1f);
12125 %}
12126
12127 ins_pipe(ialu_reg_reg_shift);
12128 %}
12129
12130 // This pattern is automatically generated from aarch64_ad.m4
12131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12132 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12133 iRegL src1, iRegL src2,
12134 immI src3) %{
12135 match(Set dst (SubL src1 (RShiftL src2 src3)));
12136
12137 ins_cost(1.9 * INSN_COST);
12138 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12139
12140 ins_encode %{
12141 __ sub(as_Register($dst$$reg),
12142 as_Register($src1$$reg),
12143 as_Register($src2$$reg),
12144 Assembler::ASR,
12145 $src3$$constant & 0x3f);
12146 %}
12147
12148 ins_pipe(ialu_reg_reg_shift);
12149 %}
12150
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12154 iRegIorL2I src1, iRegIorL2I src2,
12155 immI src3) %{
12156 match(Set dst (SubI src1 (LShiftI src2 src3)));
12157
12158 ins_cost(1.9 * INSN_COST);
12159 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12160
12161 ins_encode %{
12162 __ subw(as_Register($dst$$reg),
12163 as_Register($src1$$reg),
12164 as_Register($src2$$reg),
12165 Assembler::LSL,
12166 $src3$$constant & 0x1f);
12167 %}
12168
12169 ins_pipe(ialu_reg_reg_shift);
12170 %}
12171
12172 // This pattern is automatically generated from aarch64_ad.m4
12173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12174 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12175 iRegL src1, iRegL src2,
12176 immI src3) %{
12177 match(Set dst (SubL src1 (LShiftL src2 src3)));
12178
12179 ins_cost(1.9 * INSN_COST);
12180 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12181
12182 ins_encode %{
12183 __ sub(as_Register($dst$$reg),
12184 as_Register($src1$$reg),
12185 as_Register($src2$$reg),
12186 Assembler::LSL,
12187 $src3$$constant & 0x3f);
12188 %}
12189
12190 ins_pipe(ialu_reg_reg_shift);
12191 %}
12192
12193 // This pattern is automatically generated from aarch64_ad.m4
12194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12195
12196 // Shift Left followed by Shift Right.
12197 // This idiom is used by the compiler for the i2b bytecode etc.
12198 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12199 %{
12200 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12201 ins_cost(INSN_COST * 2);
12202 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12203 ins_encode %{
12204 int lshift = $lshift_count$$constant & 63;
12205 int rshift = $rshift_count$$constant & 63;
12206 int s = 63 - lshift;
12207 int r = (rshift - lshift) & 63;
12208 __ sbfm(as_Register($dst$$reg),
12209 as_Register($src$$reg),
12210 r, s);
12211 %}
12212
12213 ins_pipe(ialu_reg_shift);
12214 %}
12215
12216 // This pattern is automatically generated from aarch64_ad.m4
12217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12218
12219 // Shift Left followed by Shift Right.
12220 // This idiom is used by the compiler for the i2b bytecode etc.
12221 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12222 %{
12223 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12224 ins_cost(INSN_COST * 2);
12225 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12226 ins_encode %{
12227 int lshift = $lshift_count$$constant & 31;
12228 int rshift = $rshift_count$$constant & 31;
12229 int s = 31 - lshift;
12230 int r = (rshift - lshift) & 31;
12231 __ sbfmw(as_Register($dst$$reg),
12232 as_Register($src$$reg),
12233 r, s);
12234 %}
12235
12236 ins_pipe(ialu_reg_shift);
12237 %}
12238
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241
12242 // Shift Left followed by Shift Right.
12243 // This idiom is used by the compiler for the i2b bytecode etc.
12244 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12245 %{
12246 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12247 ins_cost(INSN_COST * 2);
12248 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12249 ins_encode %{
12250 int lshift = $lshift_count$$constant & 63;
12251 int rshift = $rshift_count$$constant & 63;
12252 int s = 63 - lshift;
12253 int r = (rshift - lshift) & 63;
12254 __ ubfm(as_Register($dst$$reg),
12255 as_Register($src$$reg),
12256 r, s);
12257 %}
12258
12259 ins_pipe(ialu_reg_shift);
12260 %}
12261
12262 // This pattern is automatically generated from aarch64_ad.m4
12263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12264
12265 // Shift Left followed by Shift Right.
12266 // This idiom is used by the compiler for the i2b bytecode etc.
12267 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12268 %{
12269 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12270 ins_cost(INSN_COST * 2);
12271 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12272 ins_encode %{
12273 int lshift = $lshift_count$$constant & 31;
12274 int rshift = $rshift_count$$constant & 31;
12275 int s = 31 - lshift;
12276 int r = (rshift - lshift) & 31;
12277 __ ubfmw(as_Register($dst$$reg),
12278 as_Register($src$$reg),
12279 r, s);
12280 %}
12281
12282 ins_pipe(ialu_reg_shift);
12283 %}
12284
12285 // Bitfield extract with shift & mask
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12290 %{
12291 match(Set dst (AndI (URShiftI src rshift) mask));
12292 // Make sure we are not going to exceed what ubfxw can do.
12293 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12294
12295 ins_cost(INSN_COST);
12296 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12297 ins_encode %{
12298 int rshift = $rshift$$constant & 31;
12299 intptr_t mask = $mask$$constant;
12300 int width = exact_log2(mask+1);
12301 __ ubfxw(as_Register($dst$$reg),
12302 as_Register($src$$reg), rshift, width);
12303 %}
12304 ins_pipe(ialu_reg_shift);
12305 %}
12306
12307 // This pattern is automatically generated from aarch64_ad.m4
12308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12309 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12310 %{
12311 match(Set dst (AndL (URShiftL src rshift) mask));
12312 // Make sure we are not going to exceed what ubfx can do.
12313 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12314
12315 ins_cost(INSN_COST);
12316 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12317 ins_encode %{
12318 int rshift = $rshift$$constant & 63;
12319 intptr_t mask = $mask$$constant;
12320 int width = exact_log2_long(mask+1);
12321 __ ubfx(as_Register($dst$$reg),
12322 as_Register($src$$reg), rshift, width);
12323 %}
12324 ins_pipe(ialu_reg_shift);
12325 %}
12326
12327
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330
12331 // We can use ubfx when extending an And with a mask when we know mask
12332 // is positive. We know that because immI_bitmask guarantees it.
12333 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12334 %{
12335 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12336 // Make sure we are not going to exceed what ubfxw can do.
12337 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12338
12339 ins_cost(INSN_COST * 2);
12340 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12341 ins_encode %{
12342 int rshift = $rshift$$constant & 31;
12343 intptr_t mask = $mask$$constant;
12344 int width = exact_log2(mask+1);
12345 __ ubfx(as_Register($dst$$reg),
12346 as_Register($src$$reg), rshift, width);
12347 %}
12348 ins_pipe(ialu_reg_shift);
12349 %}
12350
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354
12355 // We can use ubfiz when masking by a positive number and then left shifting the result.
12356 // We know that the mask is positive because immI_bitmask guarantees it.
12357 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12358 %{
12359 match(Set dst (LShiftI (AndI src mask) lshift));
12360 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12361
12362 ins_cost(INSN_COST);
12363 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12364 ins_encode %{
12365 int lshift = $lshift$$constant & 31;
12366 intptr_t mask = $mask$$constant;
12367 int width = exact_log2(mask+1);
12368 __ ubfizw(as_Register($dst$$reg),
12369 as_Register($src$$reg), lshift, width);
12370 %}
12371 ins_pipe(ialu_reg_shift);
12372 %}
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376
12377 // We can use ubfiz when masking by a positive number and then left shifting the result.
12378 // We know that the mask is positive because immL_bitmask guarantees it.
12379 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12380 %{
12381 match(Set dst (LShiftL (AndL src mask) lshift));
12382 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12383
12384 ins_cost(INSN_COST);
12385 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12386 ins_encode %{
12387 int lshift = $lshift$$constant & 63;
12388 intptr_t mask = $mask$$constant;
12389 int width = exact_log2_long(mask+1);
12390 __ ubfiz(as_Register($dst$$reg),
12391 as_Register($src$$reg), lshift, width);
12392 %}
12393 ins_pipe(ialu_reg_shift);
12394 %}
12395
12396 // This pattern is automatically generated from aarch64_ad.m4
12397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12398
12399 // We can use ubfiz when masking by a positive number and then left shifting the result.
12400 // We know that the mask is positive because immI_bitmask guarantees it.
12401 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12402 %{
12403 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12404 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12405
12406 ins_cost(INSN_COST);
12407 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12408 ins_encode %{
12409 int lshift = $lshift$$constant & 31;
12410 intptr_t mask = $mask$$constant;
12411 int width = exact_log2(mask+1);
12412 __ ubfizw(as_Register($dst$$reg),
12413 as_Register($src$$reg), lshift, width);
12414 %}
12415 ins_pipe(ialu_reg_shift);
12416 %}
12417
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420
12421 // We can use ubfiz when masking by a positive number and then left shifting the result.
12422 // We know that the mask is positive because immL_bitmask guarantees it.
12423 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12424 %{
12425 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12426 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12427
12428 ins_cost(INSN_COST);
12429 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12430 ins_encode %{
12431 int lshift = $lshift$$constant & 63;
12432 intptr_t mask = $mask$$constant;
12433 int width = exact_log2_long(mask+1);
12434 __ ubfiz(as_Register($dst$$reg),
12435 as_Register($src$$reg), lshift, width);
12436 %}
12437 ins_pipe(ialu_reg_shift);
12438 %}
12439
12440
12441 // This pattern is automatically generated from aarch64_ad.m4
12442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12443
12444 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12445 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12446 %{
12447 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12448 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12449
12450 ins_cost(INSN_COST);
12451 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12452 ins_encode %{
12453 int lshift = $lshift$$constant & 63;
12454 intptr_t mask = $mask$$constant;
12455 int width = exact_log2(mask+1);
12456 __ ubfiz(as_Register($dst$$reg),
12457 as_Register($src$$reg), lshift, width);
12458 %}
12459 ins_pipe(ialu_reg_shift);
12460 %}
12461
12462 // This pattern is automatically generated from aarch64_ad.m4
12463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12464
12465 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12466 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12467 %{
12468 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12469 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12470
12471 ins_cost(INSN_COST);
12472 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12473 ins_encode %{
12474 int lshift = $lshift$$constant & 31;
12475 intptr_t mask = $mask$$constant;
12476 int width = exact_log2(mask+1);
12477 __ ubfiz(as_Register($dst$$reg),
12478 as_Register($src$$reg), lshift, width);
12479 %}
12480 ins_pipe(ialu_reg_shift);
12481 %}
12482
12483 // This pattern is automatically generated from aarch64_ad.m4
12484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12485
12486 // Can skip int2long conversions after AND with small bitmask
12487 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12488 %{
12489 match(Set dst (ConvI2L (AndI src msk)));
12490 ins_cost(INSN_COST);
12491 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12492 ins_encode %{
12493 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12494 %}
12495 ins_pipe(ialu_reg_shift);
12496 %}
12497
12498
12499 // Rotations
12500
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12504 %{
12505 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12506 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12507
12508 ins_cost(INSN_COST);
12509 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12510
12511 ins_encode %{
12512 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12513 $rshift$$constant & 63);
12514 %}
12515 ins_pipe(ialu_reg_reg_extr);
12516 %}
12517
12518
12519 // This pattern is automatically generated from aarch64_ad.m4
12520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12521 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12522 %{
12523 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12524 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12525
12526 ins_cost(INSN_COST);
12527 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12528
12529 ins_encode %{
12530 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12531 $rshift$$constant & 31);
12532 %}
12533 ins_pipe(ialu_reg_reg_extr);
12534 %}
12535
12536
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12540 %{
12541 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12542 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12543
12544 ins_cost(INSN_COST);
12545 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12546
12547 ins_encode %{
12548 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12549 $rshift$$constant & 63);
12550 %}
12551 ins_pipe(ialu_reg_reg_extr);
12552 %}
12553
12554
12555 // This pattern is automatically generated from aarch64_ad.m4
12556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12557 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12558 %{
12559 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12560 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12561
12562 ins_cost(INSN_COST);
12563 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12564
12565 ins_encode %{
12566 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12567 $rshift$$constant & 31);
12568 %}
12569 ins_pipe(ialu_reg_reg_extr);
12570 %}
12571
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12575 %{
12576 match(Set dst (RotateRight src shift));
12577
12578 ins_cost(INSN_COST);
12579 format %{ "ror $dst, $src, $shift" %}
12580
12581 ins_encode %{
12582 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12583 $shift$$constant & 0x1f);
12584 %}
12585 ins_pipe(ialu_reg_reg_vshift);
12586 %}
12587
12588 // This pattern is automatically generated from aarch64_ad.m4
12589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12590 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12591 %{
12592 match(Set dst (RotateRight src shift));
12593
12594 ins_cost(INSN_COST);
12595 format %{ "ror $dst, $src, $shift" %}
12596
12597 ins_encode %{
12598 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12599 $shift$$constant & 0x3f);
12600 %}
12601 ins_pipe(ialu_reg_reg_vshift);
12602 %}
12603
12604 // This pattern is automatically generated from aarch64_ad.m4
12605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12606 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12607 %{
12608 match(Set dst (RotateRight src shift));
12609
12610 ins_cost(INSN_COST);
12611 format %{ "ror $dst, $src, $shift" %}
12612
12613 ins_encode %{
12614 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12615 %}
12616 ins_pipe(ialu_reg_reg_vshift);
12617 %}
12618
12619 // This pattern is automatically generated from aarch64_ad.m4
12620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12621 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12622 %{
12623 match(Set dst (RotateRight src shift));
12624
12625 ins_cost(INSN_COST);
12626 format %{ "ror $dst, $src, $shift" %}
12627
12628 ins_encode %{
12629 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12630 %}
12631 ins_pipe(ialu_reg_reg_vshift);
12632 %}
12633
12634 // This pattern is automatically generated from aarch64_ad.m4
12635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12636 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12637 %{
12638 match(Set dst (RotateLeft src shift));
12639
12640 ins_cost(INSN_COST);
12641 format %{ "rol $dst, $src, $shift" %}
12642
12643 ins_encode %{
12644 __ subw(rscratch1, zr, as_Register($shift$$reg));
12645 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12646 %}
12647 ins_pipe(ialu_reg_reg_vshift);
12648 %}
12649
12650 // This pattern is automatically generated from aarch64_ad.m4
12651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12652 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12653 %{
12654 match(Set dst (RotateLeft src shift));
12655
12656 ins_cost(INSN_COST);
12657 format %{ "rol $dst, $src, $shift" %}
12658
12659 ins_encode %{
12660 __ subw(rscratch1, zr, as_Register($shift$$reg));
12661 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12662 %}
12663 ins_pipe(ialu_reg_reg_vshift);
12664 %}
12665
12666
12667 // Add/subtract (extended)
12668
12669 // This pattern is automatically generated from aarch64_ad.m4
12670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12671 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12672 %{
12673 match(Set dst (AddL src1 (ConvI2L src2)));
12674 ins_cost(INSN_COST);
12675 format %{ "add $dst, $src1, $src2, sxtw" %}
12676
12677 ins_encode %{
12678 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12679 as_Register($src2$$reg), ext::sxtw);
12680 %}
12681 ins_pipe(ialu_reg_reg);
12682 %}
12683
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12687 %{
12688 match(Set dst (SubL src1 (ConvI2L src2)));
12689 ins_cost(INSN_COST);
12690 format %{ "sub $dst, $src1, $src2, sxtw" %}
12691
12692 ins_encode %{
12693 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12694 as_Register($src2$$reg), ext::sxtw);
12695 %}
12696 ins_pipe(ialu_reg_reg);
12697 %}
12698
12699 // This pattern is automatically generated from aarch64_ad.m4
12700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12701 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12702 %{
12703 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12704 ins_cost(INSN_COST);
12705 format %{ "add $dst, $src1, $src2, sxth" %}
12706
12707 ins_encode %{
12708 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12709 as_Register($src2$$reg), ext::sxth);
12710 %}
12711 ins_pipe(ialu_reg_reg);
12712 %}
12713
12714 // This pattern is automatically generated from aarch64_ad.m4
12715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12716 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12717 %{
12718 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12719 ins_cost(INSN_COST);
12720 format %{ "add $dst, $src1, $src2, sxtb" %}
12721
12722 ins_encode %{
12723 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12724 as_Register($src2$$reg), ext::sxtb);
12725 %}
12726 ins_pipe(ialu_reg_reg);
12727 %}
12728
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12732 %{
12733 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12734 ins_cost(INSN_COST);
12735 format %{ "add $dst, $src1, $src2, uxtb" %}
12736
12737 ins_encode %{
12738 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12739 as_Register($src2$$reg), ext::uxtb);
12740 %}
12741 ins_pipe(ialu_reg_reg);
12742 %}
12743
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12747 %{
12748 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12749 ins_cost(INSN_COST);
12750 format %{ "add $dst, $src1, $src2, sxth" %}
12751
12752 ins_encode %{
12753 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12754 as_Register($src2$$reg), ext::sxth);
12755 %}
12756 ins_pipe(ialu_reg_reg);
12757 %}
12758
12759 // This pattern is automatically generated from aarch64_ad.m4
12760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12761 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12762 %{
12763 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12764 ins_cost(INSN_COST);
12765 format %{ "add $dst, $src1, $src2, sxtw" %}
12766
12767 ins_encode %{
12768 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12769 as_Register($src2$$reg), ext::sxtw);
12770 %}
12771 ins_pipe(ialu_reg_reg);
12772 %}
12773
12774 // This pattern is automatically generated from aarch64_ad.m4
12775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12776 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12777 %{
12778 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12779 ins_cost(INSN_COST);
12780 format %{ "add $dst, $src1, $src2, sxtb" %}
12781
12782 ins_encode %{
12783 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12784 as_Register($src2$$reg), ext::sxtb);
12785 %}
12786 ins_pipe(ialu_reg_reg);
12787 %}
12788
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12792 %{
12793 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12794 ins_cost(INSN_COST);
12795 format %{ "add $dst, $src1, $src2, uxtb" %}
12796
12797 ins_encode %{
12798 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12799 as_Register($src2$$reg), ext::uxtb);
12800 %}
12801 ins_pipe(ialu_reg_reg);
12802 %}
12803
12804 // This pattern is automatically generated from aarch64_ad.m4
12805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12806 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12807 %{
12808 match(Set dst (AddI src1 (AndI src2 mask)));
12809 ins_cost(INSN_COST);
12810 format %{ "addw $dst, $src1, $src2, uxtb" %}
12811
12812 ins_encode %{
12813 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12814 as_Register($src2$$reg), ext::uxtb);
12815 %}
12816 ins_pipe(ialu_reg_reg);
12817 %}
12818
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12822 %{
12823 match(Set dst (AddI src1 (AndI src2 mask)));
12824 ins_cost(INSN_COST);
12825 format %{ "addw $dst, $src1, $src2, uxth" %}
12826
12827 ins_encode %{
12828 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12829 as_Register($src2$$reg), ext::uxth);
12830 %}
12831 ins_pipe(ialu_reg_reg);
12832 %}
12833
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12837 %{
12838 match(Set dst (AddL src1 (AndL src2 mask)));
12839 ins_cost(INSN_COST);
12840 format %{ "add $dst, $src1, $src2, uxtb" %}
12841
12842 ins_encode %{
12843 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12844 as_Register($src2$$reg), ext::uxtb);
12845 %}
12846 ins_pipe(ialu_reg_reg);
12847 %}
12848
12849 // This pattern is automatically generated from aarch64_ad.m4
12850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12851 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12852 %{
12853 match(Set dst (AddL src1 (AndL src2 mask)));
12854 ins_cost(INSN_COST);
12855 format %{ "add $dst, $src1, $src2, uxth" %}
12856
12857 ins_encode %{
12858 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12859 as_Register($src2$$reg), ext::uxth);
12860 %}
12861 ins_pipe(ialu_reg_reg);
12862 %}
12863
12864 // This pattern is automatically generated from aarch64_ad.m4
12865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12866 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12867 %{
12868 match(Set dst (AddL src1 (AndL src2 mask)));
12869 ins_cost(INSN_COST);
12870 format %{ "add $dst, $src1, $src2, uxtw" %}
12871
12872 ins_encode %{
12873 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12874 as_Register($src2$$reg), ext::uxtw);
12875 %}
12876 ins_pipe(ialu_reg_reg);
12877 %}
12878
12879 // This pattern is automatically generated from aarch64_ad.m4
12880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12881 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12882 %{
12883 match(Set dst (SubI src1 (AndI src2 mask)));
12884 ins_cost(INSN_COST);
12885 format %{ "subw $dst, $src1, $src2, uxtb" %}
12886
12887 ins_encode %{
12888 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12889 as_Register($src2$$reg), ext::uxtb);
12890 %}
12891 ins_pipe(ialu_reg_reg);
12892 %}
12893
12894 // This pattern is automatically generated from aarch64_ad.m4
12895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12897 %{
12898 match(Set dst (SubI src1 (AndI src2 mask)));
12899 ins_cost(INSN_COST);
12900 format %{ "subw $dst, $src1, $src2, uxth" %}
12901
12902 ins_encode %{
12903 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12904 as_Register($src2$$reg), ext::uxth);
12905 %}
12906 ins_pipe(ialu_reg_reg);
12907 %}
12908
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12912 %{
12913 match(Set dst (SubL src1 (AndL src2 mask)));
12914 ins_cost(INSN_COST);
12915 format %{ "sub $dst, $src1, $src2, uxtb" %}
12916
12917 ins_encode %{
12918 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12919 as_Register($src2$$reg), ext::uxtb);
12920 %}
12921 ins_pipe(ialu_reg_reg);
12922 %}
12923
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12927 %{
12928 match(Set dst (SubL src1 (AndL src2 mask)));
12929 ins_cost(INSN_COST);
12930 format %{ "sub $dst, $src1, $src2, uxth" %}
12931
12932 ins_encode %{
12933 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12934 as_Register($src2$$reg), ext::uxth);
12935 %}
12936 ins_pipe(ialu_reg_reg);
12937 %}
12938
12939 // This pattern is automatically generated from aarch64_ad.m4
12940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12941 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12942 %{
12943 match(Set dst (SubL src1 (AndL src2 mask)));
12944 ins_cost(INSN_COST);
12945 format %{ "sub $dst, $src1, $src2, uxtw" %}
12946
12947 ins_encode %{
12948 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12949 as_Register($src2$$reg), ext::uxtw);
12950 %}
12951 ins_pipe(ialu_reg_reg);
12952 %}
12953
12954
12955 // This pattern is automatically generated from aarch64_ad.m4
12956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12957 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12958 %{
12959 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12960 ins_cost(1.9 * INSN_COST);
12961 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12962
12963 ins_encode %{
12964 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12965 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12966 %}
12967 ins_pipe(ialu_reg_reg_shift);
12968 %}
12969
12970 // This pattern is automatically generated from aarch64_ad.m4
12971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12972 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12973 %{
12974 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12975 ins_cost(1.9 * INSN_COST);
12976 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12977
12978 ins_encode %{
12979 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12980 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12981 %}
12982 ins_pipe(ialu_reg_reg_shift);
12983 %}
12984
12985 // This pattern is automatically generated from aarch64_ad.m4
12986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12987 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12988 %{
12989 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12990 ins_cost(1.9 * INSN_COST);
12991 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12992
12993 ins_encode %{
12994 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12995 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12996 %}
12997 ins_pipe(ialu_reg_reg_shift);
12998 %}
12999
13000 // This pattern is automatically generated from aarch64_ad.m4
13001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13002 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13003 %{
13004 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13005 ins_cost(1.9 * INSN_COST);
13006 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13007
13008 ins_encode %{
13009 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13010 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13011 %}
13012 ins_pipe(ialu_reg_reg_shift);
13013 %}
13014
13015 // This pattern is automatically generated from aarch64_ad.m4
13016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13017 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13018 %{
13019 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13020 ins_cost(1.9 * INSN_COST);
13021 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13022
13023 ins_encode %{
13024 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13025 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13026 %}
13027 ins_pipe(ialu_reg_reg_shift);
13028 %}
13029
13030 // This pattern is automatically generated from aarch64_ad.m4
13031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13032 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13033 %{
13034 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13035 ins_cost(1.9 * INSN_COST);
13036 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13037
13038 ins_encode %{
13039 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13040 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13041 %}
13042 ins_pipe(ialu_reg_reg_shift);
13043 %}
13044
13045 // This pattern is automatically generated from aarch64_ad.m4
13046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13047 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13048 %{
13049 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13050 ins_cost(1.9 * INSN_COST);
13051 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13052
13053 ins_encode %{
13054 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13055 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13056 %}
13057 ins_pipe(ialu_reg_reg_shift);
13058 %}
13059
13060 // This pattern is automatically generated from aarch64_ad.m4
13061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13062 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13063 %{
13064 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13065 ins_cost(1.9 * INSN_COST);
13066 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13067
13068 ins_encode %{
13069 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13070 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13071 %}
13072 ins_pipe(ialu_reg_reg_shift);
13073 %}
13074
13075 // This pattern is automatically generated from aarch64_ad.m4
13076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13077 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13078 %{
13079 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13080 ins_cost(1.9 * INSN_COST);
13081 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13082
13083 ins_encode %{
13084 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13085 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13086 %}
13087 ins_pipe(ialu_reg_reg_shift);
13088 %}
13089
13090 // This pattern is automatically generated from aarch64_ad.m4
13091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13092 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13093 %{
13094 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13095 ins_cost(1.9 * INSN_COST);
13096 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13097
13098 ins_encode %{
13099 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13100 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13101 %}
13102 ins_pipe(ialu_reg_reg_shift);
13103 %}
13104
13105 // This pattern is automatically generated from aarch64_ad.m4
13106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13107 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13108 %{
13109 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13110 ins_cost(1.9 * INSN_COST);
13111 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13112
13113 ins_encode %{
13114 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13115 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13116 %}
13117 ins_pipe(ialu_reg_reg_shift);
13118 %}
13119
13120 // This pattern is automatically generated from aarch64_ad.m4
13121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13122 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13123 %{
13124 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13125 ins_cost(1.9 * INSN_COST);
13126 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13127
13128 ins_encode %{
13129 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13130 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13131 %}
13132 ins_pipe(ialu_reg_reg_shift);
13133 %}
13134
13135 // This pattern is automatically generated from aarch64_ad.m4
13136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13137 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13138 %{
13139 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13140 ins_cost(1.9 * INSN_COST);
13141 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13142
13143 ins_encode %{
13144 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13145 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13146 %}
13147 ins_pipe(ialu_reg_reg_shift);
13148 %}
13149
13150 // This pattern is automatically generated from aarch64_ad.m4
13151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13152 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13153 %{
13154 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13155 ins_cost(1.9 * INSN_COST);
13156 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13157
13158 ins_encode %{
13159 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13160 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13161 %}
13162 ins_pipe(ialu_reg_reg_shift);
13163 %}
13164
13165 // This pattern is automatically generated from aarch64_ad.m4
13166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13167 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13168 %{
13169 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13170 ins_cost(1.9 * INSN_COST);
13171 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13172
13173 ins_encode %{
13174 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13175 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13176 %}
13177 ins_pipe(ialu_reg_reg_shift);
13178 %}
13179
13180 // This pattern is automatically generated from aarch64_ad.m4
13181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13182 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13183 %{
13184 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13185 ins_cost(1.9 * INSN_COST);
13186 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13187
13188 ins_encode %{
13189 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13190 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13191 %}
13192 ins_pipe(ialu_reg_reg_shift);
13193 %}
13194
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13198 %{
13199 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13200 ins_cost(1.9 * INSN_COST);
13201 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13202
13203 ins_encode %{
13204 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13205 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13206 %}
13207 ins_pipe(ialu_reg_reg_shift);
13208 %}
13209
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13213 %{
13214 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13215 ins_cost(1.9 * INSN_COST);
13216 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13217
13218 ins_encode %{
13219 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13220 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13221 %}
13222 ins_pipe(ialu_reg_reg_shift);
13223 %}
13224
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13228 %{
13229 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13230 ins_cost(1.9 * INSN_COST);
13231 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13232
13233 ins_encode %{
13234 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13235 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13236 %}
13237 ins_pipe(ialu_reg_reg_shift);
13238 %}
13239
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13243 %{
13244 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13245 ins_cost(1.9 * INSN_COST);
13246 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13247
13248 ins_encode %{
13249 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13250 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13251 %}
13252 ins_pipe(ialu_reg_reg_shift);
13253 %}
13254
13255 // This pattern is automatically generated from aarch64_ad.m4
13256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13257 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13258 %{
13259 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13260 ins_cost(1.9 * INSN_COST);
13261 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13262
13263 ins_encode %{
13264 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13265 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13266 %}
13267 ins_pipe(ialu_reg_reg_shift);
13268 %}
13269
13270 // This pattern is automatically generated from aarch64_ad.m4
13271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13272 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13273 %{
13274 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13275 ins_cost(1.9 * INSN_COST);
13276 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13277
13278 ins_encode %{
13279 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13280 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13281 %}
13282 ins_pipe(ialu_reg_reg_shift);
13283 %}
13284
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13288 %{
13289 effect(DEF dst, USE src1, USE src2, USE cr);
13290 ins_cost(INSN_COST * 2);
13291 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13292
13293 ins_encode %{
13294 __ cselw($dst$$Register,
13295 $src1$$Register,
13296 $src2$$Register,
13297 Assembler::LT);
13298 %}
13299 ins_pipe(icond_reg_reg);
13300 %}
13301
13302 // This pattern is automatically generated from aarch64_ad.m4
13303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13304 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13305 %{
13306 effect(DEF dst, USE src1, USE src2, USE cr);
13307 ins_cost(INSN_COST * 2);
13308 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13309
13310 ins_encode %{
13311 __ cselw($dst$$Register,
13312 $src1$$Register,
13313 $src2$$Register,
13314 Assembler::GT);
13315 %}
13316 ins_pipe(icond_reg_reg);
13317 %}
13318
13319 // This pattern is automatically generated from aarch64_ad.m4
13320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13321 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13322 %{
13323 effect(DEF dst, USE src1, USE cr);
13324 ins_cost(INSN_COST * 2);
13325 format %{ "cselw $dst, $src1, zr lt\t" %}
13326
13327 ins_encode %{
13328 __ cselw($dst$$Register,
13329 $src1$$Register,
13330 zr,
13331 Assembler::LT);
13332 %}
13333 ins_pipe(icond_reg);
13334 %}
13335
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13339 %{
13340 effect(DEF dst, USE src1, USE cr);
13341 ins_cost(INSN_COST * 2);
13342 format %{ "cselw $dst, $src1, zr gt\t" %}
13343
13344 ins_encode %{
13345 __ cselw($dst$$Register,
13346 $src1$$Register,
13347 zr,
13348 Assembler::GT);
13349 %}
13350 ins_pipe(icond_reg);
13351 %}
13352
13353 // This pattern is automatically generated from aarch64_ad.m4
13354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13355 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13356 %{
13357 effect(DEF dst, USE src1, USE cr);
13358 ins_cost(INSN_COST * 2);
13359 format %{ "csincw $dst, $src1, zr le\t" %}
13360
13361 ins_encode %{
13362 __ csincw($dst$$Register,
13363 $src1$$Register,
13364 zr,
13365 Assembler::LE);
13366 %}
13367 ins_pipe(icond_reg);
13368 %}
13369
13370 // This pattern is automatically generated from aarch64_ad.m4
13371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13372 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13373 %{
13374 effect(DEF dst, USE src1, USE cr);
13375 ins_cost(INSN_COST * 2);
13376 format %{ "csincw $dst, $src1, zr gt\t" %}
13377
13378 ins_encode %{
13379 __ csincw($dst$$Register,
13380 $src1$$Register,
13381 zr,
13382 Assembler::GT);
13383 %}
13384 ins_pipe(icond_reg);
13385 %}
13386
13387 // This pattern is automatically generated from aarch64_ad.m4
13388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13389 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13390 %{
13391 effect(DEF dst, USE src1, USE cr);
13392 ins_cost(INSN_COST * 2);
13393 format %{ "csinvw $dst, $src1, zr lt\t" %}
13394
13395 ins_encode %{
13396 __ csinvw($dst$$Register,
13397 $src1$$Register,
13398 zr,
13399 Assembler::LT);
13400 %}
13401 ins_pipe(icond_reg);
13402 %}
13403
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13407 %{
13408 effect(DEF dst, USE src1, USE cr);
13409 ins_cost(INSN_COST * 2);
13410 format %{ "csinvw $dst, $src1, zr ge\t" %}
13411
13412 ins_encode %{
13413 __ csinvw($dst$$Register,
13414 $src1$$Register,
13415 zr,
13416 Assembler::GE);
13417 %}
13418 ins_pipe(icond_reg);
13419 %}
13420
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13424 %{
13425 match(Set dst (MinI src imm));
13426 ins_cost(INSN_COST * 3);
13427 expand %{
13428 rFlagsReg cr;
13429 compI_reg_imm0(cr, src);
13430 cmovI_reg_imm0_lt(dst, src, cr);
13431 %}
13432 %}
13433
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13437 %{
13438 match(Set dst (MinI imm src));
13439 ins_cost(INSN_COST * 3);
13440 expand %{
13441 rFlagsReg cr;
13442 compI_reg_imm0(cr, src);
13443 cmovI_reg_imm0_lt(dst, src, cr);
13444 %}
13445 %}
13446
13447 // This pattern is automatically generated from aarch64_ad.m4
13448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13449 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13450 %{
13451 match(Set dst (MinI src imm));
13452 ins_cost(INSN_COST * 3);
13453 expand %{
13454 rFlagsReg cr;
13455 compI_reg_imm0(cr, src);
13456 cmovI_reg_imm1_le(dst, src, cr);
13457 %}
13458 %}
13459
13460 // This pattern is automatically generated from aarch64_ad.m4
13461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13462 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13463 %{
13464 match(Set dst (MinI imm src));
13465 ins_cost(INSN_COST * 3);
13466 expand %{
13467 rFlagsReg cr;
13468 compI_reg_imm0(cr, src);
13469 cmovI_reg_imm1_le(dst, src, cr);
13470 %}
13471 %}
13472
13473 // This pattern is automatically generated from aarch64_ad.m4
13474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13475 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13476 %{
13477 match(Set dst (MinI src imm));
13478 ins_cost(INSN_COST * 3);
13479 expand %{
13480 rFlagsReg cr;
13481 compI_reg_imm0(cr, src);
13482 cmovI_reg_immM1_lt(dst, src, cr);
13483 %}
13484 %}
13485
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13489 %{
13490 match(Set dst (MinI imm src));
13491 ins_cost(INSN_COST * 3);
13492 expand %{
13493 rFlagsReg cr;
13494 compI_reg_imm0(cr, src);
13495 cmovI_reg_immM1_lt(dst, src, cr);
13496 %}
13497 %}
13498
13499 // This pattern is automatically generated from aarch64_ad.m4
13500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13501 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13502 %{
13503 match(Set dst (MaxI src imm));
13504 ins_cost(INSN_COST * 3);
13505 expand %{
13506 rFlagsReg cr;
13507 compI_reg_imm0(cr, src);
13508 cmovI_reg_imm0_gt(dst, src, cr);
13509 %}
13510 %}
13511
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13515 %{
13516 match(Set dst (MaxI imm src));
13517 ins_cost(INSN_COST * 3);
13518 expand %{
13519 rFlagsReg cr;
13520 compI_reg_imm0(cr, src);
13521 cmovI_reg_imm0_gt(dst, src, cr);
13522 %}
13523 %}
13524
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13528 %{
13529 match(Set dst (MaxI src imm));
13530 ins_cost(INSN_COST * 3);
13531 expand %{
13532 rFlagsReg cr;
13533 compI_reg_imm0(cr, src);
13534 cmovI_reg_imm1_gt(dst, src, cr);
13535 %}
13536 %}
13537
13538 // This pattern is automatically generated from aarch64_ad.m4
13539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13540 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13541 %{
13542 match(Set dst (MaxI imm src));
13543 ins_cost(INSN_COST * 3);
13544 expand %{
13545 rFlagsReg cr;
13546 compI_reg_imm0(cr, src);
13547 cmovI_reg_imm1_gt(dst, src, cr);
13548 %}
13549 %}
13550
13551 // This pattern is automatically generated from aarch64_ad.m4
13552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13553 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13554 %{
13555 match(Set dst (MaxI src imm));
13556 ins_cost(INSN_COST * 3);
13557 expand %{
13558 rFlagsReg cr;
13559 compI_reg_imm0(cr, src);
13560 cmovI_reg_immM1_ge(dst, src, cr);
13561 %}
13562 %}
13563
13564 // This pattern is automatically generated from aarch64_ad.m4
13565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13566 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13567 %{
13568 match(Set dst (MaxI imm src));
13569 ins_cost(INSN_COST * 3);
13570 expand %{
13571 rFlagsReg cr;
13572 compI_reg_imm0(cr, src);
13573 cmovI_reg_immM1_ge(dst, src, cr);
13574 %}
13575 %}
13576
13577 // This pattern is automatically generated from aarch64_ad.m4
13578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13579 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13580 %{
13581 match(Set dst (ReverseI src));
13582 ins_cost(INSN_COST);
13583 format %{ "rbitw $dst, $src" %}
13584 ins_encode %{
13585 __ rbitw($dst$$Register, $src$$Register);
13586 %}
13587 ins_pipe(ialu_reg);
13588 %}
13589
13590 // This pattern is automatically generated from aarch64_ad.m4
13591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13592 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13593 %{
13594 match(Set dst (ReverseL src));
13595 ins_cost(INSN_COST);
13596 format %{ "rbit $dst, $src" %}
13597 ins_encode %{
13598 __ rbit($dst$$Register, $src$$Register);
13599 %}
13600 ins_pipe(ialu_reg);
13601 %}
13602
13603
13604 // END This section of the file is automatically generated. Do not edit --------------
13605
13606
13607 // ============================================================================
13608 // Floating Point Arithmetic Instructions
13609
13610 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13611 match(Set dst (AddF src1 src2));
13612
13613 ins_cost(INSN_COST * 5);
13614 format %{ "fadds $dst, $src1, $src2" %}
13615
13616 ins_encode %{
13617 __ fadds(as_FloatRegister($dst$$reg),
13618 as_FloatRegister($src1$$reg),
13619 as_FloatRegister($src2$$reg));
13620 %}
13621
13622 ins_pipe(fp_dop_reg_reg_s);
13623 %}
13624
13625 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13626 match(Set dst (AddD src1 src2));
13627
13628 ins_cost(INSN_COST * 5);
13629 format %{ "faddd $dst, $src1, $src2" %}
13630
13631 ins_encode %{
13632 __ faddd(as_FloatRegister($dst$$reg),
13633 as_FloatRegister($src1$$reg),
13634 as_FloatRegister($src2$$reg));
13635 %}
13636
13637 ins_pipe(fp_dop_reg_reg_d);
13638 %}
13639
13640 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13641 match(Set dst (SubF src1 src2));
13642
13643 ins_cost(INSN_COST * 5);
13644 format %{ "fsubs $dst, $src1, $src2" %}
13645
13646 ins_encode %{
13647 __ fsubs(as_FloatRegister($dst$$reg),
13648 as_FloatRegister($src1$$reg),
13649 as_FloatRegister($src2$$reg));
13650 %}
13651
13652 ins_pipe(fp_dop_reg_reg_s);
13653 %}
13654
13655 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13656 match(Set dst (SubD src1 src2));
13657
13658 ins_cost(INSN_COST * 5);
13659 format %{ "fsubd $dst, $src1, $src2" %}
13660
13661 ins_encode %{
13662 __ fsubd(as_FloatRegister($dst$$reg),
13663 as_FloatRegister($src1$$reg),
13664 as_FloatRegister($src2$$reg));
13665 %}
13666
13667 ins_pipe(fp_dop_reg_reg_d);
13668 %}
13669
13670 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13671 match(Set dst (MulF src1 src2));
13672
13673 ins_cost(INSN_COST * 6);
13674 format %{ "fmuls $dst, $src1, $src2" %}
13675
13676 ins_encode %{
13677 __ fmuls(as_FloatRegister($dst$$reg),
13678 as_FloatRegister($src1$$reg),
13679 as_FloatRegister($src2$$reg));
13680 %}
13681
13682 ins_pipe(fp_dop_reg_reg_s);
13683 %}
13684
13685 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13686 match(Set dst (MulD src1 src2));
13687
13688 ins_cost(INSN_COST * 6);
13689 format %{ "fmuld $dst, $src1, $src2" %}
13690
13691 ins_encode %{
13692 __ fmuld(as_FloatRegister($dst$$reg),
13693 as_FloatRegister($src1$$reg),
13694 as_FloatRegister($src2$$reg));
13695 %}
13696
13697 ins_pipe(fp_dop_reg_reg_d);
13698 %}
13699
13700 // src1 * src2 + src3
13701 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13702 match(Set dst (FmaF src3 (Binary src1 src2)));
13703
13704 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13705
13706 ins_encode %{
13707 assert(UseFMA, "Needs FMA instructions support.");
13708 __ fmadds(as_FloatRegister($dst$$reg),
13709 as_FloatRegister($src1$$reg),
13710 as_FloatRegister($src2$$reg),
13711 as_FloatRegister($src3$$reg));
13712 %}
13713
13714 ins_pipe(pipe_class_default);
13715 %}
13716
13717 // src1 * src2 + src3
13718 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13719 match(Set dst (FmaD src3 (Binary src1 src2)));
13720
13721 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13722
13723 ins_encode %{
13724 assert(UseFMA, "Needs FMA instructions support.");
13725 __ fmaddd(as_FloatRegister($dst$$reg),
13726 as_FloatRegister($src1$$reg),
13727 as_FloatRegister($src2$$reg),
13728 as_FloatRegister($src3$$reg));
13729 %}
13730
13731 ins_pipe(pipe_class_default);
13732 %}
13733
13734 // src1 * (-src2) + src3
13735 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13736 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13737 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13738
13739 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13740
13741 ins_encode %{
13742 assert(UseFMA, "Needs FMA instructions support.");
13743 __ fmsubs(as_FloatRegister($dst$$reg),
13744 as_FloatRegister($src1$$reg),
13745 as_FloatRegister($src2$$reg),
13746 as_FloatRegister($src3$$reg));
13747 %}
13748
13749 ins_pipe(pipe_class_default);
13750 %}
13751
13752 // src1 * (-src2) + src3
13753 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13754 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13755 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13756
13757 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13758
13759 ins_encode %{
13760 assert(UseFMA, "Needs FMA instructions support.");
13761 __ fmsubd(as_FloatRegister($dst$$reg),
13762 as_FloatRegister($src1$$reg),
13763 as_FloatRegister($src2$$reg),
13764 as_FloatRegister($src3$$reg));
13765 %}
13766
13767 ins_pipe(pipe_class_default);
13768 %}
13769
13770 // src1 * (-src2) - src3
13771 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13772 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13773 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13774
13775 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13776
13777 ins_encode %{
13778 assert(UseFMA, "Needs FMA instructions support.");
13779 __ fnmadds(as_FloatRegister($dst$$reg),
13780 as_FloatRegister($src1$$reg),
13781 as_FloatRegister($src2$$reg),
13782 as_FloatRegister($src3$$reg));
13783 %}
13784
13785 ins_pipe(pipe_class_default);
13786 %}
13787
13788 // src1 * (-src2) - src3
13789 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13790 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13791 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13792
13793 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13794
13795 ins_encode %{
13796 assert(UseFMA, "Needs FMA instructions support.");
13797 __ fnmaddd(as_FloatRegister($dst$$reg),
13798 as_FloatRegister($src1$$reg),
13799 as_FloatRegister($src2$$reg),
13800 as_FloatRegister($src3$$reg));
13801 %}
13802
13803 ins_pipe(pipe_class_default);
13804 %}
13805
13806 // src1 * src2 - src3
13807 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13808 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13809
13810 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13811
13812 ins_encode %{
13813 assert(UseFMA, "Needs FMA instructions support.");
13814 __ fnmsubs(as_FloatRegister($dst$$reg),
13815 as_FloatRegister($src1$$reg),
13816 as_FloatRegister($src2$$reg),
13817 as_FloatRegister($src3$$reg));
13818 %}
13819
13820 ins_pipe(pipe_class_default);
13821 %}
13822
13823 // src1 * src2 - src3
13824 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13825 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13826
13827 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13828
13829 ins_encode %{
13830 assert(UseFMA, "Needs FMA instructions support.");
13831 // n.b. insn name should be fnmsubd
13832 __ fnmsub(as_FloatRegister($dst$$reg),
13833 as_FloatRegister($src1$$reg),
13834 as_FloatRegister($src2$$reg),
13835 as_FloatRegister($src3$$reg));
13836 %}
13837
13838 ins_pipe(pipe_class_default);
13839 %}
13840
13841
13842 // Math.max(FF)F
13843 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13844 match(Set dst (MaxF src1 src2));
13845
13846 format %{ "fmaxs $dst, $src1, $src2" %}
13847 ins_encode %{
13848 __ fmaxs(as_FloatRegister($dst$$reg),
13849 as_FloatRegister($src1$$reg),
13850 as_FloatRegister($src2$$reg));
13851 %}
13852
13853 ins_pipe(fp_dop_reg_reg_s);
13854 %}
13855
13856 // Math.min(FF)F
13857 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13858 match(Set dst (MinF src1 src2));
13859
13860 format %{ "fmins $dst, $src1, $src2" %}
13861 ins_encode %{
13862 __ fmins(as_FloatRegister($dst$$reg),
13863 as_FloatRegister($src1$$reg),
13864 as_FloatRegister($src2$$reg));
13865 %}
13866
13867 ins_pipe(fp_dop_reg_reg_s);
13868 %}
13869
13870 // Math.max(DD)D
13871 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13872 match(Set dst (MaxD src1 src2));
13873
13874 format %{ "fmaxd $dst, $src1, $src2" %}
13875 ins_encode %{
13876 __ fmaxd(as_FloatRegister($dst$$reg),
13877 as_FloatRegister($src1$$reg),
13878 as_FloatRegister($src2$$reg));
13879 %}
13880
13881 ins_pipe(fp_dop_reg_reg_d);
13882 %}
13883
13884 // Math.min(DD)D
13885 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13886 match(Set dst (MinD src1 src2));
13887
13888 format %{ "fmind $dst, $src1, $src2" %}
13889 ins_encode %{
13890 __ fmind(as_FloatRegister($dst$$reg),
13891 as_FloatRegister($src1$$reg),
13892 as_FloatRegister($src2$$reg));
13893 %}
13894
13895 ins_pipe(fp_dop_reg_reg_d);
13896 %}
13897
13898
13899 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13900 match(Set dst (DivF src1 src2));
13901
13902 ins_cost(INSN_COST * 18);
13903 format %{ "fdivs $dst, $src1, $src2" %}
13904
13905 ins_encode %{
13906 __ fdivs(as_FloatRegister($dst$$reg),
13907 as_FloatRegister($src1$$reg),
13908 as_FloatRegister($src2$$reg));
13909 %}
13910
13911 ins_pipe(fp_div_s);
13912 %}
13913
13914 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13915 match(Set dst (DivD src1 src2));
13916
13917 ins_cost(INSN_COST * 32);
13918 format %{ "fdivd $dst, $src1, $src2" %}
13919
13920 ins_encode %{
13921 __ fdivd(as_FloatRegister($dst$$reg),
13922 as_FloatRegister($src1$$reg),
13923 as_FloatRegister($src2$$reg));
13924 %}
13925
13926 ins_pipe(fp_div_d);
13927 %}
13928
13929 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13930 match(Set dst (NegF src));
13931
13932 ins_cost(INSN_COST * 3);
13933 format %{ "fneg $dst, $src" %}
13934
13935 ins_encode %{
13936 __ fnegs(as_FloatRegister($dst$$reg),
13937 as_FloatRegister($src$$reg));
13938 %}
13939
13940 ins_pipe(fp_uop_s);
13941 %}
13942
13943 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13944 match(Set dst (NegD src));
13945
13946 ins_cost(INSN_COST * 3);
13947 format %{ "fnegd $dst, $src" %}
13948
13949 ins_encode %{
13950 __ fnegd(as_FloatRegister($dst$$reg),
13951 as_FloatRegister($src$$reg));
13952 %}
13953
13954 ins_pipe(fp_uop_d);
13955 %}
13956
13957 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13958 %{
13959 match(Set dst (AbsI src));
13960
13961 effect(KILL cr);
13962 ins_cost(INSN_COST * 2);
13963 format %{ "cmpw $src, zr\n\t"
13964 "cnegw $dst, $src, Assembler::LT\t# int abs"
13965 %}
13966
13967 ins_encode %{
13968 __ cmpw(as_Register($src$$reg), zr);
13969 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13970 %}
13971 ins_pipe(pipe_class_default);
13972 %}
13973
13974 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13975 %{
13976 match(Set dst (AbsL src));
13977
13978 effect(KILL cr);
13979 ins_cost(INSN_COST * 2);
13980 format %{ "cmp $src, zr\n\t"
13981 "cneg $dst, $src, Assembler::LT\t# long abs"
13982 %}
13983
13984 ins_encode %{
13985 __ cmp(as_Register($src$$reg), zr);
13986 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13987 %}
13988 ins_pipe(pipe_class_default);
13989 %}
13990
13991 instruct absF_reg(vRegF dst, vRegF src) %{
13992 match(Set dst (AbsF src));
13993
13994 ins_cost(INSN_COST * 3);
13995 format %{ "fabss $dst, $src" %}
13996 ins_encode %{
13997 __ fabss(as_FloatRegister($dst$$reg),
13998 as_FloatRegister($src$$reg));
13999 %}
14000
14001 ins_pipe(fp_uop_s);
14002 %}
14003
14004 instruct absD_reg(vRegD dst, vRegD src) %{
14005 match(Set dst (AbsD src));
14006
14007 ins_cost(INSN_COST * 3);
14008 format %{ "fabsd $dst, $src" %}
14009 ins_encode %{
14010 __ fabsd(as_FloatRegister($dst$$reg),
14011 as_FloatRegister($src$$reg));
14012 %}
14013
14014 ins_pipe(fp_uop_d);
14015 %}
14016
14017 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14018 match(Set dst (AbsF (SubF src1 src2)));
14019
14020 ins_cost(INSN_COST * 3);
14021 format %{ "fabds $dst, $src1, $src2" %}
14022 ins_encode %{
14023 __ fabds(as_FloatRegister($dst$$reg),
14024 as_FloatRegister($src1$$reg),
14025 as_FloatRegister($src2$$reg));
14026 %}
14027
14028 ins_pipe(fp_uop_s);
14029 %}
14030
14031 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14032 match(Set dst (AbsD (SubD src1 src2)));
14033
14034 ins_cost(INSN_COST * 3);
14035 format %{ "fabdd $dst, $src1, $src2" %}
14036 ins_encode %{
14037 __ fabdd(as_FloatRegister($dst$$reg),
14038 as_FloatRegister($src1$$reg),
14039 as_FloatRegister($src2$$reg));
14040 %}
14041
14042 ins_pipe(fp_uop_d);
14043 %}
14044
14045 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14046 match(Set dst (SqrtD src));
14047
14048 ins_cost(INSN_COST * 50);
14049 format %{ "fsqrtd $dst, $src" %}
14050 ins_encode %{
14051 __ fsqrtd(as_FloatRegister($dst$$reg),
14052 as_FloatRegister($src$$reg));
14053 %}
14054
14055 ins_pipe(fp_div_s);
14056 %}
14057
14058 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14059 match(Set dst (SqrtF src));
14060
14061 ins_cost(INSN_COST * 50);
14062 format %{ "fsqrts $dst, $src" %}
14063 ins_encode %{
14064 __ fsqrts(as_FloatRegister($dst$$reg),
14065 as_FloatRegister($src$$reg));
14066 %}
14067
14068 ins_pipe(fp_div_d);
14069 %}
14070
14071 // Math.rint, floor, ceil
14072 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14073 match(Set dst (RoundDoubleMode src rmode));
14074 format %{ "frint $dst, $src, $rmode" %}
14075 ins_encode %{
14076 switch ($rmode$$constant) {
14077 case RoundDoubleModeNode::rmode_rint:
14078 __ frintnd(as_FloatRegister($dst$$reg),
14079 as_FloatRegister($src$$reg));
14080 break;
14081 case RoundDoubleModeNode::rmode_floor:
14082 __ frintmd(as_FloatRegister($dst$$reg),
14083 as_FloatRegister($src$$reg));
14084 break;
14085 case RoundDoubleModeNode::rmode_ceil:
14086 __ frintpd(as_FloatRegister($dst$$reg),
14087 as_FloatRegister($src$$reg));
14088 break;
14089 }
14090 %}
14091 ins_pipe(fp_uop_d);
14092 %}
14093
14094 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14095 match(Set dst (CopySignD src1 (Binary src2 zero)));
14096 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14097 format %{ "CopySignD $dst $src1 $src2" %}
14098 ins_encode %{
14099 FloatRegister dst = as_FloatRegister($dst$$reg),
14100 src1 = as_FloatRegister($src1$$reg),
14101 src2 = as_FloatRegister($src2$$reg),
14102 zero = as_FloatRegister($zero$$reg);
14103 __ fnegd(dst, zero);
14104 __ bsl(dst, __ T8B, src2, src1);
14105 %}
14106 ins_pipe(fp_uop_d);
14107 %}
14108
14109 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14110 match(Set dst (CopySignF src1 src2));
14111 effect(TEMP_DEF dst, USE src1, USE src2);
14112 format %{ "CopySignF $dst $src1 $src2" %}
14113 ins_encode %{
14114 FloatRegister dst = as_FloatRegister($dst$$reg),
14115 src1 = as_FloatRegister($src1$$reg),
14116 src2 = as_FloatRegister($src2$$reg);
14117 __ movi(dst, __ T2S, 0x80, 24);
14118 __ bsl(dst, __ T8B, src2, src1);
14119 %}
14120 ins_pipe(fp_uop_d);
14121 %}
14122
14123 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14124 match(Set dst (SignumD src (Binary zero one)));
14125 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14126 format %{ "signumD $dst, $src" %}
14127 ins_encode %{
14128 FloatRegister src = as_FloatRegister($src$$reg),
14129 dst = as_FloatRegister($dst$$reg),
14130 zero = as_FloatRegister($zero$$reg),
14131 one = as_FloatRegister($one$$reg);
14132 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14133 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14134 // Bit selection instruction gets bit from "one" for each enabled bit in
14135 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14136 // NaN the whole "src" will be copied because "dst" is zero. For all other
14137 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14138 // from "src", and all other bits are copied from 1.0.
14139 __ bsl(dst, __ T8B, one, src);
14140 %}
14141 ins_pipe(fp_uop_d);
14142 %}
14143
14144 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14145 match(Set dst (SignumF src (Binary zero one)));
14146 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14147 format %{ "signumF $dst, $src" %}
14148 ins_encode %{
14149 FloatRegister src = as_FloatRegister($src$$reg),
14150 dst = as_FloatRegister($dst$$reg),
14151 zero = as_FloatRegister($zero$$reg),
14152 one = as_FloatRegister($one$$reg);
14153 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14154 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14155 // Bit selection instruction gets bit from "one" for each enabled bit in
14156 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14157 // NaN the whole "src" will be copied because "dst" is zero. For all other
14158 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14159 // from "src", and all other bits are copied from 1.0.
14160 __ bsl(dst, __ T8B, one, src);
14161 %}
14162 ins_pipe(fp_uop_d);
14163 %}
14164
14165 instruct onspinwait() %{
14166 match(OnSpinWait);
14167 ins_cost(INSN_COST);
14168
14169 format %{ "onspinwait" %}
14170
14171 ins_encode %{
14172 __ spin_wait();
14173 %}
14174 ins_pipe(pipe_class_empty);
14175 %}
14176
14177 // ============================================================================
14178 // Logical Instructions
14179
14180 // Integer Logical Instructions
14181
14182 // And Instructions
14183
14184
14185 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14186 match(Set dst (AndI src1 src2));
14187
14188 format %{ "andw $dst, $src1, $src2\t# int" %}
14189
14190 ins_cost(INSN_COST);
14191 ins_encode %{
14192 __ andw(as_Register($dst$$reg),
14193 as_Register($src1$$reg),
14194 as_Register($src2$$reg));
14195 %}
14196
14197 ins_pipe(ialu_reg_reg);
14198 %}
14199
14200 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14201 match(Set dst (AndI src1 src2));
14202
14203 format %{ "andsw $dst, $src1, $src2\t# int" %}
14204
14205 ins_cost(INSN_COST);
14206 ins_encode %{
14207 __ andw(as_Register($dst$$reg),
14208 as_Register($src1$$reg),
14209 (uint64_t)($src2$$constant));
14210 %}
14211
14212 ins_pipe(ialu_reg_imm);
14213 %}
14214
14215 // Or Instructions
14216
14217 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14218 match(Set dst (OrI src1 src2));
14219
14220 format %{ "orrw $dst, $src1, $src2\t# int" %}
14221
14222 ins_cost(INSN_COST);
14223 ins_encode %{
14224 __ orrw(as_Register($dst$$reg),
14225 as_Register($src1$$reg),
14226 as_Register($src2$$reg));
14227 %}
14228
14229 ins_pipe(ialu_reg_reg);
14230 %}
14231
14232 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14233 match(Set dst (OrI src1 src2));
14234
14235 format %{ "orrw $dst, $src1, $src2\t# int" %}
14236
14237 ins_cost(INSN_COST);
14238 ins_encode %{
14239 __ orrw(as_Register($dst$$reg),
14240 as_Register($src1$$reg),
14241 (uint64_t)($src2$$constant));
14242 %}
14243
14244 ins_pipe(ialu_reg_imm);
14245 %}
14246
14247 // Xor Instructions
14248
14249 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14250 match(Set dst (XorI src1 src2));
14251
14252 format %{ "eorw $dst, $src1, $src2\t# int" %}
14253
14254 ins_cost(INSN_COST);
14255 ins_encode %{
14256 __ eorw(as_Register($dst$$reg),
14257 as_Register($src1$$reg),
14258 as_Register($src2$$reg));
14259 %}
14260
14261 ins_pipe(ialu_reg_reg);
14262 %}
14263
14264 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14265 match(Set dst (XorI src1 src2));
14266
14267 format %{ "eorw $dst, $src1, $src2\t# int" %}
14268
14269 ins_cost(INSN_COST);
14270 ins_encode %{
14271 __ eorw(as_Register($dst$$reg),
14272 as_Register($src1$$reg),
14273 (uint64_t)($src2$$constant));
14274 %}
14275
14276 ins_pipe(ialu_reg_imm);
14277 %}
14278
14279 // Long Logical Instructions
14280 // TODO
14281
14282 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14283 match(Set dst (AndL src1 src2));
14284
14285 format %{ "and $dst, $src1, $src2\t# int" %}
14286
14287 ins_cost(INSN_COST);
14288 ins_encode %{
14289 __ andr(as_Register($dst$$reg),
14290 as_Register($src1$$reg),
14291 as_Register($src2$$reg));
14292 %}
14293
14294 ins_pipe(ialu_reg_reg);
14295 %}
14296
14297 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14298 match(Set dst (AndL src1 src2));
14299
14300 format %{ "and $dst, $src1, $src2\t# int" %}
14301
14302 ins_cost(INSN_COST);
14303 ins_encode %{
14304 __ andr(as_Register($dst$$reg),
14305 as_Register($src1$$reg),
14306 (uint64_t)($src2$$constant));
14307 %}
14308
14309 ins_pipe(ialu_reg_imm);
14310 %}
14311
14312 // Or Instructions
14313
14314 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14315 match(Set dst (OrL src1 src2));
14316
14317 format %{ "orr $dst, $src1, $src2\t# int" %}
14318
14319 ins_cost(INSN_COST);
14320 ins_encode %{
14321 __ orr(as_Register($dst$$reg),
14322 as_Register($src1$$reg),
14323 as_Register($src2$$reg));
14324 %}
14325
14326 ins_pipe(ialu_reg_reg);
14327 %}
14328
14329 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14330 match(Set dst (OrL src1 src2));
14331
14332 format %{ "orr $dst, $src1, $src2\t# int" %}
14333
14334 ins_cost(INSN_COST);
14335 ins_encode %{
14336 __ orr(as_Register($dst$$reg),
14337 as_Register($src1$$reg),
14338 (uint64_t)($src2$$constant));
14339 %}
14340
14341 ins_pipe(ialu_reg_imm);
14342 %}
14343
14344 // Xor Instructions
14345
14346 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14347 match(Set dst (XorL src1 src2));
14348
14349 format %{ "eor $dst, $src1, $src2\t# int" %}
14350
14351 ins_cost(INSN_COST);
14352 ins_encode %{
14353 __ eor(as_Register($dst$$reg),
14354 as_Register($src1$$reg),
14355 as_Register($src2$$reg));
14356 %}
14357
14358 ins_pipe(ialu_reg_reg);
14359 %}
14360
14361 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14362 match(Set dst (XorL src1 src2));
14363
14364 ins_cost(INSN_COST);
14365 format %{ "eor $dst, $src1, $src2\t# int" %}
14366
14367 ins_encode %{
14368 __ eor(as_Register($dst$$reg),
14369 as_Register($src1$$reg),
14370 (uint64_t)($src2$$constant));
14371 %}
14372
14373 ins_pipe(ialu_reg_imm);
14374 %}
14375
14376 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14377 %{
14378 match(Set dst (ConvI2L src));
14379
14380 ins_cost(INSN_COST);
14381 format %{ "sxtw $dst, $src\t# i2l" %}
14382 ins_encode %{
14383 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14384 %}
14385 ins_pipe(ialu_reg_shift);
14386 %}
14387
14388 // this pattern occurs in bigmath arithmetic
14389 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14390 %{
14391 match(Set dst (AndL (ConvI2L src) mask));
14392
14393 ins_cost(INSN_COST);
14394 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14395 ins_encode %{
14396 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14397 %}
14398
14399 ins_pipe(ialu_reg_shift);
14400 %}
14401
14402 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14403 match(Set dst (ConvL2I src));
14404
14405 ins_cost(INSN_COST);
14406 format %{ "movw $dst, $src \t// l2i" %}
14407
14408 ins_encode %{
14409 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14410 %}
14411
14412 ins_pipe(ialu_reg);
14413 %}
14414
14415 instruct convD2F_reg(vRegF dst, vRegD src) %{
14416 match(Set dst (ConvD2F src));
14417
14418 ins_cost(INSN_COST * 5);
14419 format %{ "fcvtd $dst, $src \t// d2f" %}
14420
14421 ins_encode %{
14422 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14423 %}
14424
14425 ins_pipe(fp_d2f);
14426 %}
14427
14428 instruct convF2D_reg(vRegD dst, vRegF src) %{
14429 match(Set dst (ConvF2D src));
14430
14431 ins_cost(INSN_COST * 5);
14432 format %{ "fcvts $dst, $src \t// f2d" %}
14433
14434 ins_encode %{
14435 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14436 %}
14437
14438 ins_pipe(fp_f2d);
14439 %}
14440
14441 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14442 match(Set dst (ConvF2I src));
14443
14444 ins_cost(INSN_COST * 5);
14445 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14446
14447 ins_encode %{
14448 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14449 %}
14450
14451 ins_pipe(fp_f2i);
14452 %}
14453
14454 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14455 match(Set dst (ConvF2L src));
14456
14457 ins_cost(INSN_COST * 5);
14458 format %{ "fcvtzs $dst, $src \t// f2l" %}
14459
14460 ins_encode %{
14461 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14462 %}
14463
14464 ins_pipe(fp_f2l);
14465 %}
14466
14467 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14468 match(Set dst (ConvF2HF src));
14469 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14470 "smov $dst, $tmp\t# move result from $tmp to $dst"
14471 %}
14472 effect(TEMP tmp);
14473 ins_encode %{
14474 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14475 %}
14476 ins_pipe(pipe_slow);
14477 %}
14478
14479 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14480 match(Set dst (ConvHF2F src));
14481 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14482 "fcvt $dst, $tmp\t# convert half to single precision"
14483 %}
14484 effect(TEMP tmp);
14485 ins_encode %{
14486 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14487 %}
14488 ins_pipe(pipe_slow);
14489 %}
14490
14491 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14492 match(Set dst (ConvI2F src));
14493
14494 ins_cost(INSN_COST * 5);
14495 format %{ "scvtfws $dst, $src \t// i2f" %}
14496
14497 ins_encode %{
14498 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14499 %}
14500
14501 ins_pipe(fp_i2f);
14502 %}
14503
14504 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14505 match(Set dst (ConvL2F src));
14506
14507 ins_cost(INSN_COST * 5);
14508 format %{ "scvtfs $dst, $src \t// l2f" %}
14509
14510 ins_encode %{
14511 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14512 %}
14513
14514 ins_pipe(fp_l2f);
14515 %}
14516
14517 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14518 match(Set dst (ConvD2I src));
14519
14520 ins_cost(INSN_COST * 5);
14521 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14522
14523 ins_encode %{
14524 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14525 %}
14526
14527 ins_pipe(fp_d2i);
14528 %}
14529
14530 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14531 match(Set dst (ConvD2L src));
14532
14533 ins_cost(INSN_COST * 5);
14534 format %{ "fcvtzd $dst, $src \t// d2l" %}
14535
14536 ins_encode %{
14537 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14538 %}
14539
14540 ins_pipe(fp_d2l);
14541 %}
14542
14543 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14544 match(Set dst (ConvI2D src));
14545
14546 ins_cost(INSN_COST * 5);
14547 format %{ "scvtfwd $dst, $src \t// i2d" %}
14548
14549 ins_encode %{
14550 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14551 %}
14552
14553 ins_pipe(fp_i2d);
14554 %}
14555
14556 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14557 match(Set dst (ConvL2D src));
14558
14559 ins_cost(INSN_COST * 5);
14560 format %{ "scvtfd $dst, $src \t// l2d" %}
14561
14562 ins_encode %{
14563 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14564 %}
14565
14566 ins_pipe(fp_l2d);
14567 %}
14568
14569 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14570 %{
14571 match(Set dst (RoundD src));
14572 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14573 format %{ "java_round_double $dst,$src"%}
14574 ins_encode %{
14575 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14576 as_FloatRegister($ftmp$$reg));
14577 %}
14578 ins_pipe(pipe_slow);
14579 %}
14580
14581 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14582 %{
14583 match(Set dst (RoundF src));
14584 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14585 format %{ "java_round_float $dst,$src"%}
14586 ins_encode %{
14587 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14588 as_FloatRegister($ftmp$$reg));
14589 %}
14590 ins_pipe(pipe_slow);
14591 %}
14592
14593 // stack <-> reg and reg <-> reg shuffles with no conversion
14594
14595 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14596
14597 match(Set dst (MoveF2I src));
14598
14599 effect(DEF dst, USE src);
14600
14601 ins_cost(4 * INSN_COST);
14602
14603 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14604
14605 ins_encode %{
14606 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14607 %}
14608
14609 ins_pipe(iload_reg_reg);
14610
14611 %}
14612
14613 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14614
14615 match(Set dst (MoveI2F src));
14616
14617 effect(DEF dst, USE src);
14618
14619 ins_cost(4 * INSN_COST);
14620
14621 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14622
14623 ins_encode %{
14624 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14625 %}
14626
14627 ins_pipe(pipe_class_memory);
14628
14629 %}
14630
14631 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14632
14633 match(Set dst (MoveD2L src));
14634
14635 effect(DEF dst, USE src);
14636
14637 ins_cost(4 * INSN_COST);
14638
14639 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14640
14641 ins_encode %{
14642 __ ldr($dst$$Register, Address(sp, $src$$disp));
14643 %}
14644
14645 ins_pipe(iload_reg_reg);
14646
14647 %}
14648
14649 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14650
14651 match(Set dst (MoveL2D src));
14652
14653 effect(DEF dst, USE src);
14654
14655 ins_cost(4 * INSN_COST);
14656
14657 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14658
14659 ins_encode %{
14660 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14661 %}
14662
14663 ins_pipe(pipe_class_memory);
14664
14665 %}
14666
14667 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14668
14669 match(Set dst (MoveF2I src));
14670
14671 effect(DEF dst, USE src);
14672
14673 ins_cost(INSN_COST);
14674
14675 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14676
14677 ins_encode %{
14678 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14679 %}
14680
14681 ins_pipe(pipe_class_memory);
14682
14683 %}
14684
14685 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14686
14687 match(Set dst (MoveI2F src));
14688
14689 effect(DEF dst, USE src);
14690
14691 ins_cost(INSN_COST);
14692
14693 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14694
14695 ins_encode %{
14696 __ strw($src$$Register, Address(sp, $dst$$disp));
14697 %}
14698
14699 ins_pipe(istore_reg_reg);
14700
14701 %}
14702
14703 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14704
14705 match(Set dst (MoveD2L src));
14706
14707 effect(DEF dst, USE src);
14708
14709 ins_cost(INSN_COST);
14710
14711 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14712
14713 ins_encode %{
14714 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14715 %}
14716
14717 ins_pipe(pipe_class_memory);
14718
14719 %}
14720
14721 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14722
14723 match(Set dst (MoveL2D src));
14724
14725 effect(DEF dst, USE src);
14726
14727 ins_cost(INSN_COST);
14728
14729 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14730
14731 ins_encode %{
14732 __ str($src$$Register, Address(sp, $dst$$disp));
14733 %}
14734
14735 ins_pipe(istore_reg_reg);
14736
14737 %}
14738
14739 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14740
14741 match(Set dst (MoveF2I src));
14742
14743 effect(DEF dst, USE src);
14744
14745 ins_cost(INSN_COST);
14746
14747 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14748
14749 ins_encode %{
14750 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14751 %}
14752
14753 ins_pipe(fp_f2i);
14754
14755 %}
14756
14757 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14758
14759 match(Set dst (MoveI2F src));
14760
14761 effect(DEF dst, USE src);
14762
14763 ins_cost(INSN_COST);
14764
14765 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14766
14767 ins_encode %{
14768 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14769 %}
14770
14771 ins_pipe(fp_i2f);
14772
14773 %}
14774
14775 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14776
14777 match(Set dst (MoveD2L src));
14778
14779 effect(DEF dst, USE src);
14780
14781 ins_cost(INSN_COST);
14782
14783 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14784
14785 ins_encode %{
14786 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14787 %}
14788
14789 ins_pipe(fp_d2l);
14790
14791 %}
14792
14793 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14794
14795 match(Set dst (MoveL2D src));
14796
14797 effect(DEF dst, USE src);
14798
14799 ins_cost(INSN_COST);
14800
14801 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14802
14803 ins_encode %{
14804 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14805 %}
14806
14807 ins_pipe(fp_l2d);
14808
14809 %}
14810
14811 // ============================================================================
14812 // clearing of an array
14813
14814 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14815 %{
14816 match(Set dummy (ClearArray cnt base));
14817 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14818
14819 ins_cost(4 * INSN_COST);
14820 format %{ "ClearArray $cnt, $base" %}
14821
14822 ins_encode %{
14823 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14824 if (tpc == nullptr) {
14825 ciEnv::current()->record_failure("CodeCache is full");
14826 return;
14827 }
14828 %}
14829
14830 ins_pipe(pipe_class_memory);
14831 %}
14832
14833 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14834 %{
14835 predicate((uint64_t)n->in(2)->get_long()
14836 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14837 match(Set dummy (ClearArray cnt base));
14838 effect(TEMP temp, USE_KILL base, KILL cr);
14839
14840 ins_cost(4 * INSN_COST);
14841 format %{ "ClearArray $cnt, $base" %}
14842
14843 ins_encode %{
14844 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14845 if (tpc == nullptr) {
14846 ciEnv::current()->record_failure("CodeCache is full");
14847 return;
14848 }
14849 %}
14850
14851 ins_pipe(pipe_class_memory);
14852 %}
14853
14854 // ============================================================================
14855 // Overflow Math Instructions
14856
14857 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14858 %{
14859 match(Set cr (OverflowAddI op1 op2));
14860
14861 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14862 ins_cost(INSN_COST);
14863 ins_encode %{
14864 __ cmnw($op1$$Register, $op2$$Register);
14865 %}
14866
14867 ins_pipe(icmp_reg_reg);
14868 %}
14869
14870 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14871 %{
14872 match(Set cr (OverflowAddI op1 op2));
14873
14874 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14875 ins_cost(INSN_COST);
14876 ins_encode %{
14877 __ cmnw($op1$$Register, $op2$$constant);
14878 %}
14879
14880 ins_pipe(icmp_reg_imm);
14881 %}
14882
14883 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14884 %{
14885 match(Set cr (OverflowAddL op1 op2));
14886
14887 format %{ "cmn $op1, $op2\t# overflow check long" %}
14888 ins_cost(INSN_COST);
14889 ins_encode %{
14890 __ cmn($op1$$Register, $op2$$Register);
14891 %}
14892
14893 ins_pipe(icmp_reg_reg);
14894 %}
14895
14896 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14897 %{
14898 match(Set cr (OverflowAddL op1 op2));
14899
14900 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14901 ins_cost(INSN_COST);
14902 ins_encode %{
14903 __ adds(zr, $op1$$Register, $op2$$constant);
14904 %}
14905
14906 ins_pipe(icmp_reg_imm);
14907 %}
14908
14909 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14910 %{
14911 match(Set cr (OverflowSubI op1 op2));
14912
14913 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14914 ins_cost(INSN_COST);
14915 ins_encode %{
14916 __ cmpw($op1$$Register, $op2$$Register);
14917 %}
14918
14919 ins_pipe(icmp_reg_reg);
14920 %}
14921
14922 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14923 %{
14924 match(Set cr (OverflowSubI op1 op2));
14925
14926 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14927 ins_cost(INSN_COST);
14928 ins_encode %{
14929 __ cmpw($op1$$Register, $op2$$constant);
14930 %}
14931
14932 ins_pipe(icmp_reg_imm);
14933 %}
14934
14935 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14936 %{
14937 match(Set cr (OverflowSubL op1 op2));
14938
14939 format %{ "cmp $op1, $op2\t# overflow check long" %}
14940 ins_cost(INSN_COST);
14941 ins_encode %{
14942 __ cmp($op1$$Register, $op2$$Register);
14943 %}
14944
14945 ins_pipe(icmp_reg_reg);
14946 %}
14947
14948 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14949 %{
14950 match(Set cr (OverflowSubL op1 op2));
14951
14952 format %{ "cmp $op1, $op2\t# overflow check long" %}
14953 ins_cost(INSN_COST);
14954 ins_encode %{
14955 __ subs(zr, $op1$$Register, $op2$$constant);
14956 %}
14957
14958 ins_pipe(icmp_reg_imm);
14959 %}
14960
14961 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14962 %{
14963 match(Set cr (OverflowSubI zero op1));
14964
14965 format %{ "cmpw zr, $op1\t# overflow check int" %}
14966 ins_cost(INSN_COST);
14967 ins_encode %{
14968 __ cmpw(zr, $op1$$Register);
14969 %}
14970
14971 ins_pipe(icmp_reg_imm);
14972 %}
14973
14974 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14975 %{
14976 match(Set cr (OverflowSubL zero op1));
14977
14978 format %{ "cmp zr, $op1\t# overflow check long" %}
14979 ins_cost(INSN_COST);
14980 ins_encode %{
14981 __ cmp(zr, $op1$$Register);
14982 %}
14983
14984 ins_pipe(icmp_reg_imm);
14985 %}
14986
14987 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14988 %{
14989 match(Set cr (OverflowMulI op1 op2));
14990
14991 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14992 "cmp rscratch1, rscratch1, sxtw\n\t"
14993 "movw rscratch1, #0x80000000\n\t"
14994 "cselw rscratch1, rscratch1, zr, NE\n\t"
14995 "cmpw rscratch1, #1" %}
14996 ins_cost(5 * INSN_COST);
14997 ins_encode %{
14998 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14999 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15000 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15001 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15002 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15003 %}
15004
15005 ins_pipe(pipe_slow);
15006 %}
15007
15008 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15009 %{
15010 match(If cmp (OverflowMulI op1 op2));
15011 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15012 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15013 effect(USE labl, KILL cr);
15014
15015 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15016 "cmp rscratch1, rscratch1, sxtw\n\t"
15017 "b$cmp $labl" %}
15018 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15019 ins_encode %{
15020 Label* L = $labl$$label;
15021 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15022 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15023 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15024 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15025 %}
15026
15027 ins_pipe(pipe_serial);
15028 %}
15029
15030 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15031 %{
15032 match(Set cr (OverflowMulL op1 op2));
15033
15034 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15035 "smulh rscratch2, $op1, $op2\n\t"
15036 "cmp rscratch2, rscratch1, ASR #63\n\t"
15037 "movw rscratch1, #0x80000000\n\t"
15038 "cselw rscratch1, rscratch1, zr, NE\n\t"
15039 "cmpw rscratch1, #1" %}
15040 ins_cost(6 * INSN_COST);
15041 ins_encode %{
15042 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15043 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15044 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15045 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15046 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15047 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15048 %}
15049
15050 ins_pipe(pipe_slow);
15051 %}
15052
15053 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15054 %{
15055 match(If cmp (OverflowMulL op1 op2));
15056 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15057 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15058 effect(USE labl, KILL cr);
15059
15060 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15061 "smulh rscratch2, $op1, $op2\n\t"
15062 "cmp rscratch2, rscratch1, ASR #63\n\t"
15063 "b$cmp $labl" %}
15064 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15065 ins_encode %{
15066 Label* L = $labl$$label;
15067 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15068 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15069 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15070 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15071 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15072 %}
15073
15074 ins_pipe(pipe_serial);
15075 %}
15076
15077 // ============================================================================
15078 // Compare Instructions
15079
15080 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15081 %{
15082 match(Set cr (CmpI op1 op2));
15083
15084 effect(DEF cr, USE op1, USE op2);
15085
15086 ins_cost(INSN_COST);
15087 format %{ "cmpw $op1, $op2" %}
15088
15089 ins_encode(aarch64_enc_cmpw(op1, op2));
15090
15091 ins_pipe(icmp_reg_reg);
15092 %}
15093
15094 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15095 %{
15096 match(Set cr (CmpI op1 zero));
15097
15098 effect(DEF cr, USE op1);
15099
15100 ins_cost(INSN_COST);
15101 format %{ "cmpw $op1, 0" %}
15102
15103 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15104
15105 ins_pipe(icmp_reg_imm);
15106 %}
15107
15108 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15109 %{
15110 match(Set cr (CmpI op1 op2));
15111
15112 effect(DEF cr, USE op1);
15113
15114 ins_cost(INSN_COST);
15115 format %{ "cmpw $op1, $op2" %}
15116
15117 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15118
15119 ins_pipe(icmp_reg_imm);
15120 %}
15121
15122 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15123 %{
15124 match(Set cr (CmpI op1 op2));
15125
15126 effect(DEF cr, USE op1);
15127
15128 ins_cost(INSN_COST * 2);
15129 format %{ "cmpw $op1, $op2" %}
15130
15131 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15132
15133 ins_pipe(icmp_reg_imm);
15134 %}
15135
15136 // Unsigned compare Instructions; really, same as signed compare
15137 // except it should only be used to feed an If or a CMovI which takes a
15138 // cmpOpU.
15139
15140 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15141 %{
15142 match(Set cr (CmpU op1 op2));
15143
15144 effect(DEF cr, USE op1, USE op2);
15145
15146 ins_cost(INSN_COST);
15147 format %{ "cmpw $op1, $op2\t# unsigned" %}
15148
15149 ins_encode(aarch64_enc_cmpw(op1, op2));
15150
15151 ins_pipe(icmp_reg_reg);
15152 %}
15153
15154 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15155 %{
15156 match(Set cr (CmpU op1 zero));
15157
15158 effect(DEF cr, USE op1);
15159
15160 ins_cost(INSN_COST);
15161 format %{ "cmpw $op1, #0\t# unsigned" %}
15162
15163 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15164
15165 ins_pipe(icmp_reg_imm);
15166 %}
15167
15168 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15169 %{
15170 match(Set cr (CmpU op1 op2));
15171
15172 effect(DEF cr, USE op1);
15173
15174 ins_cost(INSN_COST);
15175 format %{ "cmpw $op1, $op2\t# unsigned" %}
15176
15177 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15178
15179 ins_pipe(icmp_reg_imm);
15180 %}
15181
15182 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15183 %{
15184 match(Set cr (CmpU op1 op2));
15185
15186 effect(DEF cr, USE op1);
15187
15188 ins_cost(INSN_COST * 2);
15189 format %{ "cmpw $op1, $op2\t# unsigned" %}
15190
15191 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15192
15193 ins_pipe(icmp_reg_imm);
15194 %}
15195
15196 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15197 %{
15198 match(Set cr (CmpL op1 op2));
15199
15200 effect(DEF cr, USE op1, USE op2);
15201
15202 ins_cost(INSN_COST);
15203 format %{ "cmp $op1, $op2" %}
15204
15205 ins_encode(aarch64_enc_cmp(op1, op2));
15206
15207 ins_pipe(icmp_reg_reg);
15208 %}
15209
15210 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15211 %{
15212 match(Set cr (CmpL op1 zero));
15213
15214 effect(DEF cr, USE op1);
15215
15216 ins_cost(INSN_COST);
15217 format %{ "tst $op1" %}
15218
15219 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15220
15221 ins_pipe(icmp_reg_imm);
15222 %}
15223
15224 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15225 %{
15226 match(Set cr (CmpL op1 op2));
15227
15228 effect(DEF cr, USE op1);
15229
15230 ins_cost(INSN_COST);
15231 format %{ "cmp $op1, $op2" %}
15232
15233 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15234
15235 ins_pipe(icmp_reg_imm);
15236 %}
15237
15238 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15239 %{
15240 match(Set cr (CmpL op1 op2));
15241
15242 effect(DEF cr, USE op1);
15243
15244 ins_cost(INSN_COST * 2);
15245 format %{ "cmp $op1, $op2" %}
15246
15247 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15248
15249 ins_pipe(icmp_reg_imm);
15250 %}
15251
15252 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15253 %{
15254 match(Set cr (CmpUL op1 op2));
15255
15256 effect(DEF cr, USE op1, USE op2);
15257
15258 ins_cost(INSN_COST);
15259 format %{ "cmp $op1, $op2" %}
15260
15261 ins_encode(aarch64_enc_cmp(op1, op2));
15262
15263 ins_pipe(icmp_reg_reg);
15264 %}
15265
15266 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15267 %{
15268 match(Set cr (CmpUL op1 zero));
15269
15270 effect(DEF cr, USE op1);
15271
15272 ins_cost(INSN_COST);
15273 format %{ "tst $op1" %}
15274
15275 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15276
15277 ins_pipe(icmp_reg_imm);
15278 %}
15279
15280 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15281 %{
15282 match(Set cr (CmpUL op1 op2));
15283
15284 effect(DEF cr, USE op1);
15285
15286 ins_cost(INSN_COST);
15287 format %{ "cmp $op1, $op2" %}
15288
15289 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15290
15291 ins_pipe(icmp_reg_imm);
15292 %}
15293
15294 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15295 %{
15296 match(Set cr (CmpUL op1 op2));
15297
15298 effect(DEF cr, USE op1);
15299
15300 ins_cost(INSN_COST * 2);
15301 format %{ "cmp $op1, $op2" %}
15302
15303 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15304
15305 ins_pipe(icmp_reg_imm);
15306 %}
15307
15308 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15309 %{
15310 match(Set cr (CmpP op1 op2));
15311
15312 effect(DEF cr, USE op1, USE op2);
15313
15314 ins_cost(INSN_COST);
15315 format %{ "cmp $op1, $op2\t // ptr" %}
15316
15317 ins_encode(aarch64_enc_cmpp(op1, op2));
15318
15319 ins_pipe(icmp_reg_reg);
15320 %}
15321
15322 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15323 %{
15324 match(Set cr (CmpN op1 op2));
15325
15326 effect(DEF cr, USE op1, USE op2);
15327
15328 ins_cost(INSN_COST);
15329 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15330
15331 ins_encode(aarch64_enc_cmpn(op1, op2));
15332
15333 ins_pipe(icmp_reg_reg);
15334 %}
15335
15336 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15337 %{
15338 match(Set cr (CmpP op1 zero));
15339
15340 effect(DEF cr, USE op1, USE zero);
15341
15342 ins_cost(INSN_COST);
15343 format %{ "cmp $op1, 0\t // ptr" %}
15344
15345 ins_encode(aarch64_enc_testp(op1));
15346
15347 ins_pipe(icmp_reg_imm);
15348 %}
15349
15350 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15351 %{
15352 match(Set cr (CmpN op1 zero));
15353
15354 effect(DEF cr, USE op1, USE zero);
15355
15356 ins_cost(INSN_COST);
15357 format %{ "cmp $op1, 0\t // compressed ptr" %}
15358
15359 ins_encode(aarch64_enc_testn(op1));
15360
15361 ins_pipe(icmp_reg_imm);
15362 %}
15363
15364 // FP comparisons
15365 //
15366 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15367 // using normal cmpOp. See declaration of rFlagsReg for details.
15368
15369 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15370 %{
15371 match(Set cr (CmpF src1 src2));
15372
15373 ins_cost(3 * INSN_COST);
15374 format %{ "fcmps $src1, $src2" %}
15375
15376 ins_encode %{
15377 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15378 %}
15379
15380 ins_pipe(pipe_class_compare);
15381 %}
15382
15383 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15384 %{
15385 match(Set cr (CmpF src1 src2));
15386
15387 ins_cost(3 * INSN_COST);
15388 format %{ "fcmps $src1, 0.0" %}
15389
15390 ins_encode %{
15391 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15392 %}
15393
15394 ins_pipe(pipe_class_compare);
15395 %}
15396 // FROM HERE
15397
15398 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15399 %{
15400 match(Set cr (CmpD src1 src2));
15401
15402 ins_cost(3 * INSN_COST);
15403 format %{ "fcmpd $src1, $src2" %}
15404
15405 ins_encode %{
15406 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15407 %}
15408
15409 ins_pipe(pipe_class_compare);
15410 %}
15411
15412 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15413 %{
15414 match(Set cr (CmpD src1 src2));
15415
15416 ins_cost(3 * INSN_COST);
15417 format %{ "fcmpd $src1, 0.0" %}
15418
15419 ins_encode %{
15420 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15421 %}
15422
15423 ins_pipe(pipe_class_compare);
15424 %}
15425
15426 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15427 %{
15428 match(Set dst (CmpF3 src1 src2));
15429 effect(KILL cr);
15430
15431 ins_cost(5 * INSN_COST);
15432 format %{ "fcmps $src1, $src2\n\t"
15433 "csinvw($dst, zr, zr, eq\n\t"
15434 "csnegw($dst, $dst, $dst, lt)"
15435 %}
15436
15437 ins_encode %{
15438 Label done;
15439 FloatRegister s1 = as_FloatRegister($src1$$reg);
15440 FloatRegister s2 = as_FloatRegister($src2$$reg);
15441 Register d = as_Register($dst$$reg);
15442 __ fcmps(s1, s2);
15443 // installs 0 if EQ else -1
15444 __ csinvw(d, zr, zr, Assembler::EQ);
15445 // keeps -1 if less or unordered else installs 1
15446 __ csnegw(d, d, d, Assembler::LT);
15447 __ bind(done);
15448 %}
15449
15450 ins_pipe(pipe_class_default);
15451
15452 %}
15453
15454 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15455 %{
15456 match(Set dst (CmpD3 src1 src2));
15457 effect(KILL cr);
15458
15459 ins_cost(5 * INSN_COST);
15460 format %{ "fcmpd $src1, $src2\n\t"
15461 "csinvw($dst, zr, zr, eq\n\t"
15462 "csnegw($dst, $dst, $dst, lt)"
15463 %}
15464
15465 ins_encode %{
15466 Label done;
15467 FloatRegister s1 = as_FloatRegister($src1$$reg);
15468 FloatRegister s2 = as_FloatRegister($src2$$reg);
15469 Register d = as_Register($dst$$reg);
15470 __ fcmpd(s1, s2);
15471 // installs 0 if EQ else -1
15472 __ csinvw(d, zr, zr, Assembler::EQ);
15473 // keeps -1 if less or unordered else installs 1
15474 __ csnegw(d, d, d, Assembler::LT);
15475 __ bind(done);
15476 %}
15477 ins_pipe(pipe_class_default);
15478
15479 %}
15480
15481 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15482 %{
15483 match(Set dst (CmpF3 src1 zero));
15484 effect(KILL cr);
15485
15486 ins_cost(5 * INSN_COST);
15487 format %{ "fcmps $src1, 0.0\n\t"
15488 "csinvw($dst, zr, zr, eq\n\t"
15489 "csnegw($dst, $dst, $dst, lt)"
15490 %}
15491
15492 ins_encode %{
15493 Label done;
15494 FloatRegister s1 = as_FloatRegister($src1$$reg);
15495 Register d = as_Register($dst$$reg);
15496 __ fcmps(s1, 0.0);
15497 // installs 0 if EQ else -1
15498 __ csinvw(d, zr, zr, Assembler::EQ);
15499 // keeps -1 if less or unordered else installs 1
15500 __ csnegw(d, d, d, Assembler::LT);
15501 __ bind(done);
15502 %}
15503
15504 ins_pipe(pipe_class_default);
15505
15506 %}
15507
15508 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15509 %{
15510 match(Set dst (CmpD3 src1 zero));
15511 effect(KILL cr);
15512
15513 ins_cost(5 * INSN_COST);
15514 format %{ "fcmpd $src1, 0.0\n\t"
15515 "csinvw($dst, zr, zr, eq\n\t"
15516 "csnegw($dst, $dst, $dst, lt)"
15517 %}
15518
15519 ins_encode %{
15520 Label done;
15521 FloatRegister s1 = as_FloatRegister($src1$$reg);
15522 Register d = as_Register($dst$$reg);
15523 __ fcmpd(s1, 0.0);
15524 // installs 0 if EQ else -1
15525 __ csinvw(d, zr, zr, Assembler::EQ);
15526 // keeps -1 if less or unordered else installs 1
15527 __ csnegw(d, d, d, Assembler::LT);
15528 __ bind(done);
15529 %}
15530 ins_pipe(pipe_class_default);
15531
15532 %}
15533
15534 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15535 %{
15536 match(Set dst (CmpLTMask p q));
15537 effect(KILL cr);
15538
15539 ins_cost(3 * INSN_COST);
15540
15541 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15542 "csetw $dst, lt\n\t"
15543 "subw $dst, zr, $dst"
15544 %}
15545
15546 ins_encode %{
15547 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15548 __ csetw(as_Register($dst$$reg), Assembler::LT);
15549 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15550 %}
15551
15552 ins_pipe(ialu_reg_reg);
15553 %}
15554
15555 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15556 %{
15557 match(Set dst (CmpLTMask src zero));
15558 effect(KILL cr);
15559
15560 ins_cost(INSN_COST);
15561
15562 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15563
15564 ins_encode %{
15565 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15566 %}
15567
15568 ins_pipe(ialu_reg_shift);
15569 %}
15570
15571 // ============================================================================
15572 // Max and Min
15573
15574 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15575
15576 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15577 %{
15578 effect(DEF cr, USE src);
15579 ins_cost(INSN_COST);
15580 format %{ "cmpw $src, 0" %}
15581
15582 ins_encode %{
15583 __ cmpw($src$$Register, 0);
15584 %}
15585 ins_pipe(icmp_reg_imm);
15586 %}
15587
15588 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15589 %{
15590 match(Set dst (MinI src1 src2));
15591 ins_cost(INSN_COST * 3);
15592
15593 expand %{
15594 rFlagsReg cr;
15595 compI_reg_reg(cr, src1, src2);
15596 cmovI_reg_reg_lt(dst, src1, src2, cr);
15597 %}
15598 %}
15599
15600 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15601 %{
15602 match(Set dst (MaxI src1 src2));
15603 ins_cost(INSN_COST * 3);
15604
15605 expand %{
15606 rFlagsReg cr;
15607 compI_reg_reg(cr, src1, src2);
15608 cmovI_reg_reg_gt(dst, src1, src2, cr);
15609 %}
15610 %}
15611
15612
15613 // ============================================================================
15614 // Branch Instructions
15615
15616 // Direct Branch.
15617 instruct branch(label lbl)
15618 %{
15619 match(Goto);
15620
15621 effect(USE lbl);
15622
15623 ins_cost(BRANCH_COST);
15624 format %{ "b $lbl" %}
15625
15626 ins_encode(aarch64_enc_b(lbl));
15627
15628 ins_pipe(pipe_branch);
15629 %}
15630
15631 // Conditional Near Branch
15632 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15633 %{
15634 // Same match rule as `branchConFar'.
15635 match(If cmp cr);
15636
15637 effect(USE lbl);
15638
15639 ins_cost(BRANCH_COST);
15640 // If set to 1 this indicates that the current instruction is a
15641 // short variant of a long branch. This avoids using this
15642 // instruction in first-pass matching. It will then only be used in
15643 // the `Shorten_branches' pass.
15644 // ins_short_branch(1);
15645 format %{ "b$cmp $lbl" %}
15646
15647 ins_encode(aarch64_enc_br_con(cmp, lbl));
15648
15649 ins_pipe(pipe_branch_cond);
15650 %}
15651
15652 // Conditional Near Branch Unsigned
15653 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15654 %{
15655 // Same match rule as `branchConFar'.
15656 match(If cmp cr);
15657
15658 effect(USE lbl);
15659
15660 ins_cost(BRANCH_COST);
15661 // If set to 1 this indicates that the current instruction is a
15662 // short variant of a long branch. This avoids using this
15663 // instruction in first-pass matching. It will then only be used in
15664 // the `Shorten_branches' pass.
15665 // ins_short_branch(1);
15666 format %{ "b$cmp $lbl\t# unsigned" %}
15667
15668 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15669
15670 ins_pipe(pipe_branch_cond);
15671 %}
15672
15673 // Make use of CBZ and CBNZ. These instructions, as well as being
15674 // shorter than (cmp; branch), have the additional benefit of not
15675 // killing the flags.
15676
15677 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15678 match(If cmp (CmpI op1 op2));
15679 effect(USE labl);
15680
15681 ins_cost(BRANCH_COST);
15682 format %{ "cbw$cmp $op1, $labl" %}
15683 ins_encode %{
15684 Label* L = $labl$$label;
15685 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15686 if (cond == Assembler::EQ)
15687 __ cbzw($op1$$Register, *L);
15688 else
15689 __ cbnzw($op1$$Register, *L);
15690 %}
15691 ins_pipe(pipe_cmp_branch);
15692 %}
15693
15694 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15695 match(If cmp (CmpL op1 op2));
15696 effect(USE labl);
15697
15698 ins_cost(BRANCH_COST);
15699 format %{ "cb$cmp $op1, $labl" %}
15700 ins_encode %{
15701 Label* L = $labl$$label;
15702 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15703 if (cond == Assembler::EQ)
15704 __ cbz($op1$$Register, *L);
15705 else
15706 __ cbnz($op1$$Register, *L);
15707 %}
15708 ins_pipe(pipe_cmp_branch);
15709 %}
15710
15711 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15712 match(If cmp (CmpP op1 op2));
15713 effect(USE labl);
15714
15715 ins_cost(BRANCH_COST);
15716 format %{ "cb$cmp $op1, $labl" %}
15717 ins_encode %{
15718 Label* L = $labl$$label;
15719 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15720 if (cond == Assembler::EQ)
15721 __ cbz($op1$$Register, *L);
15722 else
15723 __ cbnz($op1$$Register, *L);
15724 %}
15725 ins_pipe(pipe_cmp_branch);
15726 %}
15727
15728 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15729 match(If cmp (CmpN op1 op2));
15730 effect(USE labl);
15731
15732 ins_cost(BRANCH_COST);
15733 format %{ "cbw$cmp $op1, $labl" %}
15734 ins_encode %{
15735 Label* L = $labl$$label;
15736 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15737 if (cond == Assembler::EQ)
15738 __ cbzw($op1$$Register, *L);
15739 else
15740 __ cbnzw($op1$$Register, *L);
15741 %}
15742 ins_pipe(pipe_cmp_branch);
15743 %}
15744
15745 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15746 match(If cmp (CmpP (DecodeN oop) zero));
15747 effect(USE labl);
15748
15749 ins_cost(BRANCH_COST);
15750 format %{ "cb$cmp $oop, $labl" %}
15751 ins_encode %{
15752 Label* L = $labl$$label;
15753 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15754 if (cond == Assembler::EQ)
15755 __ cbzw($oop$$Register, *L);
15756 else
15757 __ cbnzw($oop$$Register, *L);
15758 %}
15759 ins_pipe(pipe_cmp_branch);
15760 %}
15761
15762 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15763 match(If cmp (CmpU op1 op2));
15764 effect(USE labl);
15765
15766 ins_cost(BRANCH_COST);
15767 format %{ "cbw$cmp $op1, $labl" %}
15768 ins_encode %{
15769 Label* L = $labl$$label;
15770 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15771 if (cond == Assembler::EQ || cond == Assembler::LS) {
15772 __ cbzw($op1$$Register, *L);
15773 } else {
15774 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15775 __ cbnzw($op1$$Register, *L);
15776 }
15777 %}
15778 ins_pipe(pipe_cmp_branch);
15779 %}
15780
15781 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15782 match(If cmp (CmpUL op1 op2));
15783 effect(USE labl);
15784
15785 ins_cost(BRANCH_COST);
15786 format %{ "cb$cmp $op1, $labl" %}
15787 ins_encode %{
15788 Label* L = $labl$$label;
15789 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15790 if (cond == Assembler::EQ || cond == Assembler::LS) {
15791 __ cbz($op1$$Register, *L);
15792 } else {
15793 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15794 __ cbnz($op1$$Register, *L);
15795 }
15796 %}
15797 ins_pipe(pipe_cmp_branch);
15798 %}
15799
15800 // Test bit and Branch
15801
15802 // Patterns for short (< 32KiB) variants
15803 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15804 match(If cmp (CmpL op1 op2));
15805 effect(USE labl);
15806
15807 ins_cost(BRANCH_COST);
15808 format %{ "cb$cmp $op1, $labl # long" %}
15809 ins_encode %{
15810 Label* L = $labl$$label;
15811 Assembler::Condition cond =
15812 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15813 __ tbr(cond, $op1$$Register, 63, *L);
15814 %}
15815 ins_pipe(pipe_cmp_branch);
15816 ins_short_branch(1);
15817 %}
15818
15819 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15820 match(If cmp (CmpI op1 op2));
15821 effect(USE labl);
15822
15823 ins_cost(BRANCH_COST);
15824 format %{ "cb$cmp $op1, $labl # int" %}
15825 ins_encode %{
15826 Label* L = $labl$$label;
15827 Assembler::Condition cond =
15828 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15829 __ tbr(cond, $op1$$Register, 31, *L);
15830 %}
15831 ins_pipe(pipe_cmp_branch);
15832 ins_short_branch(1);
15833 %}
15834
15835 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15836 match(If cmp (CmpL (AndL op1 op2) op3));
15837 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15838 effect(USE labl);
15839
15840 ins_cost(BRANCH_COST);
15841 format %{ "tb$cmp $op1, $op2, $labl" %}
15842 ins_encode %{
15843 Label* L = $labl$$label;
15844 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15845 int bit = exact_log2_long($op2$$constant);
15846 __ tbr(cond, $op1$$Register, bit, *L);
15847 %}
15848 ins_pipe(pipe_cmp_branch);
15849 ins_short_branch(1);
15850 %}
15851
15852 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15853 match(If cmp (CmpI (AndI op1 op2) op3));
15854 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15855 effect(USE labl);
15856
15857 ins_cost(BRANCH_COST);
15858 format %{ "tb$cmp $op1, $op2, $labl" %}
15859 ins_encode %{
15860 Label* L = $labl$$label;
15861 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15862 int bit = exact_log2((juint)$op2$$constant);
15863 __ tbr(cond, $op1$$Register, bit, *L);
15864 %}
15865 ins_pipe(pipe_cmp_branch);
15866 ins_short_branch(1);
15867 %}
15868
15869 // And far variants
15870 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15871 match(If cmp (CmpL op1 op2));
15872 effect(USE labl);
15873
15874 ins_cost(BRANCH_COST);
15875 format %{ "cb$cmp $op1, $labl # long" %}
15876 ins_encode %{
15877 Label* L = $labl$$label;
15878 Assembler::Condition cond =
15879 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15880 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15881 %}
15882 ins_pipe(pipe_cmp_branch);
15883 %}
15884
15885 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15886 match(If cmp (CmpI op1 op2));
15887 effect(USE labl);
15888
15889 ins_cost(BRANCH_COST);
15890 format %{ "cb$cmp $op1, $labl # int" %}
15891 ins_encode %{
15892 Label* L = $labl$$label;
15893 Assembler::Condition cond =
15894 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15895 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15896 %}
15897 ins_pipe(pipe_cmp_branch);
15898 %}
15899
15900 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15901 match(If cmp (CmpL (AndL op1 op2) op3));
15902 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15903 effect(USE labl);
15904
15905 ins_cost(BRANCH_COST);
15906 format %{ "tb$cmp $op1, $op2, $labl" %}
15907 ins_encode %{
15908 Label* L = $labl$$label;
15909 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15910 int bit = exact_log2_long($op2$$constant);
15911 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15912 %}
15913 ins_pipe(pipe_cmp_branch);
15914 %}
15915
15916 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15917 match(If cmp (CmpI (AndI op1 op2) op3));
15918 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15919 effect(USE labl);
15920
15921 ins_cost(BRANCH_COST);
15922 format %{ "tb$cmp $op1, $op2, $labl" %}
15923 ins_encode %{
15924 Label* L = $labl$$label;
15925 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15926 int bit = exact_log2((juint)$op2$$constant);
15927 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15928 %}
15929 ins_pipe(pipe_cmp_branch);
15930 %}
15931
15932 // Test bits
15933
15934 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15935 match(Set cr (CmpL (AndL op1 op2) op3));
15936 predicate(Assembler::operand_valid_for_logical_immediate
15937 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15938
15939 ins_cost(INSN_COST);
15940 format %{ "tst $op1, $op2 # long" %}
15941 ins_encode %{
15942 __ tst($op1$$Register, $op2$$constant);
15943 %}
15944 ins_pipe(ialu_reg_reg);
15945 %}
15946
15947 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15948 match(Set cr (CmpI (AndI op1 op2) op3));
15949 predicate(Assembler::operand_valid_for_logical_immediate
15950 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15951
15952 ins_cost(INSN_COST);
15953 format %{ "tst $op1, $op2 # int" %}
15954 ins_encode %{
15955 __ tstw($op1$$Register, $op2$$constant);
15956 %}
15957 ins_pipe(ialu_reg_reg);
15958 %}
15959
15960 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15961 match(Set cr (CmpL (AndL op1 op2) op3));
15962
15963 ins_cost(INSN_COST);
15964 format %{ "tst $op1, $op2 # long" %}
15965 ins_encode %{
15966 __ tst($op1$$Register, $op2$$Register);
15967 %}
15968 ins_pipe(ialu_reg_reg);
15969 %}
15970
15971 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15972 match(Set cr (CmpI (AndI op1 op2) op3));
15973
15974 ins_cost(INSN_COST);
15975 format %{ "tstw $op1, $op2 # int" %}
15976 ins_encode %{
15977 __ tstw($op1$$Register, $op2$$Register);
15978 %}
15979 ins_pipe(ialu_reg_reg);
15980 %}
15981
15982
15983 // Conditional Far Branch
15984 // Conditional Far Branch Unsigned
15985 // TODO: fixme
15986
15987 // counted loop end branch near
15988 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15989 %{
15990 match(CountedLoopEnd cmp cr);
15991
15992 effect(USE lbl);
15993
15994 ins_cost(BRANCH_COST);
15995 // short variant.
15996 // ins_short_branch(1);
15997 format %{ "b$cmp $lbl \t// counted loop end" %}
15998
15999 ins_encode(aarch64_enc_br_con(cmp, lbl));
16000
16001 ins_pipe(pipe_branch);
16002 %}
16003
16004 // counted loop end branch far
16005 // TODO: fixme
16006
16007 // ============================================================================
16008 // inlined locking and unlocking
16009
16010 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16011 %{
16012 predicate(LockingMode != LM_LIGHTWEIGHT);
16013 match(Set cr (FastLock object box));
16014 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16015
16016 ins_cost(5 * INSN_COST);
16017 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16018
16019 ins_encode %{
16020 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16021 %}
16022
16023 ins_pipe(pipe_serial);
16024 %}
16025
16026 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16027 %{
16028 predicate(LockingMode != LM_LIGHTWEIGHT);
16029 match(Set cr (FastUnlock object box));
16030 effect(TEMP tmp, TEMP tmp2);
16031
16032 ins_cost(5 * INSN_COST);
16033 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16034
16035 ins_encode %{
16036 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16037 %}
16038
16039 ins_pipe(pipe_serial);
16040 %}
16041
16042 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16043 %{
16044 predicate(LockingMode == LM_LIGHTWEIGHT);
16045 match(Set cr (FastLock object box));
16046 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16047
16048 ins_cost(5 * INSN_COST);
16049 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16050
16051 ins_encode %{
16052 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16053 %}
16054
16055 ins_pipe(pipe_serial);
16056 %}
16057
16058 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16059 %{
16060 predicate(LockingMode == LM_LIGHTWEIGHT);
16061 match(Set cr (FastUnlock object box));
16062 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16063
16064 ins_cost(5 * INSN_COST);
16065 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16066
16067 ins_encode %{
16068 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16069 %}
16070
16071 ins_pipe(pipe_serial);
16072 %}
16073
16074 // ============================================================================
16075 // Safepoint Instructions
16076
16077 // TODO
16078 // provide a near and far version of this code
16079
16080 instruct safePoint(rFlagsReg cr, iRegP poll)
16081 %{
16082 match(SafePoint poll);
16083 effect(KILL cr);
16084
16085 format %{
16086 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16087 %}
16088 ins_encode %{
16089 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16090 %}
16091 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16092 %}
16093
16094
16095 // ============================================================================
16096 // Procedure Call/Return Instructions
16097
16098 // Call Java Static Instruction
16099
16100 instruct CallStaticJavaDirect(method meth)
16101 %{
16102 match(CallStaticJava);
16103
16104 effect(USE meth);
16105
16106 ins_cost(CALL_COST);
16107
16108 format %{ "call,static $meth \t// ==> " %}
16109
16110 ins_encode(aarch64_enc_java_static_call(meth),
16111 aarch64_enc_call_epilog);
16112
16113 ins_pipe(pipe_class_call);
16114 %}
16115
16116 // TO HERE
16117
16118 // Call Java Dynamic Instruction
16119 instruct CallDynamicJavaDirect(method meth)
16120 %{
16121 match(CallDynamicJava);
16122
16123 effect(USE meth);
16124
16125 ins_cost(CALL_COST);
16126
16127 format %{ "CALL,dynamic $meth \t// ==> " %}
16128
16129 ins_encode(aarch64_enc_java_dynamic_call(meth),
16130 aarch64_enc_call_epilog);
16131
16132 ins_pipe(pipe_class_call);
16133 %}
16134
16135 // Call Runtime Instruction
16136
16137 instruct CallRuntimeDirect(method meth)
16138 %{
16139 match(CallRuntime);
16140
16141 effect(USE meth);
16142
16143 ins_cost(CALL_COST);
16144
16145 format %{ "CALL, runtime $meth" %}
16146
16147 ins_encode( aarch64_enc_java_to_runtime(meth) );
16148
16149 ins_pipe(pipe_class_call);
16150 %}
16151
16152 // Call Runtime Instruction
16153
16154 instruct CallLeafDirect(method meth)
16155 %{
16156 match(CallLeaf);
16157
16158 effect(USE meth);
16159
16160 ins_cost(CALL_COST);
16161
16162 format %{ "CALL, runtime leaf $meth" %}
16163
16164 ins_encode( aarch64_enc_java_to_runtime(meth) );
16165
16166 ins_pipe(pipe_class_call);
16167 %}
16168
16169 // Call Runtime Instruction without safepoint and with vector arguments
16170 instruct CallLeafDirectVector(method meth)
16171 %{
16172 match(CallLeafVector);
16173
16174 effect(USE meth);
16175
16176 ins_cost(CALL_COST);
16177
16178 format %{ "CALL, runtime leaf vector $meth" %}
16179
16180 ins_encode(aarch64_enc_java_to_runtime(meth));
16181
16182 ins_pipe(pipe_class_call);
16183 %}
16184
16185 // Call Runtime Instruction
16186
16187 instruct CallLeafNoFPDirect(method meth)
16188 %{
16189 match(CallLeafNoFP);
16190
16191 effect(USE meth);
16192
16193 ins_cost(CALL_COST);
16194
16195 format %{ "CALL, runtime leaf nofp $meth" %}
16196
16197 ins_encode( aarch64_enc_java_to_runtime(meth) );
16198
16199 ins_pipe(pipe_class_call);
16200 %}
16201
16202 // Tail Call; Jump from runtime stub to Java code.
16203 // Also known as an 'interprocedural jump'.
16204 // Target of jump will eventually return to caller.
16205 // TailJump below removes the return address.
16206 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16207 // emitted just above the TailCall which has reset rfp to the caller state.
16208 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16209 %{
16210 match(TailCall jump_target method_ptr);
16211
16212 ins_cost(CALL_COST);
16213
16214 format %{ "br $jump_target\t# $method_ptr holds method" %}
16215
16216 ins_encode(aarch64_enc_tail_call(jump_target));
16217
16218 ins_pipe(pipe_class_call);
16219 %}
16220
16221 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16222 %{
16223 match(TailJump jump_target ex_oop);
16224
16225 ins_cost(CALL_COST);
16226
16227 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16228
16229 ins_encode(aarch64_enc_tail_jmp(jump_target));
16230
16231 ins_pipe(pipe_class_call);
16232 %}
16233
16234 // Forward exception.
16235 instruct ForwardExceptionjmp()
16236 %{
16237 match(ForwardException);
16238 ins_cost(CALL_COST);
16239
16240 format %{ "b forward_exception_stub" %}
16241 ins_encode %{
16242 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16243 %}
16244 ins_pipe(pipe_class_call);
16245 %}
16246
16247 // Create exception oop: created by stack-crawling runtime code.
16248 // Created exception is now available to this handler, and is setup
16249 // just prior to jumping to this handler. No code emitted.
16250 // TODO check
16251 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16252 instruct CreateException(iRegP_R0 ex_oop)
16253 %{
16254 match(Set ex_oop (CreateEx));
16255
16256 format %{ " -- \t// exception oop; no code emitted" %}
16257
16258 size(0);
16259
16260 ins_encode( /*empty*/ );
16261
16262 ins_pipe(pipe_class_empty);
16263 %}
16264
16265 // Rethrow exception: The exception oop will come in the first
16266 // argument position. Then JUMP (not call) to the rethrow stub code.
16267 instruct RethrowException() %{
16268 match(Rethrow);
16269 ins_cost(CALL_COST);
16270
16271 format %{ "b rethrow_stub" %}
16272
16273 ins_encode( aarch64_enc_rethrow() );
16274
16275 ins_pipe(pipe_class_call);
16276 %}
16277
16278
16279 // Return Instruction
16280 // epilog node loads ret address into lr as part of frame pop
16281 instruct Ret()
16282 %{
16283 match(Return);
16284
16285 format %{ "ret\t// return register" %}
16286
16287 ins_encode( aarch64_enc_ret() );
16288
16289 ins_pipe(pipe_branch);
16290 %}
16291
16292 // Die now.
16293 instruct ShouldNotReachHere() %{
16294 match(Halt);
16295
16296 ins_cost(CALL_COST);
16297 format %{ "ShouldNotReachHere" %}
16298
16299 ins_encode %{
16300 if (is_reachable()) {
16301 __ stop(_halt_reason);
16302 }
16303 %}
16304
16305 ins_pipe(pipe_class_default);
16306 %}
16307
16308 // ============================================================================
16309 // Partial Subtype Check
16310 //
16311 // superklass array for an instance of the superklass. Set a hidden
16312 // internal cache on a hit (cache is checked with exposed code in
16313 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16314 // encoding ALSO sets flags.
16315
16316 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16317 %{
16318 match(Set result (PartialSubtypeCheck sub super));
16319 predicate(!UseSecondarySupersTable);
16320 effect(KILL cr, KILL temp);
16321
16322 ins_cost(20 * INSN_COST); // slightly larger than the next version
16323 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16324
16325 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16326
16327 opcode(0x1); // Force zero of result reg on hit
16328
16329 ins_pipe(pipe_class_memory);
16330 %}
16331
16332 // Two versions of partialSubtypeCheck, both used when we need to
16333 // search for a super class in the secondary supers array. The first
16334 // is used when we don't know _a priori_ the class being searched
16335 // for. The second, far more common, is used when we do know: this is
16336 // used for instanceof, checkcast, and any case where C2 can determine
16337 // it by constant propagation.
16338
16339 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16340 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16341 rFlagsReg cr)
16342 %{
16343 match(Set result (PartialSubtypeCheck sub super));
16344 predicate(UseSecondarySupersTable);
16345 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16346
16347 ins_cost(10 * INSN_COST); // slightly larger than the next version
16348 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16349
16350 ins_encode %{
16351 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16352 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16353 $vtemp$$FloatRegister,
16354 $result$$Register, /*L_success*/nullptr);
16355 %}
16356
16357 ins_pipe(pipe_class_memory);
16358 %}
16359
16360 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16361 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16362 rFlagsReg cr)
16363 %{
16364 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16365 predicate(UseSecondarySupersTable);
16366 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16367
16368 ins_cost(5 * INSN_COST); // smaller than the next version
16369 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16370
16371 ins_encode %{
16372 bool success = false;
16373 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16374 if (InlineSecondarySupersTest) {
16375 success =
16376 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16377 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16378 $vtemp$$FloatRegister,
16379 $result$$Register,
16380 super_klass_slot);
16381 } else {
16382 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16383 success = (call != nullptr);
16384 }
16385 if (!success) {
16386 ciEnv::current()->record_failure("CodeCache is full");
16387 return;
16388 }
16389 %}
16390
16391 ins_pipe(pipe_class_memory);
16392 %}
16393
16394 // Intrisics for String.compareTo()
16395
16396 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16397 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16398 %{
16399 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16400 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16401 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16402
16403 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16404 ins_encode %{
16405 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16406 __ string_compare($str1$$Register, $str2$$Register,
16407 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16408 $tmp1$$Register, $tmp2$$Register,
16409 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16410 %}
16411 ins_pipe(pipe_class_memory);
16412 %}
16413
16414 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16415 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16416 %{
16417 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16418 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16419 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16420
16421 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16422 ins_encode %{
16423 __ string_compare($str1$$Register, $str2$$Register,
16424 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16425 $tmp1$$Register, $tmp2$$Register,
16426 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16427 %}
16428 ins_pipe(pipe_class_memory);
16429 %}
16430
16431 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16432 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16433 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16434 %{
16435 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16436 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16437 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16438 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16439
16440 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16441 ins_encode %{
16442 __ string_compare($str1$$Register, $str2$$Register,
16443 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16444 $tmp1$$Register, $tmp2$$Register,
16445 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16446 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16447 %}
16448 ins_pipe(pipe_class_memory);
16449 %}
16450
16451 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16452 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16453 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16454 %{
16455 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16456 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16457 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16458 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16459
16460 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16461 ins_encode %{
16462 __ string_compare($str1$$Register, $str2$$Register,
16463 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16464 $tmp1$$Register, $tmp2$$Register,
16465 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16466 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16467 %}
16468 ins_pipe(pipe_class_memory);
16469 %}
16470
16471 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16472 // these string_compare variants as NEON register type for convenience so that the prototype of
16473 // string_compare can be shared with all variants.
16474
16475 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16476 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16477 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16478 pRegGov_P1 pgtmp2, rFlagsReg cr)
16479 %{
16480 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16481 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16482 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16483 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16484
16485 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16486 ins_encode %{
16487 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16488 __ string_compare($str1$$Register, $str2$$Register,
16489 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16490 $tmp1$$Register, $tmp2$$Register,
16491 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16492 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16493 StrIntrinsicNode::LL);
16494 %}
16495 ins_pipe(pipe_class_memory);
16496 %}
16497
16498 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16499 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16500 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16501 pRegGov_P1 pgtmp2, rFlagsReg cr)
16502 %{
16503 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16504 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16505 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16506 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16507
16508 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16509 ins_encode %{
16510 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16511 __ string_compare($str1$$Register, $str2$$Register,
16512 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16513 $tmp1$$Register, $tmp2$$Register,
16514 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16515 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16516 StrIntrinsicNode::LU);
16517 %}
16518 ins_pipe(pipe_class_memory);
16519 %}
16520
16521 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16522 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16523 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16524 pRegGov_P1 pgtmp2, rFlagsReg cr)
16525 %{
16526 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16527 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16528 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16529 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16530
16531 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16532 ins_encode %{
16533 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16534 __ string_compare($str1$$Register, $str2$$Register,
16535 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16536 $tmp1$$Register, $tmp2$$Register,
16537 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16538 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16539 StrIntrinsicNode::UL);
16540 %}
16541 ins_pipe(pipe_class_memory);
16542 %}
16543
16544 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16545 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16546 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16547 pRegGov_P1 pgtmp2, rFlagsReg cr)
16548 %{
16549 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16550 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16551 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16552 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16553
16554 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16555 ins_encode %{
16556 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16557 __ string_compare($str1$$Register, $str2$$Register,
16558 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16559 $tmp1$$Register, $tmp2$$Register,
16560 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16561 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16562 StrIntrinsicNode::UU);
16563 %}
16564 ins_pipe(pipe_class_memory);
16565 %}
16566
16567 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16568 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16569 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16570 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16571 %{
16572 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16573 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16574 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16575 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16576 TEMP vtmp0, TEMP vtmp1, KILL cr);
16577 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16578 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16579
16580 ins_encode %{
16581 __ string_indexof($str1$$Register, $str2$$Register,
16582 $cnt1$$Register, $cnt2$$Register,
16583 $tmp1$$Register, $tmp2$$Register,
16584 $tmp3$$Register, $tmp4$$Register,
16585 $tmp5$$Register, $tmp6$$Register,
16586 -1, $result$$Register, StrIntrinsicNode::UU);
16587 %}
16588 ins_pipe(pipe_class_memory);
16589 %}
16590
16591 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16592 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16593 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16594 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16595 %{
16596 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16597 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16598 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16599 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16600 TEMP vtmp0, TEMP vtmp1, KILL cr);
16601 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16602 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16603
16604 ins_encode %{
16605 __ string_indexof($str1$$Register, $str2$$Register,
16606 $cnt1$$Register, $cnt2$$Register,
16607 $tmp1$$Register, $tmp2$$Register,
16608 $tmp3$$Register, $tmp4$$Register,
16609 $tmp5$$Register, $tmp6$$Register,
16610 -1, $result$$Register, StrIntrinsicNode::LL);
16611 %}
16612 ins_pipe(pipe_class_memory);
16613 %}
16614
16615 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16616 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16617 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16618 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16619 %{
16620 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16621 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16622 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16623 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16624 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16625 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16626 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16627
16628 ins_encode %{
16629 __ string_indexof($str1$$Register, $str2$$Register,
16630 $cnt1$$Register, $cnt2$$Register,
16631 $tmp1$$Register, $tmp2$$Register,
16632 $tmp3$$Register, $tmp4$$Register,
16633 $tmp5$$Register, $tmp6$$Register,
16634 -1, $result$$Register, StrIntrinsicNode::UL);
16635 %}
16636 ins_pipe(pipe_class_memory);
16637 %}
16638
16639 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16640 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16641 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16642 %{
16643 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16644 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16645 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16646 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16647 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16648 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16649
16650 ins_encode %{
16651 int icnt2 = (int)$int_cnt2$$constant;
16652 __ string_indexof($str1$$Register, $str2$$Register,
16653 $cnt1$$Register, zr,
16654 $tmp1$$Register, $tmp2$$Register,
16655 $tmp3$$Register, $tmp4$$Register, zr, zr,
16656 icnt2, $result$$Register, StrIntrinsicNode::UU);
16657 %}
16658 ins_pipe(pipe_class_memory);
16659 %}
16660
16661 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16662 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16663 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16664 %{
16665 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16666 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16667 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16668 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16669 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16670 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16671
16672 ins_encode %{
16673 int icnt2 = (int)$int_cnt2$$constant;
16674 __ string_indexof($str1$$Register, $str2$$Register,
16675 $cnt1$$Register, zr,
16676 $tmp1$$Register, $tmp2$$Register,
16677 $tmp3$$Register, $tmp4$$Register, zr, zr,
16678 icnt2, $result$$Register, StrIntrinsicNode::LL);
16679 %}
16680 ins_pipe(pipe_class_memory);
16681 %}
16682
16683 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16684 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16685 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16686 %{
16687 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16688 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16689 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16690 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16691 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16692 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16693
16694 ins_encode %{
16695 int icnt2 = (int)$int_cnt2$$constant;
16696 __ string_indexof($str1$$Register, $str2$$Register,
16697 $cnt1$$Register, zr,
16698 $tmp1$$Register, $tmp2$$Register,
16699 $tmp3$$Register, $tmp4$$Register, zr, zr,
16700 icnt2, $result$$Register, StrIntrinsicNode::UL);
16701 %}
16702 ins_pipe(pipe_class_memory);
16703 %}
16704
16705 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16706 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16707 iRegINoSp tmp3, rFlagsReg cr)
16708 %{
16709 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16710 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16711 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16712 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16713
16714 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16715
16716 ins_encode %{
16717 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16718 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16719 $tmp3$$Register);
16720 %}
16721 ins_pipe(pipe_class_memory);
16722 %}
16723
16724 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16725 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16726 iRegINoSp tmp3, rFlagsReg cr)
16727 %{
16728 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16729 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16730 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16731 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16732
16733 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16734
16735 ins_encode %{
16736 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16737 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16738 $tmp3$$Register);
16739 %}
16740 ins_pipe(pipe_class_memory);
16741 %}
16742
16743 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16744 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16745 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16746 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16747 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16748 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16749 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16750 ins_encode %{
16751 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16752 $result$$Register, $ztmp1$$FloatRegister,
16753 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16754 $ptmp$$PRegister, true /* isL */);
16755 %}
16756 ins_pipe(pipe_class_memory);
16757 %}
16758
16759 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16760 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16761 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16762 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16763 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16764 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16765 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16766 ins_encode %{
16767 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16768 $result$$Register, $ztmp1$$FloatRegister,
16769 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16770 $ptmp$$PRegister, false /* isL */);
16771 %}
16772 ins_pipe(pipe_class_memory);
16773 %}
16774
16775 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16776 iRegI_R0 result, rFlagsReg cr)
16777 %{
16778 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16779 match(Set result (StrEquals (Binary str1 str2) cnt));
16780 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16781
16782 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16783 ins_encode %{
16784 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16785 __ string_equals($str1$$Register, $str2$$Register,
16786 $result$$Register, $cnt$$Register);
16787 %}
16788 ins_pipe(pipe_class_memory);
16789 %}
16790
16791 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16792 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16793 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16794 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16795 iRegP_R10 tmp, rFlagsReg cr)
16796 %{
16797 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16798 match(Set result (AryEq ary1 ary2));
16799 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16800 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16801 TEMP vtmp6, TEMP vtmp7, KILL cr);
16802
16803 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16804 ins_encode %{
16805 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16806 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16807 $result$$Register, $tmp$$Register, 1);
16808 if (tpc == nullptr) {
16809 ciEnv::current()->record_failure("CodeCache is full");
16810 return;
16811 }
16812 %}
16813 ins_pipe(pipe_class_memory);
16814 %}
16815
16816 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16817 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16818 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16819 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16820 iRegP_R10 tmp, rFlagsReg cr)
16821 %{
16822 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16823 match(Set result (AryEq ary1 ary2));
16824 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16825 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16826 TEMP vtmp6, TEMP vtmp7, KILL cr);
16827
16828 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16829 ins_encode %{
16830 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16831 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16832 $result$$Register, $tmp$$Register, 2);
16833 if (tpc == nullptr) {
16834 ciEnv::current()->record_failure("CodeCache is full");
16835 return;
16836 }
16837 %}
16838 ins_pipe(pipe_class_memory);
16839 %}
16840
16841 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16842 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16843 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16844 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16845 %{
16846 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16847 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16848 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16849
16850 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16851 ins_encode %{
16852 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16853 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16854 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16855 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16856 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16857 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16858 (BasicType)$basic_type$$constant);
16859 if (tpc == nullptr) {
16860 ciEnv::current()->record_failure("CodeCache is full");
16861 return;
16862 }
16863 %}
16864 ins_pipe(pipe_class_memory);
16865 %}
16866
16867 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16868 %{
16869 match(Set result (CountPositives ary1 len));
16870 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16871 format %{ "count positives byte[] $ary1,$len -> $result" %}
16872 ins_encode %{
16873 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16874 if (tpc == nullptr) {
16875 ciEnv::current()->record_failure("CodeCache is full");
16876 return;
16877 }
16878 %}
16879 ins_pipe( pipe_slow );
16880 %}
16881
16882 // fast char[] to byte[] compression
16883 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16884 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16885 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16886 iRegI_R0 result, rFlagsReg cr)
16887 %{
16888 match(Set result (StrCompressedCopy src (Binary dst len)));
16889 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16890 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16891
16892 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16893 ins_encode %{
16894 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16895 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16896 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16897 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16898 %}
16899 ins_pipe(pipe_slow);
16900 %}
16901
16902 // fast byte[] to char[] inflation
16903 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16904 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16905 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16906 %{
16907 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16908 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16909 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16910 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16911
16912 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16913 ins_encode %{
16914 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16915 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16916 $vtmp2$$FloatRegister, $tmp$$Register);
16917 if (tpc == nullptr) {
16918 ciEnv::current()->record_failure("CodeCache is full");
16919 return;
16920 }
16921 %}
16922 ins_pipe(pipe_class_memory);
16923 %}
16924
16925 // encode char[] to byte[] in ISO_8859_1
16926 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16927 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16928 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16929 iRegI_R0 result, rFlagsReg cr)
16930 %{
16931 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16932 match(Set result (EncodeISOArray src (Binary dst len)));
16933 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16934 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16935
16936 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16937 ins_encode %{
16938 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16939 $result$$Register, false,
16940 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16941 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16942 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16943 %}
16944 ins_pipe(pipe_class_memory);
16945 %}
16946
16947 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16948 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16949 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16950 iRegI_R0 result, rFlagsReg cr)
16951 %{
16952 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16953 match(Set result (EncodeISOArray src (Binary dst len)));
16954 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16955 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16956
16957 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16958 ins_encode %{
16959 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16960 $result$$Register, true,
16961 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16962 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16963 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16964 %}
16965 ins_pipe(pipe_class_memory);
16966 %}
16967
16968 //----------------------------- CompressBits/ExpandBits ------------------------
16969
16970 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16971 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16972 match(Set dst (CompressBits src mask));
16973 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16974 format %{ "mov $tsrc, $src\n\t"
16975 "mov $tmask, $mask\n\t"
16976 "bext $tdst, $tsrc, $tmask\n\t"
16977 "mov $dst, $tdst"
16978 %}
16979 ins_encode %{
16980 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16981 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16982 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16983 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16984 %}
16985 ins_pipe(pipe_slow);
16986 %}
16987
16988 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16989 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16990 match(Set dst (CompressBits (LoadI mem) mask));
16991 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16992 format %{ "ldrs $tsrc, $mem\n\t"
16993 "ldrs $tmask, $mask\n\t"
16994 "bext $tdst, $tsrc, $tmask\n\t"
16995 "mov $dst, $tdst"
16996 %}
16997 ins_encode %{
16998 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17000 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17001 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17002 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17003 %}
17004 ins_pipe(pipe_slow);
17005 %}
17006
17007 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17008 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17009 match(Set dst (CompressBits src mask));
17010 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17011 format %{ "mov $tsrc, $src\n\t"
17012 "mov $tmask, $mask\n\t"
17013 "bext $tdst, $tsrc, $tmask\n\t"
17014 "mov $dst, $tdst"
17015 %}
17016 ins_encode %{
17017 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17018 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17019 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17020 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17021 %}
17022 ins_pipe(pipe_slow);
17023 %}
17024
17025 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17026 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17027 match(Set dst (CompressBits (LoadL mem) mask));
17028 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17029 format %{ "ldrd $tsrc, $mem\n\t"
17030 "ldrd $tmask, $mask\n\t"
17031 "bext $tdst, $tsrc, $tmask\n\t"
17032 "mov $dst, $tdst"
17033 %}
17034 ins_encode %{
17035 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17036 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17037 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17038 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17039 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17040 %}
17041 ins_pipe(pipe_slow);
17042 %}
17043
17044 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17045 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17046 match(Set dst (ExpandBits src mask));
17047 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17048 format %{ "mov $tsrc, $src\n\t"
17049 "mov $tmask, $mask\n\t"
17050 "bdep $tdst, $tsrc, $tmask\n\t"
17051 "mov $dst, $tdst"
17052 %}
17053 ins_encode %{
17054 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17055 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17056 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17057 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17058 %}
17059 ins_pipe(pipe_slow);
17060 %}
17061
17062 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17063 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17064 match(Set dst (ExpandBits (LoadI mem) mask));
17065 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17066 format %{ "ldrs $tsrc, $mem\n\t"
17067 "ldrs $tmask, $mask\n\t"
17068 "bdep $tdst, $tsrc, $tmask\n\t"
17069 "mov $dst, $tdst"
17070 %}
17071 ins_encode %{
17072 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17073 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17074 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17075 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17076 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17077 %}
17078 ins_pipe(pipe_slow);
17079 %}
17080
17081 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17082 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17083 match(Set dst (ExpandBits src mask));
17084 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17085 format %{ "mov $tsrc, $src\n\t"
17086 "mov $tmask, $mask\n\t"
17087 "bdep $tdst, $tsrc, $tmask\n\t"
17088 "mov $dst, $tdst"
17089 %}
17090 ins_encode %{
17091 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17092 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17093 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17094 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17095 %}
17096 ins_pipe(pipe_slow);
17097 %}
17098
17099
17100 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17101 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17102 match(Set dst (ExpandBits (LoadL mem) mask));
17103 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17104 format %{ "ldrd $tsrc, $mem\n\t"
17105 "ldrd $tmask, $mask\n\t"
17106 "bdep $tdst, $tsrc, $tmask\n\t"
17107 "mov $dst, $tdst"
17108 %}
17109 ins_encode %{
17110 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17111 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17112 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17113 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17114 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17115 %}
17116 ins_pipe(pipe_slow);
17117 %}
17118
17119 // ============================================================================
17120 // This name is KNOWN by the ADLC and cannot be changed.
17121 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17122 // for this guy.
17123 instruct tlsLoadP(thread_RegP dst)
17124 %{
17125 match(Set dst (ThreadLocal));
17126
17127 ins_cost(0);
17128
17129 format %{ " -- \t// $dst=Thread::current(), empty" %}
17130
17131 size(0);
17132
17133 ins_encode( /*empty*/ );
17134
17135 ins_pipe(pipe_class_empty);
17136 %}
17137
17138 //----------PEEPHOLE RULES-----------------------------------------------------
17139 // These must follow all instruction definitions as they use the names
17140 // defined in the instructions definitions.
17141 //
17142 // peepmatch ( root_instr_name [preceding_instruction]* );
17143 //
17144 // peepconstraint %{
17145 // (instruction_number.operand_name relational_op instruction_number.operand_name
17146 // [, ...] );
17147 // // instruction numbers are zero-based using left to right order in peepmatch
17148 //
17149 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17150 // // provide an instruction_number.operand_name for each operand that appears
17151 // // in the replacement instruction's match rule
17152 //
17153 // ---------VM FLAGS---------------------------------------------------------
17154 //
17155 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17156 //
17157 // Each peephole rule is given an identifying number starting with zero and
17158 // increasing by one in the order seen by the parser. An individual peephole
17159 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17160 // on the command-line.
17161 //
17162 // ---------CURRENT LIMITATIONS----------------------------------------------
17163 //
17164 // Only match adjacent instructions in same basic block
17165 // Only equality constraints
17166 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17167 // Only one replacement instruction
17168 //
17169 // ---------EXAMPLE----------------------------------------------------------
17170 //
17171 // // pertinent parts of existing instructions in architecture description
17172 // instruct movI(iRegINoSp dst, iRegI src)
17173 // %{
17174 // match(Set dst (CopyI src));
17175 // %}
17176 //
17177 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17178 // %{
17179 // match(Set dst (AddI dst src));
17180 // effect(KILL cr);
17181 // %}
17182 //
17183 // // Change (inc mov) to lea
17184 // peephole %{
17185 // // increment preceded by register-register move
17186 // peepmatch ( incI_iReg movI );
17187 // // require that the destination register of the increment
17188 // // match the destination register of the move
17189 // peepconstraint ( 0.dst == 1.dst );
17190 // // construct a replacement instruction that sets
17191 // // the destination to ( move's source register + one )
17192 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17193 // %}
17194 //
17195
17196 // Implementation no longer uses movX instructions since
17197 // machine-independent system no longer uses CopyX nodes.
17198 //
17199 // peephole
17200 // %{
17201 // peepmatch (incI_iReg movI);
17202 // peepconstraint (0.dst == 1.dst);
17203 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17204 // %}
17205
17206 // peephole
17207 // %{
17208 // peepmatch (decI_iReg movI);
17209 // peepconstraint (0.dst == 1.dst);
17210 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17211 // %}
17212
17213 // peephole
17214 // %{
17215 // peepmatch (addI_iReg_imm movI);
17216 // peepconstraint (0.dst == 1.dst);
17217 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17218 // %}
17219
17220 // peephole
17221 // %{
17222 // peepmatch (incL_iReg movL);
17223 // peepconstraint (0.dst == 1.dst);
17224 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17225 // %}
17226
17227 // peephole
17228 // %{
17229 // peepmatch (decL_iReg movL);
17230 // peepconstraint (0.dst == 1.dst);
17231 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17232 // %}
17233
17234 // peephole
17235 // %{
17236 // peepmatch (addL_iReg_imm movL);
17237 // peepconstraint (0.dst == 1.dst);
17238 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17239 // %}
17240
17241 // peephole
17242 // %{
17243 // peepmatch (addP_iReg_imm movP);
17244 // peepconstraint (0.dst == 1.dst);
17245 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17246 // %}
17247
17248 // // Change load of spilled value to only a spill
17249 // instruct storeI(memory mem, iRegI src)
17250 // %{
17251 // match(Set mem (StoreI mem src));
17252 // %}
17253 //
17254 // instruct loadI(iRegINoSp dst, memory mem)
17255 // %{
17256 // match(Set dst (LoadI mem));
17257 // %}
17258 //
17259
17260 //----------SMARTSPILL RULES---------------------------------------------------
17261 // These must follow all instruction definitions as they use the names
17262 // defined in the instructions definitions.
17263
17264 // Local Variables:
17265 // mode: c++
17266 // End: