1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 bool Matcher::supports_vector_calling_convention(void) {
2311 return EnableVectorSupport && UseVectorStubs;
2312 }
2313
2314 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2315 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2316 int lo = V0_num;
2317 int hi = V0_H_num;
2318 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2319 hi = V0_K_num;
2320 }
2321 return OptoRegPair(hi, lo);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635
2636 // Loads and stores with indirect memory input (e.g., volatile loads and
2637 // stores) do not subsume the input into complex addressing expressions. If
2638 // the addressing expression is input to at least one such load or store, do
2639 // not clone the addressing expression. Query needs_acquiring_load and
2640 // needs_releasing_store as a proxy for indirect memory input, as it is not
2641 // possible to directly query for indirect memory input at this stage.
2642 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2643 Node* n = m->fast_out(i);
2644 if (n->is_Load() && needs_acquiring_load(n)) {
2645 return false;
2646 }
2647 if (n->is_Store() && needs_releasing_store(n)) {
2648 return false;
2649 }
2650 }
2651
2652 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2653 return true;
2654 }
2655
2656 Node *off = m->in(AddPNode::Offset);
2657 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2658 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2659 // Are there other uses besides address expressions?
2660 !is_visited(off)) {
2661 address_visited.set(off->_idx); // Flag as address_visited
2662 mstack.push(off->in(2), Visit);
2663 Node *conv = off->in(1);
2664 if (conv->Opcode() == Op_ConvI2L &&
2665 // Are there other uses besides address expressions?
2666 !is_visited(conv)) {
2667 address_visited.set(conv->_idx); // Flag as address_visited
2668 mstack.push(conv->in(1), Pre_Visit);
2669 } else {
2670 mstack.push(conv, Pre_Visit);
2671 }
2672 address_visited.test_set(m->_idx); // Flag as address_visited
2673 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2674 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2675 return true;
2676 } else if (off->Opcode() == Op_ConvI2L &&
2677 // Are there other uses besides address expressions?
2678 !is_visited(off)) {
2679 address_visited.test_set(m->_idx); // Flag as address_visited
2680 address_visited.set(off->_idx); // Flag as address_visited
2681 mstack.push(off->in(1), Pre_Visit);
2682 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2683 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2684 return true;
2685 }
2686 return false;
2687 }
2688
2689 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2690 { \
2691 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2692 guarantee(DISP == 0, "mode not permitted for volatile"); \
2693 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2694 __ INSN(REG, as_Register(BASE)); \
2695 }
2696
2697
2698 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2699 {
2700 Address::extend scale;
2701
2702 // Hooboy, this is fugly. We need a way to communicate to the
2703 // encoder that the index needs to be sign extended, so we have to
2704 // enumerate all the cases.
2705 switch (opcode) {
2706 case INDINDEXSCALEDI2L:
2707 case INDINDEXSCALEDI2LN:
2708 case INDINDEXI2L:
2709 case INDINDEXI2LN:
2710 scale = Address::sxtw(size);
2711 break;
2712 default:
2713 scale = Address::lsl(size);
2714 }
2715
2716 if (index == -1) {
2717 return Address(base, disp);
2718 } else {
2719 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2720 return Address(base, as_Register(index), scale);
2721 }
2722 }
2723
2724
2725 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2727 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2728 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2729 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2730
2731 // Used for all non-volatile memory accesses. The use of
2732 // $mem->opcode() to discover whether this pattern uses sign-extended
2733 // offsets is something of a kludge.
2734 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2735 Register reg, int opcode,
2736 Register base, int index, int scale, int disp,
2737 int size_in_memory)
2738 {
2739 Address addr = mem2address(opcode, base, index, scale, disp);
2740 if (addr.getMode() == Address::base_plus_offset) {
2741 /* Fix up any out-of-range offsets. */
2742 assert_different_registers(rscratch1, base);
2743 assert_different_registers(rscratch1, reg);
2744 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2745 }
2746 (masm->*insn)(reg, addr);
2747 }
2748
2749 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2750 FloatRegister reg, int opcode,
2751 Register base, int index, int size, int disp,
2752 int size_in_memory)
2753 {
2754 Address::extend scale;
2755
2756 switch (opcode) {
2757 case INDINDEXSCALEDI2L:
2758 case INDINDEXSCALEDI2LN:
2759 scale = Address::sxtw(size);
2760 break;
2761 default:
2762 scale = Address::lsl(size);
2763 }
2764
2765 if (index == -1) {
2766 // Fix up any out-of-range offsets.
2767 assert_different_registers(rscratch1, base);
2768 Address addr = Address(base, disp);
2769 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2770 (masm->*insn)(reg, addr);
2771 } else {
2772 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2773 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2774 }
2775 }
2776
2777 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2778 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2779 int opcode, Register base, int index, int size, int disp)
2780 {
2781 if (index == -1) {
2782 (masm->*insn)(reg, T, Address(base, disp));
2783 } else {
2784 assert(disp == 0, "unsupported address mode");
2785 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2786 }
2787 }
2788
2789 %}
2790
2791
2792
2793 //----------ENCODING BLOCK-----------------------------------------------------
2794 // This block specifies the encoding classes used by the compiler to
2795 // output byte streams. Encoding classes are parameterized macros
2796 // used by Machine Instruction Nodes in order to generate the bit
2797 // encoding of the instruction. Operands specify their base encoding
2798 // interface with the interface keyword. There are currently
2799 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2800 // COND_INTER. REG_INTER causes an operand to generate a function
2801 // which returns its register number when queried. CONST_INTER causes
2802 // an operand to generate a function which returns the value of the
2803 // constant when queried. MEMORY_INTER causes an operand to generate
2804 // four functions which return the Base Register, the Index Register,
2805 // the Scale Value, and the Offset Value of the operand when queried.
2806 // COND_INTER causes an operand to generate six functions which return
2807 // the encoding code (ie - encoding bits for the instruction)
2808 // associated with each basic boolean condition for a conditional
2809 // instruction.
2810 //
2811 // Instructions specify two basic values for encoding. Again, a
2812 // function is available to check if the constant displacement is an
2813 // oop. They use the ins_encode keyword to specify their encoding
2814 // classes (which must be a sequence of enc_class names, and their
2815 // parameters, specified in the encoding block), and they use the
2816 // opcode keyword to specify, in order, their primary, secondary, and
2817 // tertiary opcode. Only the opcode sections which a particular
2818 // instruction needs for encoding need to be specified.
2819 encode %{
2820 // Build emit functions for each basic byte or larger field in the
2821 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2822 // from C++ code in the enc_class source block. Emit functions will
2823 // live in the main source block for now. In future, we can
2824 // generalize this by adding a syntax that specifies the sizes of
2825 // fields in an order, so that the adlc can build the emit functions
2826 // automagically
2827
2828 // catch all for unimplemented encodings
2829 enc_class enc_unimplemented %{
2830 __ unimplemented("C2 catch all");
2831 %}
2832
2833 // BEGIN Non-volatile memory access
2834
2835 // This encoding class is generated automatically from ad_encode.m4.
2836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2837 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2838 Register dst_reg = as_Register($dst$$reg);
2839 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2841 %}
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2934 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2942 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2950 Register src_reg = as_Register($src$$reg);
2951 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_strb0(memory1 mem) %{
2958 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strh0(memory2 mem) %{
2973 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2980 Register src_reg = as_Register($src$$reg);
2981 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strw0(memory4 mem) %{
2988 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2989 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2990 %}
2991
2992 // This encoding class is generated automatically from ad_encode.m4.
2993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2994 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2995 Register src_reg = as_Register($src$$reg);
2996 // we sometimes get asked to store the stack pointer into the
2997 // current thread -- we cannot do that directly on AArch64
2998 if (src_reg == r31_sp) {
2999 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3000 __ mov(rscratch2, sp);
3001 src_reg = rscratch2;
3002 }
3003 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_str0(memory8 mem) %{
3010 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3017 FloatRegister src_reg = as_FloatRegister($src$$reg);
3018 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3025 FloatRegister src_reg = as_FloatRegister($src$$reg);
3026 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3028 %}
3029
3030 // This encoding class is generated automatically from ad_encode.m4.
3031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3032 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3033 __ membar(Assembler::StoreStore);
3034 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3036 %}
3037
3038 // END Non-volatile memory access
3039
3040 // Vector loads and stores
3041 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3042 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3043 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3044 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3045 %}
3046
3047 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3048 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3049 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3050 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3051 %}
3052
3053 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3066 FloatRegister src_reg = as_FloatRegister($src$$reg);
3067 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3072 FloatRegister src_reg = as_FloatRegister($src$$reg);
3073 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 // volatile loads and stores
3090
3091 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3092 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3093 rscratch1, stlrb);
3094 %}
3095
3096 enc_class aarch64_enc_stlrb0(memory mem) %{
3097 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3102 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrh);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh0(memory mem) %{
3107 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3112 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrw);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw0(memory mem) %{
3117 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3122 Register dst_reg = as_Register($dst$$reg);
3123 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3124 rscratch1, ldarb);
3125 __ sxtbw(dst_reg, dst_reg);
3126 %}
3127
3128 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarb);
3132 __ sxtb(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3136 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, ldarb);
3138 %}
3139
3140 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3146 Register dst_reg = as_Register($dst$$reg);
3147 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, ldarh);
3149 __ sxthw(dst_reg, dst_reg);
3150 %}
3151
3152 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3153 Register dst_reg = as_Register($dst$$reg);
3154 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarh);
3156 __ sxth(dst_reg, dst_reg);
3157 %}
3158
3159 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3160 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarh);
3162 %}
3163
3164 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarw);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldar);
3182 %}
3183
3184 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3185 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldarw);
3187 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3188 %}
3189
3190 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3191 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldar);
3193 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3194 %}
3195
3196 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3197 Register src_reg = as_Register($src$$reg);
3198 // we sometimes get asked to store the stack pointer into the
3199 // current thread -- we cannot do that directly on AArch64
3200 if (src_reg == r31_sp) {
3201 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3202 __ mov(rscratch2, sp);
3203 src_reg = rscratch2;
3204 }
3205 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, stlr);
3207 %}
3208
3209 enc_class aarch64_enc_stlr0(memory mem) %{
3210 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, stlr);
3212 %}
3213
3214 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3215 {
3216 FloatRegister src_reg = as_FloatRegister($src$$reg);
3217 __ fmovs(rscratch2, src_reg);
3218 }
3219 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, stlrw);
3221 %}
3222
3223 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3224 {
3225 FloatRegister src_reg = as_FloatRegister($src$$reg);
3226 __ fmovd(rscratch2, src_reg);
3227 }
3228 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, stlr);
3230 %}
3231
3232 // synchronized read/update encodings
3233
3234 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3235 Register dst_reg = as_Register($dst$$reg);
3236 Register base = as_Register($mem$$base);
3237 int index = $mem$$index;
3238 int scale = $mem$$scale;
3239 int disp = $mem$$disp;
3240 if (index == -1) {
3241 if (disp != 0) {
3242 __ lea(rscratch1, Address(base, disp));
3243 __ ldaxr(dst_reg, rscratch1);
3244 } else {
3245 // TODO
3246 // should we ever get anything other than this case?
3247 __ ldaxr(dst_reg, base);
3248 }
3249 } else {
3250 Register index_reg = as_Register(index);
3251 if (disp == 0) {
3252 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3253 __ ldaxr(dst_reg, rscratch1);
3254 } else {
3255 __ lea(rscratch1, Address(base, disp));
3256 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3257 __ ldaxr(dst_reg, rscratch1);
3258 }
3259 }
3260 %}
3261
3262 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3263 Register src_reg = as_Register($src$$reg);
3264 Register base = as_Register($mem$$base);
3265 int index = $mem$$index;
3266 int scale = $mem$$scale;
3267 int disp = $mem$$disp;
3268 if (index == -1) {
3269 if (disp != 0) {
3270 __ lea(rscratch2, Address(base, disp));
3271 __ stlxr(rscratch1, src_reg, rscratch2);
3272 } else {
3273 // TODO
3274 // should we ever get anything other than this case?
3275 __ stlxr(rscratch1, src_reg, base);
3276 }
3277 } else {
3278 Register index_reg = as_Register(index);
3279 if (disp == 0) {
3280 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3281 __ stlxr(rscratch1, src_reg, rscratch2);
3282 } else {
3283 __ lea(rscratch2, Address(base, disp));
3284 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3285 __ stlxr(rscratch1, src_reg, rscratch2);
3286 }
3287 }
3288 __ cmpw(rscratch1, zr);
3289 %}
3290
3291 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3292 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3293 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3294 Assembler::xword, /*acquire*/ false, /*release*/ true,
3295 /*weak*/ false, noreg);
3296 %}
3297
3298 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3299 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3300 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3301 Assembler::word, /*acquire*/ false, /*release*/ true,
3302 /*weak*/ false, noreg);
3303 %}
3304
3305 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3306 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3307 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3308 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3309 /*weak*/ false, noreg);
3310 %}
3311
3312 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3313 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3314 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3315 Assembler::byte, /*acquire*/ false, /*release*/ true,
3316 /*weak*/ false, noreg);
3317 %}
3318
3319
3320 // The only difference between aarch64_enc_cmpxchg and
3321 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3322 // CompareAndSwap sequence to serve as a barrier on acquiring a
3323 // lock.
3324 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::xword, /*acquire*/ true, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3332 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3333 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3334 Assembler::word, /*acquire*/ true, /*release*/ true,
3335 /*weak*/ false, noreg);
3336 %}
3337
3338 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3339 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3340 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3341 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3342 /*weak*/ false, noreg);
3343 %}
3344
3345 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3346 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3347 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3348 Assembler::byte, /*acquire*/ true, /*release*/ true,
3349 /*weak*/ false, noreg);
3350 %}
3351
3352 // auxiliary used for CompareAndSwapX to set result register
3353 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3354 Register res_reg = as_Register($res$$reg);
3355 __ cset(res_reg, Assembler::EQ);
3356 %}
3357
3358 // prefetch encodings
3359
3360 enc_class aarch64_enc_prefetchw(memory mem) %{
3361 Register base = as_Register($mem$$base);
3362 int index = $mem$$index;
3363 int scale = $mem$$scale;
3364 int disp = $mem$$disp;
3365 if (index == -1) {
3366 // Fix up any out-of-range offsets.
3367 assert_different_registers(rscratch1, base);
3368 Address addr = Address(base, disp);
3369 addr = __ legitimize_address(addr, 8, rscratch1);
3370 __ prfm(addr, PSTL1KEEP);
3371 } else {
3372 Register index_reg = as_Register(index);
3373 if (disp == 0) {
3374 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3375 } else {
3376 __ lea(rscratch1, Address(base, disp));
3377 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3378 }
3379 }
3380 %}
3381
3382 // mov encodings
3383
3384 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3385 uint32_t con = (uint32_t)$src$$constant;
3386 Register dst_reg = as_Register($dst$$reg);
3387 if (con == 0) {
3388 __ movw(dst_reg, zr);
3389 } else {
3390 __ movw(dst_reg, con);
3391 }
3392 %}
3393
3394 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3395 Register dst_reg = as_Register($dst$$reg);
3396 uint64_t con = (uint64_t)$src$$constant;
3397 if (con == 0) {
3398 __ mov(dst_reg, zr);
3399 } else {
3400 __ mov(dst_reg, con);
3401 }
3402 %}
3403
3404 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3405 Register dst_reg = as_Register($dst$$reg);
3406 address con = (address)$src$$constant;
3407 if (con == nullptr || con == (address)1) {
3408 ShouldNotReachHere();
3409 } else {
3410 relocInfo::relocType rtype = $src->constant_reloc();
3411 if (rtype == relocInfo::oop_type) {
3412 __ movoop(dst_reg, (jobject)con);
3413 } else if (rtype == relocInfo::metadata_type) {
3414 __ mov_metadata(dst_reg, (Metadata*)con);
3415 } else {
3416 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3417 if (! __ is_valid_AArch64_address(con) ||
3418 con < (address)(uintptr_t)os::vm_page_size()) {
3419 __ mov(dst_reg, con);
3420 } else {
3421 uint64_t offset;
3422 __ adrp(dst_reg, con, offset);
3423 __ add(dst_reg, dst_reg, offset);
3424 }
3425 }
3426 }
3427 %}
3428
3429 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3430 Register dst_reg = as_Register($dst$$reg);
3431 __ mov(dst_reg, zr);
3432 %}
3433
3434 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, (uint64_t)1);
3437 %}
3438
3439 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 address con = (address)$src$$constant;
3442 if (con == nullptr) {
3443 ShouldNotReachHere();
3444 } else {
3445 relocInfo::relocType rtype = $src->constant_reloc();
3446 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3447 __ set_narrow_oop(dst_reg, (jobject)con);
3448 }
3449 %}
3450
3451 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3452 Register dst_reg = as_Register($dst$$reg);
3453 __ mov(dst_reg, zr);
3454 %}
3455
3456 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3457 Register dst_reg = as_Register($dst$$reg);
3458 address con = (address)$src$$constant;
3459 if (con == nullptr) {
3460 ShouldNotReachHere();
3461 } else {
3462 relocInfo::relocType rtype = $src->constant_reloc();
3463 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3464 __ set_narrow_klass(dst_reg, (Klass *)con);
3465 }
3466 %}
3467
3468 // arithmetic encodings
3469
3470 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3471 Register dst_reg = as_Register($dst$$reg);
3472 Register src_reg = as_Register($src1$$reg);
3473 int32_t con = (int32_t)$src2$$constant;
3474 // add has primary == 0, subtract has primary == 1
3475 if ($primary) { con = -con; }
3476 if (con < 0) {
3477 __ subw(dst_reg, src_reg, -con);
3478 } else {
3479 __ addw(dst_reg, src_reg, con);
3480 }
3481 %}
3482
3483 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src_reg = as_Register($src1$$reg);
3486 int32_t con = (int32_t)$src2$$constant;
3487 // add has primary == 0, subtract has primary == 1
3488 if ($primary) { con = -con; }
3489 if (con < 0) {
3490 __ sub(dst_reg, src_reg, -con);
3491 } else {
3492 __ add(dst_reg, src_reg, con);
3493 }
3494 %}
3495
3496 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3497 Register dst_reg = as_Register($dst$$reg);
3498 Register src1_reg = as_Register($src1$$reg);
3499 Register src2_reg = as_Register($src2$$reg);
3500 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3501 %}
3502
3503 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3504 Register dst_reg = as_Register($dst$$reg);
3505 Register src1_reg = as_Register($src1$$reg);
3506 Register src2_reg = as_Register($src2$$reg);
3507 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3508 %}
3509
3510 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3511 Register dst_reg = as_Register($dst$$reg);
3512 Register src1_reg = as_Register($src1$$reg);
3513 Register src2_reg = as_Register($src2$$reg);
3514 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3515 %}
3516
3517 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3518 Register dst_reg = as_Register($dst$$reg);
3519 Register src1_reg = as_Register($src1$$reg);
3520 Register src2_reg = as_Register($src2$$reg);
3521 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3522 %}
3523
3524 // compare instruction encodings
3525
3526 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3527 Register reg1 = as_Register($src1$$reg);
3528 Register reg2 = as_Register($src2$$reg);
3529 __ cmpw(reg1, reg2);
3530 %}
3531
3532 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3533 Register reg = as_Register($src1$$reg);
3534 int32_t val = $src2$$constant;
3535 if (val >= 0) {
3536 __ subsw(zr, reg, val);
3537 } else {
3538 __ addsw(zr, reg, -val);
3539 }
3540 %}
3541
3542 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3543 Register reg1 = as_Register($src1$$reg);
3544 uint32_t val = (uint32_t)$src2$$constant;
3545 __ movw(rscratch1, val);
3546 __ cmpw(reg1, rscratch1);
3547 %}
3548
3549 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3550 Register reg1 = as_Register($src1$$reg);
3551 Register reg2 = as_Register($src2$$reg);
3552 __ cmp(reg1, reg2);
3553 %}
3554
3555 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3556 Register reg = as_Register($src1$$reg);
3557 int64_t val = $src2$$constant;
3558 if (val >= 0) {
3559 __ subs(zr, reg, val);
3560 } else if (val != -val) {
3561 __ adds(zr, reg, -val);
3562 } else {
3563 // aargh, Long.MIN_VALUE is a special case
3564 __ orr(rscratch1, zr, (uint64_t)val);
3565 __ subs(zr, reg, rscratch1);
3566 }
3567 %}
3568
3569 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3570 Register reg1 = as_Register($src1$$reg);
3571 uint64_t val = (uint64_t)$src2$$constant;
3572 __ mov(rscratch1, val);
3573 __ cmp(reg1, rscratch1);
3574 %}
3575
3576 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3577 Register reg1 = as_Register($src1$$reg);
3578 Register reg2 = as_Register($src2$$reg);
3579 __ cmp(reg1, reg2);
3580 %}
3581
3582 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3583 Register reg1 = as_Register($src1$$reg);
3584 Register reg2 = as_Register($src2$$reg);
3585 __ cmpw(reg1, reg2);
3586 %}
3587
3588 enc_class aarch64_enc_testp(iRegP src) %{
3589 Register reg = as_Register($src$$reg);
3590 __ cmp(reg, zr);
3591 %}
3592
3593 enc_class aarch64_enc_testn(iRegN src) %{
3594 Register reg = as_Register($src$$reg);
3595 __ cmpw(reg, zr);
3596 %}
3597
3598 enc_class aarch64_enc_b(label lbl) %{
3599 Label *L = $lbl$$label;
3600 __ b(*L);
3601 %}
3602
3603 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3604 Label *L = $lbl$$label;
3605 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3606 %}
3607
3608 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3609 Label *L = $lbl$$label;
3610 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3611 %}
3612
3613 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3614 %{
3615 Register sub_reg = as_Register($sub$$reg);
3616 Register super_reg = as_Register($super$$reg);
3617 Register temp_reg = as_Register($temp$$reg);
3618 Register result_reg = as_Register($result$$reg);
3619
3620 Label miss;
3621 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3622 nullptr, &miss,
3623 /*set_cond_codes:*/ true);
3624 if ($primary) {
3625 __ mov(result_reg, zr);
3626 }
3627 __ bind(miss);
3628 %}
3629
3630 enc_class aarch64_enc_java_static_call(method meth) %{
3631 address addr = (address)$meth$$method;
3632 address call;
3633 if (!_method) {
3634 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3635 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3636 if (call == nullptr) {
3637 ciEnv::current()->record_failure("CodeCache is full");
3638 return;
3639 }
3640 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3641 // The NOP here is purely to ensure that eliding a call to
3642 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3643 __ nop();
3644 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3645 } else {
3646 int method_index = resolved_method_index(masm);
3647 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3648 : static_call_Relocation::spec(method_index);
3649 call = __ trampoline_call(Address(addr, rspec));
3650 if (call == nullptr) {
3651 ciEnv::current()->record_failure("CodeCache is full");
3652 return;
3653 }
3654 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3655 // Calls of the same statically bound method can share
3656 // a stub to the interpreter.
3657 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3658 } else {
3659 // Emit stub for static call
3660 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3661 if (stub == nullptr) {
3662 ciEnv::current()->record_failure("CodeCache is full");
3663 return;
3664 }
3665 }
3666 }
3667
3668 __ post_call_nop();
3669
3670 // Only non uncommon_trap calls need to reinitialize ptrue.
3671 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3672 __ reinitialize_ptrue();
3673 }
3674 %}
3675
3676 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3677 int method_index = resolved_method_index(masm);
3678 address call = __ ic_call((address)$meth$$method, method_index);
3679 if (call == nullptr) {
3680 ciEnv::current()->record_failure("CodeCache is full");
3681 return;
3682 }
3683 __ post_call_nop();
3684 if (Compile::current()->max_vector_size() > 0) {
3685 __ reinitialize_ptrue();
3686 }
3687 %}
3688
3689 enc_class aarch64_enc_call_epilog() %{
3690 if (VerifyStackAtCalls) {
3691 // Check that stack depth is unchanged: find majik cookie on stack
3692 __ call_Unimplemented();
3693 }
3694 %}
3695
3696 enc_class aarch64_enc_java_to_runtime(method meth) %{
3697 // some calls to generated routines (arraycopy code) are scheduled
3698 // by C2 as runtime calls. if so we can call them using a br (they
3699 // will be in a reachable segment) otherwise we have to use a blr
3700 // which loads the absolute address into a register.
3701 address entry = (address)$meth$$method;
3702 CodeBlob *cb = CodeCache::find_blob(entry);
3703 if (cb) {
3704 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3705 if (call == nullptr) {
3706 ciEnv::current()->record_failure("CodeCache is full");
3707 return;
3708 }
3709 __ post_call_nop();
3710 } else {
3711 Label retaddr;
3712 // Make the anchor frame walkable
3713 __ adr(rscratch2, retaddr);
3714 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3715 __ lea(rscratch1, RuntimeAddress(entry));
3716 __ blr(rscratch1);
3717 __ bind(retaddr);
3718 __ post_call_nop();
3719 }
3720 if (Compile::current()->max_vector_size() > 0) {
3721 __ reinitialize_ptrue();
3722 }
3723 %}
3724
3725 enc_class aarch64_enc_rethrow() %{
3726 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3727 %}
3728
3729 enc_class aarch64_enc_ret() %{
3730 #ifdef ASSERT
3731 if (Compile::current()->max_vector_size() > 0) {
3732 __ verify_ptrue();
3733 }
3734 #endif
3735 __ ret(lr);
3736 %}
3737
3738 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3739 Register target_reg = as_Register($jump_target$$reg);
3740 __ br(target_reg);
3741 %}
3742
3743 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3744 Register target_reg = as_Register($jump_target$$reg);
3745 // exception oop should be in r0
3746 // ret addr has been popped into lr
3747 // callee expects it in r3
3748 __ mov(r3, lr);
3749 __ br(target_reg);
3750 %}
3751
3752 %}
3753
3754 //----------FRAME--------------------------------------------------------------
3755 // Definition of frame structure and management information.
3756 //
3757 // S T A C K L A Y O U T Allocators stack-slot number
3758 // | (to get allocators register number
3759 // G Owned by | | v add OptoReg::stack0())
3760 // r CALLER | |
3761 // o | +--------+ pad to even-align allocators stack-slot
3762 // w V | pad0 | numbers; owned by CALLER
3763 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3764 // h ^ | in | 5
3765 // | | args | 4 Holes in incoming args owned by SELF
3766 // | | | | 3
3767 // | | +--------+
3768 // V | | old out| Empty on Intel, window on Sparc
3769 // | old |preserve| Must be even aligned.
3770 // | SP-+--------+----> Matcher::_old_SP, even aligned
3771 // | | in | 3 area for Intel ret address
3772 // Owned by |preserve| Empty on Sparc.
3773 // SELF +--------+
3774 // | | pad2 | 2 pad to align old SP
3775 // | +--------+ 1
3776 // | | locks | 0
3777 // | +--------+----> OptoReg::stack0(), even aligned
3778 // | | pad1 | 11 pad to align new SP
3779 // | +--------+
3780 // | | | 10
3781 // | | spills | 9 spills
3782 // V | | 8 (pad0 slot for callee)
3783 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3784 // ^ | out | 7
3785 // | | args | 6 Holes in outgoing args owned by CALLEE
3786 // Owned by +--------+
3787 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3788 // | new |preserve| Must be even-aligned.
3789 // | SP-+--------+----> Matcher::_new_SP, even aligned
3790 // | | |
3791 //
3792 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3793 // known from SELF's arguments and the Java calling convention.
3794 // Region 6-7 is determined per call site.
3795 // Note 2: If the calling convention leaves holes in the incoming argument
3796 // area, those holes are owned by SELF. Holes in the outgoing area
3797 // are owned by the CALLEE. Holes should not be necessary in the
3798 // incoming area, as the Java calling convention is completely under
3799 // the control of the AD file. Doubles can be sorted and packed to
3800 // avoid holes. Holes in the outgoing arguments may be necessary for
3801 // varargs C calling conventions.
3802 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3803 // even aligned with pad0 as needed.
3804 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3805 // (the latter is true on Intel but is it false on AArch64?)
3806 // region 6-11 is even aligned; it may be padded out more so that
3807 // the region from SP to FP meets the minimum stack alignment.
3808 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3809 // alignment. Region 11, pad1, may be dynamically extended so that
3810 // SP meets the minimum alignment.
3811
3812 frame %{
3813 // These three registers define part of the calling convention
3814 // between compiled code and the interpreter.
3815
3816 // Inline Cache Register or Method for I2C.
3817 inline_cache_reg(R12);
3818
3819 // Number of stack slots consumed by locking an object
3820 sync_stack_slots(2);
3821
3822 // Compiled code's Frame Pointer
3823 frame_pointer(R31);
3824
3825 // Interpreter stores its frame pointer in a register which is
3826 // stored to the stack by I2CAdaptors.
3827 // I2CAdaptors convert from interpreted java to compiled java.
3828 interpreter_frame_pointer(R29);
3829
3830 // Stack alignment requirement
3831 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3832
3833 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3834 // for calls to C. Supports the var-args backing area for register parms.
3835 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3836
3837 // The after-PROLOG location of the return address. Location of
3838 // return address specifies a type (REG or STACK) and a number
3839 // representing the register number (i.e. - use a register name) or
3840 // stack slot.
3841 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3842 // Otherwise, it is above the locks and verification slot and alignment word
3843 // TODO this may well be correct but need to check why that - 2 is there
3844 // ppc port uses 0 but we definitely need to allow for fixed_slots
3845 // which folds in the space used for monitors
3846 return_addr(STACK - 2 +
3847 align_up((Compile::current()->in_preserve_stack_slots() +
3848 Compile::current()->fixed_slots()),
3849 stack_alignment_in_slots()));
3850
3851 // Location of compiled Java return values. Same as C for now.
3852 return_value
3853 %{
3854 // TODO do we allow ideal_reg == Op_RegN???
3855 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3856 "only return normal values");
3857
3858 static const int lo[Op_RegL + 1] = { // enum name
3859 0, // Op_Node
3860 0, // Op_Set
3861 R0_num, // Op_RegN
3862 R0_num, // Op_RegI
3863 R0_num, // Op_RegP
3864 V0_num, // Op_RegF
3865 V0_num, // Op_RegD
3866 R0_num // Op_RegL
3867 };
3868
3869 static const int hi[Op_RegL + 1] = { // enum name
3870 0, // Op_Node
3871 0, // Op_Set
3872 OptoReg::Bad, // Op_RegN
3873 OptoReg::Bad, // Op_RegI
3874 R0_H_num, // Op_RegP
3875 OptoReg::Bad, // Op_RegF
3876 V0_H_num, // Op_RegD
3877 R0_H_num // Op_RegL
3878 };
3879
3880 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3881 %}
3882 %}
3883
3884 //----------ATTRIBUTES---------------------------------------------------------
3885 //----------Operand Attributes-------------------------------------------------
3886 op_attrib op_cost(1); // Required cost attribute
3887
3888 //----------Instruction Attributes---------------------------------------------
3889 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3890 ins_attrib ins_size(32); // Required size attribute (in bits)
3891 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3892 // a non-matching short branch variant
3893 // of some long branch?
3894 ins_attrib ins_alignment(4); // Required alignment attribute (must
3895 // be a power of 2) specifies the
3896 // alignment that some part of the
3897 // instruction (not necessarily the
3898 // start) requires. If > 1, a
3899 // compute_padding() function must be
3900 // provided for the instruction
3901
3902 //----------OPERANDS-----------------------------------------------------------
3903 // Operand definitions must precede instruction definitions for correct parsing
3904 // in the ADLC because operands constitute user defined types which are used in
3905 // instruction definitions.
3906
3907 //----------Simple Operands----------------------------------------------------
3908
3909 // Integer operands 32 bit
3910 // 32 bit immediate
3911 operand immI()
3912 %{
3913 match(ConI);
3914
3915 op_cost(0);
3916 format %{ %}
3917 interface(CONST_INTER);
3918 %}
3919
3920 // 32 bit zero
3921 operand immI0()
3922 %{
3923 predicate(n->get_int() == 0);
3924 match(ConI);
3925
3926 op_cost(0);
3927 format %{ %}
3928 interface(CONST_INTER);
3929 %}
3930
3931 // 32 bit unit increment
3932 operand immI_1()
3933 %{
3934 predicate(n->get_int() == 1);
3935 match(ConI);
3936
3937 op_cost(0);
3938 format %{ %}
3939 interface(CONST_INTER);
3940 %}
3941
3942 // 32 bit unit decrement
3943 operand immI_M1()
3944 %{
3945 predicate(n->get_int() == -1);
3946 match(ConI);
3947
3948 op_cost(0);
3949 format %{ %}
3950 interface(CONST_INTER);
3951 %}
3952
3953 // Shift values for add/sub extension shift
3954 operand immIExt()
3955 %{
3956 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3957 match(ConI);
3958
3959 op_cost(0);
3960 format %{ %}
3961 interface(CONST_INTER);
3962 %}
3963
3964 operand immI_gt_1()
3965 %{
3966 predicate(n->get_int() > 1);
3967 match(ConI);
3968
3969 op_cost(0);
3970 format %{ %}
3971 interface(CONST_INTER);
3972 %}
3973
3974 operand immI_le_4()
3975 %{
3976 predicate(n->get_int() <= 4);
3977 match(ConI);
3978
3979 op_cost(0);
3980 format %{ %}
3981 interface(CONST_INTER);
3982 %}
3983
3984 operand immI_16()
3985 %{
3986 predicate(n->get_int() == 16);
3987 match(ConI);
3988
3989 op_cost(0);
3990 format %{ %}
3991 interface(CONST_INTER);
3992 %}
3993
3994 operand immI_24()
3995 %{
3996 predicate(n->get_int() == 24);
3997 match(ConI);
3998
3999 op_cost(0);
4000 format %{ %}
4001 interface(CONST_INTER);
4002 %}
4003
4004 operand immI_32()
4005 %{
4006 predicate(n->get_int() == 32);
4007 match(ConI);
4008
4009 op_cost(0);
4010 format %{ %}
4011 interface(CONST_INTER);
4012 %}
4013
4014 operand immI_48()
4015 %{
4016 predicate(n->get_int() == 48);
4017 match(ConI);
4018
4019 op_cost(0);
4020 format %{ %}
4021 interface(CONST_INTER);
4022 %}
4023
4024 operand immI_56()
4025 %{
4026 predicate(n->get_int() == 56);
4027 match(ConI);
4028
4029 op_cost(0);
4030 format %{ %}
4031 interface(CONST_INTER);
4032 %}
4033
4034 operand immI_255()
4035 %{
4036 predicate(n->get_int() == 255);
4037 match(ConI);
4038
4039 op_cost(0);
4040 format %{ %}
4041 interface(CONST_INTER);
4042 %}
4043
4044 operand immI_65535()
4045 %{
4046 predicate(n->get_int() == 65535);
4047 match(ConI);
4048
4049 op_cost(0);
4050 format %{ %}
4051 interface(CONST_INTER);
4052 %}
4053
4054 operand immI_positive()
4055 %{
4056 predicate(n->get_int() > 0);
4057 match(ConI);
4058
4059 op_cost(0);
4060 format %{ %}
4061 interface(CONST_INTER);
4062 %}
4063
4064 // BoolTest condition for signed compare
4065 operand immI_cmp_cond()
4066 %{
4067 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4068 match(ConI);
4069
4070 op_cost(0);
4071 format %{ %}
4072 interface(CONST_INTER);
4073 %}
4074
4075 // BoolTest condition for unsigned compare
4076 operand immI_cmpU_cond()
4077 %{
4078 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4079 match(ConI);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 operand immL_255()
4087 %{
4088 predicate(n->get_long() == 255L);
4089 match(ConL);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 operand immL_65535()
4097 %{
4098 predicate(n->get_long() == 65535L);
4099 match(ConL);
4100
4101 op_cost(0);
4102 format %{ %}
4103 interface(CONST_INTER);
4104 %}
4105
4106 operand immL_4294967295()
4107 %{
4108 predicate(n->get_long() == 4294967295L);
4109 match(ConL);
4110
4111 op_cost(0);
4112 format %{ %}
4113 interface(CONST_INTER);
4114 %}
4115
4116 operand immL_bitmask()
4117 %{
4118 predicate((n->get_long() != 0)
4119 && ((n->get_long() & 0xc000000000000000l) == 0)
4120 && is_power_of_2(n->get_long() + 1));
4121 match(ConL);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immI_bitmask()
4129 %{
4130 predicate((n->get_int() != 0)
4131 && ((n->get_int() & 0xc0000000) == 0)
4132 && is_power_of_2(n->get_int() + 1));
4133 match(ConI);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 operand immL_positive_bitmaskI()
4141 %{
4142 predicate((n->get_long() != 0)
4143 && ((julong)n->get_long() < 0x80000000ULL)
4144 && is_power_of_2(n->get_long() + 1));
4145 match(ConL);
4146
4147 op_cost(0);
4148 format %{ %}
4149 interface(CONST_INTER);
4150 %}
4151
4152 // Scale values for scaled offset addressing modes (up to long but not quad)
4153 operand immIScale()
4154 %{
4155 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4156 match(ConI);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 // 5 bit signed integer
4164 operand immI5()
4165 %{
4166 predicate(Assembler::is_simm(n->get_int(), 5));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 // 7 bit unsigned integer
4175 operand immIU7()
4176 %{
4177 predicate(Assembler::is_uimm(n->get_int(), 7));
4178 match(ConI);
4179
4180 op_cost(0);
4181 format %{ %}
4182 interface(CONST_INTER);
4183 %}
4184
4185 // Offset for scaled or unscaled immediate loads and stores
4186 operand immIOffset()
4187 %{
4188 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4189 match(ConI);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 operand immIOffset1()
4197 %{
4198 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4199 match(ConI);
4200
4201 op_cost(0);
4202 format %{ %}
4203 interface(CONST_INTER);
4204 %}
4205
4206 operand immIOffset2()
4207 %{
4208 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4209 match(ConI);
4210
4211 op_cost(0);
4212 format %{ %}
4213 interface(CONST_INTER);
4214 %}
4215
4216 operand immIOffset4()
4217 %{
4218 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4219 match(ConI);
4220
4221 op_cost(0);
4222 format %{ %}
4223 interface(CONST_INTER);
4224 %}
4225
4226 operand immIOffset8()
4227 %{
4228 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4229 match(ConI);
4230
4231 op_cost(0);
4232 format %{ %}
4233 interface(CONST_INTER);
4234 %}
4235
4236 operand immIOffset16()
4237 %{
4238 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4239 match(ConI);
4240
4241 op_cost(0);
4242 format %{ %}
4243 interface(CONST_INTER);
4244 %}
4245
4246 operand immLOffset()
4247 %{
4248 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4249 match(ConL);
4250
4251 op_cost(0);
4252 format %{ %}
4253 interface(CONST_INTER);
4254 %}
4255
4256 operand immLoffset1()
4257 %{
4258 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4259 match(ConL);
4260
4261 op_cost(0);
4262 format %{ %}
4263 interface(CONST_INTER);
4264 %}
4265
4266 operand immLoffset2()
4267 %{
4268 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4269 match(ConL);
4270
4271 op_cost(0);
4272 format %{ %}
4273 interface(CONST_INTER);
4274 %}
4275
4276 operand immLoffset4()
4277 %{
4278 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4279 match(ConL);
4280
4281 op_cost(0);
4282 format %{ %}
4283 interface(CONST_INTER);
4284 %}
4285
4286 operand immLoffset8()
4287 %{
4288 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4289 match(ConL);
4290
4291 op_cost(0);
4292 format %{ %}
4293 interface(CONST_INTER);
4294 %}
4295
4296 operand immLoffset16()
4297 %{
4298 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4299 match(ConL);
4300
4301 op_cost(0);
4302 format %{ %}
4303 interface(CONST_INTER);
4304 %}
4305
4306 // 5 bit signed long integer
4307 operand immL5()
4308 %{
4309 predicate(Assembler::is_simm(n->get_long(), 5));
4310 match(ConL);
4311
4312 op_cost(0);
4313 format %{ %}
4314 interface(CONST_INTER);
4315 %}
4316
4317 // 7 bit unsigned long integer
4318 operand immLU7()
4319 %{
4320 predicate(Assembler::is_uimm(n->get_long(), 7));
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 // 8 bit signed value.
4329 operand immI8()
4330 %{
4331 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4332 match(ConI);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 // 8 bit signed value (simm8), or #simm8 LSL 8.
4340 operand immI8_shift8()
4341 %{
4342 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4343 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // 8 bit signed value (simm8), or #simm8 LSL 8.
4352 operand immL8_shift8()
4353 %{
4354 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4355 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4356 match(ConL);
4357
4358 op_cost(0);
4359 format %{ %}
4360 interface(CONST_INTER);
4361 %}
4362
4363 // 8 bit integer valid for vector add sub immediate
4364 operand immBAddSubV()
4365 %{
4366 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4367 match(ConI);
4368
4369 op_cost(0);
4370 format %{ %}
4371 interface(CONST_INTER);
4372 %}
4373
4374 // 32 bit integer valid for add sub immediate
4375 operand immIAddSub()
4376 %{
4377 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4378 match(ConI);
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 // 32 bit integer valid for vector add sub immediate
4385 operand immIAddSubV()
4386 %{
4387 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4388 match(ConI);
4389
4390 op_cost(0);
4391 format %{ %}
4392 interface(CONST_INTER);
4393 %}
4394
4395 // 32 bit unsigned integer valid for logical immediate
4396
4397 operand immBLog()
4398 %{
4399 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4400 match(ConI);
4401
4402 op_cost(0);
4403 format %{ %}
4404 interface(CONST_INTER);
4405 %}
4406
4407 operand immSLog()
4408 %{
4409 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4410 match(ConI);
4411
4412 op_cost(0);
4413 format %{ %}
4414 interface(CONST_INTER);
4415 %}
4416
4417 operand immILog()
4418 %{
4419 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4420 match(ConI);
4421
4422 op_cost(0);
4423 format %{ %}
4424 interface(CONST_INTER);
4425 %}
4426
4427 // Integer operands 64 bit
4428 // 64 bit immediate
4429 operand immL()
4430 %{
4431 match(ConL);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // 64 bit zero
4439 operand immL0()
4440 %{
4441 predicate(n->get_long() == 0);
4442 match(ConL);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // 64 bit unit decrement
4450 operand immL_M1()
4451 %{
4452 predicate(n->get_long() == -1);
4453 match(ConL);
4454
4455 op_cost(0);
4456 format %{ %}
4457 interface(CONST_INTER);
4458 %}
4459
4460 // 64 bit integer valid for add sub immediate
4461 operand immLAddSub()
4462 %{
4463 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4464 match(ConL);
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 // 64 bit integer valid for addv subv immediate
4471 operand immLAddSubV()
4472 %{
4473 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4474 match(ConL);
4475
4476 op_cost(0);
4477 format %{ %}
4478 interface(CONST_INTER);
4479 %}
4480
4481 // 64 bit integer valid for logical immediate
4482 operand immLLog()
4483 %{
4484 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4485 match(ConL);
4486 op_cost(0);
4487 format %{ %}
4488 interface(CONST_INTER);
4489 %}
4490
4491 // Long Immediate: low 32-bit mask
4492 operand immL_32bits()
4493 %{
4494 predicate(n->get_long() == 0xFFFFFFFFL);
4495 match(ConL);
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // Pointer operands
4502 // Pointer Immediate
4503 operand immP()
4504 %{
4505 match(ConP);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // nullptr Pointer Immediate
4513 operand immP0()
4514 %{
4515 predicate(n->get_ptr() == 0);
4516 match(ConP);
4517
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // Pointer Immediate One
4524 // this is used in object initialization (initial object header)
4525 operand immP_1()
4526 %{
4527 predicate(n->get_ptr() == 1);
4528 match(ConP);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 // Card Table Byte Map Base
4536 operand immByteMapBase()
4537 %{
4538 // Get base of card map
4539 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4540 is_card_table_address((address)(n->get_ptr())));
4541 match(ConP);
4542
4543 op_cost(0);
4544 format %{ %}
4545 interface(CONST_INTER);
4546 %}
4547
4548 // AOT Runtime Constants Address
4549 operand immAOTRuntimeConstantsAddress()
4550 %{
4551 // Check if the address is in the range of AOT Runtime Constants
4552 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4553 match(ConP);
4554
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // Float and Double operands
4561 // Double Immediate
4562 operand immD()
4563 %{
4564 match(ConD);
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Double Immediate: +0.0d
4571 operand immD0()
4572 %{
4573 predicate(jlong_cast(n->getd()) == 0);
4574 match(ConD);
4575
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // constant 'double +0.0'.
4582 operand immDPacked()
4583 %{
4584 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4585 match(ConD);
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // Float Immediate
4592 operand immF()
4593 %{
4594 match(ConF);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Float Immediate: +0.0f.
4601 operand immF0()
4602 %{
4603 predicate(jint_cast(n->getf()) == 0);
4604 match(ConF);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 //
4612 operand immFPacked()
4613 %{
4614 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4615 match(ConF);
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Narrow pointer operands
4622 // Narrow Pointer Immediate
4623 operand immN()
4624 %{
4625 match(ConN);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // Narrow nullptr Pointer Immediate
4633 operand immN0()
4634 %{
4635 predicate(n->get_narrowcon() == 0);
4636 match(ConN);
4637
4638 op_cost(0);
4639 format %{ %}
4640 interface(CONST_INTER);
4641 %}
4642
4643 operand immNKlass()
4644 %{
4645 match(ConNKlass);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 // Integer 32 bit Register Operands
4653 // Integer 32 bitRegister (excludes SP)
4654 operand iRegI()
4655 %{
4656 constraint(ALLOC_IN_RC(any_reg32));
4657 match(RegI);
4658 match(iRegINoSp);
4659 op_cost(0);
4660 format %{ %}
4661 interface(REG_INTER);
4662 %}
4663
4664 // Integer 32 bit Register not Special
4665 operand iRegINoSp()
4666 %{
4667 constraint(ALLOC_IN_RC(no_special_reg32));
4668 match(RegI);
4669 op_cost(0);
4670 format %{ %}
4671 interface(REG_INTER);
4672 %}
4673
4674 // Integer 64 bit Register Operands
4675 // Integer 64 bit Register (includes SP)
4676 operand iRegL()
4677 %{
4678 constraint(ALLOC_IN_RC(any_reg));
4679 match(RegL);
4680 match(iRegLNoSp);
4681 op_cost(0);
4682 format %{ %}
4683 interface(REG_INTER);
4684 %}
4685
4686 // Integer 64 bit Register not Special
4687 operand iRegLNoSp()
4688 %{
4689 constraint(ALLOC_IN_RC(no_special_reg));
4690 match(RegL);
4691 match(iRegL_R0);
4692 format %{ %}
4693 interface(REG_INTER);
4694 %}
4695
4696 // Pointer Register Operands
4697 // Pointer Register
4698 operand iRegP()
4699 %{
4700 constraint(ALLOC_IN_RC(ptr_reg));
4701 match(RegP);
4702 match(iRegPNoSp);
4703 match(iRegP_R0);
4704 //match(iRegP_R2);
4705 //match(iRegP_R4);
4706 match(iRegP_R5);
4707 match(thread_RegP);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Pointer 64 bit Register not Special
4714 operand iRegPNoSp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4717 match(RegP);
4718 // match(iRegP);
4719 // match(iRegP_R0);
4720 // match(iRegP_R2);
4721 // match(iRegP_R4);
4722 // match(iRegP_R5);
4723 // match(thread_RegP);
4724 op_cost(0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // This operand is not allowed to use rfp even if
4730 // rfp is not used to hold the frame pointer.
4731 operand iRegPNoSpNoRfp()
4732 %{
4733 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4734 match(RegP);
4735 match(iRegPNoSp);
4736 op_cost(0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer 64 bit Register R0 only
4742 operand iRegP_R0()
4743 %{
4744 constraint(ALLOC_IN_RC(r0_reg));
4745 match(RegP);
4746 // match(iRegP);
4747 match(iRegPNoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Pointer 64 bit Register R1 only
4754 operand iRegP_R1()
4755 %{
4756 constraint(ALLOC_IN_RC(r1_reg));
4757 match(RegP);
4758 // match(iRegP);
4759 match(iRegPNoSp);
4760 op_cost(0);
4761 format %{ %}
4762 interface(REG_INTER);
4763 %}
4764
4765 // Pointer 64 bit Register R2 only
4766 operand iRegP_R2()
4767 %{
4768 constraint(ALLOC_IN_RC(r2_reg));
4769 match(RegP);
4770 // match(iRegP);
4771 match(iRegPNoSp);
4772 op_cost(0);
4773 format %{ %}
4774 interface(REG_INTER);
4775 %}
4776
4777 // Pointer 64 bit Register R3 only
4778 operand iRegP_R3()
4779 %{
4780 constraint(ALLOC_IN_RC(r3_reg));
4781 match(RegP);
4782 // match(iRegP);
4783 match(iRegPNoSp);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 // Pointer 64 bit Register R4 only
4790 operand iRegP_R4()
4791 %{
4792 constraint(ALLOC_IN_RC(r4_reg));
4793 match(RegP);
4794 // match(iRegP);
4795 match(iRegPNoSp);
4796 op_cost(0);
4797 format %{ %}
4798 interface(REG_INTER);
4799 %}
4800
4801 // Pointer 64 bit Register R5 only
4802 operand iRegP_R5()
4803 %{
4804 constraint(ALLOC_IN_RC(r5_reg));
4805 match(RegP);
4806 // match(iRegP);
4807 match(iRegPNoSp);
4808 op_cost(0);
4809 format %{ %}
4810 interface(REG_INTER);
4811 %}
4812
4813 // Pointer 64 bit Register R10 only
4814 operand iRegP_R10()
4815 %{
4816 constraint(ALLOC_IN_RC(r10_reg));
4817 match(RegP);
4818 // match(iRegP);
4819 match(iRegPNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Long 64 bit Register R0 only
4826 operand iRegL_R0()
4827 %{
4828 constraint(ALLOC_IN_RC(r0_reg));
4829 match(RegL);
4830 match(iRegLNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Long 64 bit Register R11 only
4837 operand iRegL_R11()
4838 %{
4839 constraint(ALLOC_IN_RC(r11_reg));
4840 match(RegL);
4841 match(iRegLNoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Register R0 only
4848 operand iRegI_R0()
4849 %{
4850 constraint(ALLOC_IN_RC(int_r0_reg));
4851 match(RegI);
4852 match(iRegINoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Register R2 only
4859 operand iRegI_R2()
4860 %{
4861 constraint(ALLOC_IN_RC(int_r2_reg));
4862 match(RegI);
4863 match(iRegINoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869 // Register R3 only
4870 operand iRegI_R3()
4871 %{
4872 constraint(ALLOC_IN_RC(int_r3_reg));
4873 match(RegI);
4874 match(iRegINoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880
4881 // Register R4 only
4882 operand iRegI_R4()
4883 %{
4884 constraint(ALLOC_IN_RC(int_r4_reg));
4885 match(RegI);
4886 match(iRegINoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892
4893 // Pointer Register Operands
4894 // Narrow Pointer Register
4895 operand iRegN()
4896 %{
4897 constraint(ALLOC_IN_RC(any_reg32));
4898 match(RegN);
4899 match(iRegNNoSp);
4900 op_cost(0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Integer 64 bit Register not Special
4906 operand iRegNNoSp()
4907 %{
4908 constraint(ALLOC_IN_RC(no_special_reg32));
4909 match(RegN);
4910 op_cost(0);
4911 format %{ %}
4912 interface(REG_INTER);
4913 %}
4914
4915 // Float Register
4916 // Float register operands
4917 operand vRegF()
4918 %{
4919 constraint(ALLOC_IN_RC(float_reg));
4920 match(RegF);
4921
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Double Register
4928 // Double register operands
4929 operand vRegD()
4930 %{
4931 constraint(ALLOC_IN_RC(double_reg));
4932 match(RegD);
4933
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 // Generic vector class. This will be used for
4940 // all vector operands, including NEON and SVE.
4941 operand vReg()
4942 %{
4943 constraint(ALLOC_IN_RC(dynamic));
4944 match(VecA);
4945 match(VecD);
4946 match(VecX);
4947
4948 op_cost(0);
4949 format %{ %}
4950 interface(REG_INTER);
4951 %}
4952
4953 operand vecA()
4954 %{
4955 constraint(ALLOC_IN_RC(vectora_reg));
4956 match(VecA);
4957
4958 op_cost(0);
4959 format %{ %}
4960 interface(REG_INTER);
4961 %}
4962
4963 operand vecD()
4964 %{
4965 constraint(ALLOC_IN_RC(vectord_reg));
4966 match(VecD);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vecX()
4974 %{
4975 constraint(ALLOC_IN_RC(vectorx_reg));
4976 match(VecX);
4977
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 operand vRegD_V0()
4984 %{
4985 constraint(ALLOC_IN_RC(v0_reg));
4986 match(RegD);
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vRegD_V1()
4993 %{
4994 constraint(ALLOC_IN_RC(v1_reg));
4995 match(RegD);
4996 op_cost(0);
4997 format %{ %}
4998 interface(REG_INTER);
4999 %}
5000
5001 operand vRegD_V2()
5002 %{
5003 constraint(ALLOC_IN_RC(v2_reg));
5004 match(RegD);
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 operand vRegD_V3()
5011 %{
5012 constraint(ALLOC_IN_RC(v3_reg));
5013 match(RegD);
5014 op_cost(0);
5015 format %{ %}
5016 interface(REG_INTER);
5017 %}
5018
5019 operand vRegD_V4()
5020 %{
5021 constraint(ALLOC_IN_RC(v4_reg));
5022 match(RegD);
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vRegD_V5()
5029 %{
5030 constraint(ALLOC_IN_RC(v5_reg));
5031 match(RegD);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand vRegD_V6()
5038 %{
5039 constraint(ALLOC_IN_RC(v6_reg));
5040 match(RegD);
5041 op_cost(0);
5042 format %{ %}
5043 interface(REG_INTER);
5044 %}
5045
5046 operand vRegD_V7()
5047 %{
5048 constraint(ALLOC_IN_RC(v7_reg));
5049 match(RegD);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 operand vRegD_V12()
5056 %{
5057 constraint(ALLOC_IN_RC(v12_reg));
5058 match(RegD);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 operand vRegD_V13()
5065 %{
5066 constraint(ALLOC_IN_RC(v13_reg));
5067 match(RegD);
5068 op_cost(0);
5069 format %{ %}
5070 interface(REG_INTER);
5071 %}
5072
5073 operand pReg()
5074 %{
5075 constraint(ALLOC_IN_RC(pr_reg));
5076 match(RegVectMask);
5077 match(pRegGov);
5078 op_cost(0);
5079 format %{ %}
5080 interface(REG_INTER);
5081 %}
5082
5083 operand pRegGov()
5084 %{
5085 constraint(ALLOC_IN_RC(gov_pr));
5086 match(RegVectMask);
5087 match(pReg);
5088 op_cost(0);
5089 format %{ %}
5090 interface(REG_INTER);
5091 %}
5092
5093 operand pRegGov_P0()
5094 %{
5095 constraint(ALLOC_IN_RC(p0_reg));
5096 match(RegVectMask);
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 operand pRegGov_P1()
5103 %{
5104 constraint(ALLOC_IN_RC(p1_reg));
5105 match(RegVectMask);
5106 op_cost(0);
5107 format %{ %}
5108 interface(REG_INTER);
5109 %}
5110
5111 // Flags register, used as output of signed compare instructions
5112
5113 // note that on AArch64 we also use this register as the output for
5114 // for floating point compare instructions (CmpF CmpD). this ensures
5115 // that ordered inequality tests use GT, GE, LT or LE none of which
5116 // pass through cases where the result is unordered i.e. one or both
5117 // inputs to the compare is a NaN. this means that the ideal code can
5118 // replace e.g. a GT with an LE and not end up capturing the NaN case
5119 // (where the comparison should always fail). EQ and NE tests are
5120 // always generated in ideal code so that unordered folds into the NE
5121 // case, matching the behaviour of AArch64 NE.
5122 //
5123 // This differs from x86 where the outputs of FP compares use a
5124 // special FP flags registers and where compares based on this
5125 // register are distinguished into ordered inequalities (cmpOpUCF) and
5126 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5127 // to explicitly handle the unordered case in branches. x86 also has
5128 // to include extra CMoveX rules to accept a cmpOpUCF input.
5129
5130 operand rFlagsReg()
5131 %{
5132 constraint(ALLOC_IN_RC(int_flags));
5133 match(RegFlags);
5134
5135 op_cost(0);
5136 format %{ "RFLAGS" %}
5137 interface(REG_INTER);
5138 %}
5139
5140 // Flags register, used as output of unsigned compare instructions
5141 operand rFlagsRegU()
5142 %{
5143 constraint(ALLOC_IN_RC(int_flags));
5144 match(RegFlags);
5145
5146 op_cost(0);
5147 format %{ "RFLAGSU" %}
5148 interface(REG_INTER);
5149 %}
5150
5151 // Special Registers
5152
5153 // Method Register
5154 operand inline_cache_RegP(iRegP reg)
5155 %{
5156 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5157 match(reg);
5158 match(iRegPNoSp);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 // Thread Register
5165 operand thread_RegP(iRegP reg)
5166 %{
5167 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5168 match(reg);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 //----------Memory Operands----------------------------------------------------
5175
5176 operand indirect(iRegP reg)
5177 %{
5178 constraint(ALLOC_IN_RC(ptr_reg));
5179 match(reg);
5180 op_cost(0);
5181 format %{ "[$reg]" %}
5182 interface(MEMORY_INTER) %{
5183 base($reg);
5184 index(0xffffffff);
5185 scale(0x0);
5186 disp(0x0);
5187 %}
5188 %}
5189
5190 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5191 %{
5192 constraint(ALLOC_IN_RC(ptr_reg));
5193 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5194 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5195 op_cost(0);
5196 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5197 interface(MEMORY_INTER) %{
5198 base($reg);
5199 index($ireg);
5200 scale($scale);
5201 disp(0x0);
5202 %}
5203 %}
5204
5205 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5206 %{
5207 constraint(ALLOC_IN_RC(ptr_reg));
5208 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5209 match(AddP reg (LShiftL lreg scale));
5210 op_cost(0);
5211 format %{ "$reg, $lreg lsl($scale)" %}
5212 interface(MEMORY_INTER) %{
5213 base($reg);
5214 index($lreg);
5215 scale($scale);
5216 disp(0x0);
5217 %}
5218 %}
5219
5220 operand indIndexI2L(iRegP reg, iRegI ireg)
5221 %{
5222 constraint(ALLOC_IN_RC(ptr_reg));
5223 match(AddP reg (ConvI2L ireg));
5224 op_cost(0);
5225 format %{ "$reg, $ireg, 0, I2L" %}
5226 interface(MEMORY_INTER) %{
5227 base($reg);
5228 index($ireg);
5229 scale(0x0);
5230 disp(0x0);
5231 %}
5232 %}
5233
5234 operand indIndex(iRegP reg, iRegL lreg)
5235 %{
5236 constraint(ALLOC_IN_RC(ptr_reg));
5237 match(AddP reg lreg);
5238 op_cost(0);
5239 format %{ "$reg, $lreg" %}
5240 interface(MEMORY_INTER) %{
5241 base($reg);
5242 index($lreg);
5243 scale(0x0);
5244 disp(0x0);
5245 %}
5246 %}
5247
5248 operand indOffI1(iRegP reg, immIOffset1 off)
5249 %{
5250 constraint(ALLOC_IN_RC(ptr_reg));
5251 match(AddP reg off);
5252 op_cost(0);
5253 format %{ "[$reg, $off]" %}
5254 interface(MEMORY_INTER) %{
5255 base($reg);
5256 index(0xffffffff);
5257 scale(0x0);
5258 disp($off);
5259 %}
5260 %}
5261
5262 operand indOffI2(iRegP reg, immIOffset2 off)
5263 %{
5264 constraint(ALLOC_IN_RC(ptr_reg));
5265 match(AddP reg off);
5266 op_cost(0);
5267 format %{ "[$reg, $off]" %}
5268 interface(MEMORY_INTER) %{
5269 base($reg);
5270 index(0xffffffff);
5271 scale(0x0);
5272 disp($off);
5273 %}
5274 %}
5275
5276 operand indOffI4(iRegP reg, immIOffset4 off)
5277 %{
5278 constraint(ALLOC_IN_RC(ptr_reg));
5279 match(AddP reg off);
5280 op_cost(0);
5281 format %{ "[$reg, $off]" %}
5282 interface(MEMORY_INTER) %{
5283 base($reg);
5284 index(0xffffffff);
5285 scale(0x0);
5286 disp($off);
5287 %}
5288 %}
5289
5290 operand indOffI8(iRegP reg, immIOffset8 off)
5291 %{
5292 constraint(ALLOC_IN_RC(ptr_reg));
5293 match(AddP reg off);
5294 op_cost(0);
5295 format %{ "[$reg, $off]" %}
5296 interface(MEMORY_INTER) %{
5297 base($reg);
5298 index(0xffffffff);
5299 scale(0x0);
5300 disp($off);
5301 %}
5302 %}
5303
5304 operand indOffI16(iRegP reg, immIOffset16 off)
5305 %{
5306 constraint(ALLOC_IN_RC(ptr_reg));
5307 match(AddP reg off);
5308 op_cost(0);
5309 format %{ "[$reg, $off]" %}
5310 interface(MEMORY_INTER) %{
5311 base($reg);
5312 index(0xffffffff);
5313 scale(0x0);
5314 disp($off);
5315 %}
5316 %}
5317
5318 operand indOffL1(iRegP reg, immLoffset1 off)
5319 %{
5320 constraint(ALLOC_IN_RC(ptr_reg));
5321 match(AddP reg off);
5322 op_cost(0);
5323 format %{ "[$reg, $off]" %}
5324 interface(MEMORY_INTER) %{
5325 base($reg);
5326 index(0xffffffff);
5327 scale(0x0);
5328 disp($off);
5329 %}
5330 %}
5331
5332 operand indOffL2(iRegP reg, immLoffset2 off)
5333 %{
5334 constraint(ALLOC_IN_RC(ptr_reg));
5335 match(AddP reg off);
5336 op_cost(0);
5337 format %{ "[$reg, $off]" %}
5338 interface(MEMORY_INTER) %{
5339 base($reg);
5340 index(0xffffffff);
5341 scale(0x0);
5342 disp($off);
5343 %}
5344 %}
5345
5346 operand indOffL4(iRegP reg, immLoffset4 off)
5347 %{
5348 constraint(ALLOC_IN_RC(ptr_reg));
5349 match(AddP reg off);
5350 op_cost(0);
5351 format %{ "[$reg, $off]" %}
5352 interface(MEMORY_INTER) %{
5353 base($reg);
5354 index(0xffffffff);
5355 scale(0x0);
5356 disp($off);
5357 %}
5358 %}
5359
5360 operand indOffL8(iRegP reg, immLoffset8 off)
5361 %{
5362 constraint(ALLOC_IN_RC(ptr_reg));
5363 match(AddP reg off);
5364 op_cost(0);
5365 format %{ "[$reg, $off]" %}
5366 interface(MEMORY_INTER) %{
5367 base($reg);
5368 index(0xffffffff);
5369 scale(0x0);
5370 disp($off);
5371 %}
5372 %}
5373
5374 operand indOffL16(iRegP reg, immLoffset16 off)
5375 %{
5376 constraint(ALLOC_IN_RC(ptr_reg));
5377 match(AddP reg off);
5378 op_cost(0);
5379 format %{ "[$reg, $off]" %}
5380 interface(MEMORY_INTER) %{
5381 base($reg);
5382 index(0xffffffff);
5383 scale(0x0);
5384 disp($off);
5385 %}
5386 %}
5387
5388 operand indirectX2P(iRegL reg)
5389 %{
5390 constraint(ALLOC_IN_RC(ptr_reg));
5391 match(CastX2P reg);
5392 op_cost(0);
5393 format %{ "[$reg]\t# long -> ptr" %}
5394 interface(MEMORY_INTER) %{
5395 base($reg);
5396 index(0xffffffff);
5397 scale(0x0);
5398 disp(0x0);
5399 %}
5400 %}
5401
5402 operand indOffX2P(iRegL reg, immLOffset off)
5403 %{
5404 constraint(ALLOC_IN_RC(ptr_reg));
5405 match(AddP (CastX2P reg) off);
5406 op_cost(0);
5407 format %{ "[$reg, $off]\t# long -> ptr" %}
5408 interface(MEMORY_INTER) %{
5409 base($reg);
5410 index(0xffffffff);
5411 scale(0x0);
5412 disp($off);
5413 %}
5414 %}
5415
5416 operand indirectN(iRegN reg)
5417 %{
5418 predicate(CompressedOops::shift() == 0);
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(DecodeN reg);
5421 op_cost(0);
5422 format %{ "[$reg]\t# narrow" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp(0x0);
5428 %}
5429 %}
5430
5431 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5432 %{
5433 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5434 constraint(ALLOC_IN_RC(ptr_reg));
5435 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5436 op_cost(0);
5437 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5438 interface(MEMORY_INTER) %{
5439 base($reg);
5440 index($ireg);
5441 scale($scale);
5442 disp(0x0);
5443 %}
5444 %}
5445
5446 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5447 %{
5448 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(AddP (DecodeN reg) (LShiftL lreg scale));
5451 op_cost(0);
5452 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index($lreg);
5456 scale($scale);
5457 disp(0x0);
5458 %}
5459 %}
5460
5461 operand indIndexI2LN(iRegN reg, iRegI ireg)
5462 %{
5463 predicate(CompressedOops::shift() == 0);
5464 constraint(ALLOC_IN_RC(ptr_reg));
5465 match(AddP (DecodeN reg) (ConvI2L ireg));
5466 op_cost(0);
5467 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5468 interface(MEMORY_INTER) %{
5469 base($reg);
5470 index($ireg);
5471 scale(0x0);
5472 disp(0x0);
5473 %}
5474 %}
5475
5476 operand indIndexN(iRegN reg, iRegL lreg)
5477 %{
5478 predicate(CompressedOops::shift() == 0);
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(AddP (DecodeN reg) lreg);
5481 op_cost(0);
5482 format %{ "$reg, $lreg\t# narrow" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index($lreg);
5486 scale(0x0);
5487 disp(0x0);
5488 %}
5489 %}
5490
5491 operand indOffIN(iRegN reg, immIOffset off)
5492 %{
5493 predicate(CompressedOops::shift() == 0);
5494 constraint(ALLOC_IN_RC(ptr_reg));
5495 match(AddP (DecodeN reg) off);
5496 op_cost(0);
5497 format %{ "[$reg, $off]\t# narrow" %}
5498 interface(MEMORY_INTER) %{
5499 base($reg);
5500 index(0xffffffff);
5501 scale(0x0);
5502 disp($off);
5503 %}
5504 %}
5505
5506 operand indOffLN(iRegN reg, immLOffset off)
5507 %{
5508 predicate(CompressedOops::shift() == 0);
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(AddP (DecodeN reg) off);
5511 op_cost(0);
5512 format %{ "[$reg, $off]\t# narrow" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index(0xffffffff);
5516 scale(0x0);
5517 disp($off);
5518 %}
5519 %}
5520
5521
5522 //----------Special Memory Operands--------------------------------------------
5523 // Stack Slot Operand - This operand is used for loading and storing temporary
5524 // values on the stack where a match requires a value to
5525 // flow through memory.
5526 operand stackSlotP(sRegP reg)
5527 %{
5528 constraint(ALLOC_IN_RC(stack_slots));
5529 op_cost(100);
5530 // No match rule because this operand is only generated in matching
5531 // match(RegP);
5532 format %{ "[$reg]" %}
5533 interface(MEMORY_INTER) %{
5534 base(0x1e); // RSP
5535 index(0x0); // No Index
5536 scale(0x0); // No Scale
5537 disp($reg); // Stack Offset
5538 %}
5539 %}
5540
5541 operand stackSlotI(sRegI reg)
5542 %{
5543 constraint(ALLOC_IN_RC(stack_slots));
5544 // No match rule because this operand is only generated in matching
5545 // match(RegI);
5546 format %{ "[$reg]" %}
5547 interface(MEMORY_INTER) %{
5548 base(0x1e); // RSP
5549 index(0x0); // No Index
5550 scale(0x0); // No Scale
5551 disp($reg); // Stack Offset
5552 %}
5553 %}
5554
5555 operand stackSlotF(sRegF reg)
5556 %{
5557 constraint(ALLOC_IN_RC(stack_slots));
5558 // No match rule because this operand is only generated in matching
5559 // match(RegF);
5560 format %{ "[$reg]" %}
5561 interface(MEMORY_INTER) %{
5562 base(0x1e); // RSP
5563 index(0x0); // No Index
5564 scale(0x0); // No Scale
5565 disp($reg); // Stack Offset
5566 %}
5567 %}
5568
5569 operand stackSlotD(sRegD reg)
5570 %{
5571 constraint(ALLOC_IN_RC(stack_slots));
5572 // No match rule because this operand is only generated in matching
5573 // match(RegD);
5574 format %{ "[$reg]" %}
5575 interface(MEMORY_INTER) %{
5576 base(0x1e); // RSP
5577 index(0x0); // No Index
5578 scale(0x0); // No Scale
5579 disp($reg); // Stack Offset
5580 %}
5581 %}
5582
5583 operand stackSlotL(sRegL reg)
5584 %{
5585 constraint(ALLOC_IN_RC(stack_slots));
5586 // No match rule because this operand is only generated in matching
5587 // match(RegL);
5588 format %{ "[$reg]" %}
5589 interface(MEMORY_INTER) %{
5590 base(0x1e); // RSP
5591 index(0x0); // No Index
5592 scale(0x0); // No Scale
5593 disp($reg); // Stack Offset
5594 %}
5595 %}
5596
5597 // Operands for expressing Control Flow
5598 // NOTE: Label is a predefined operand which should not be redefined in
5599 // the AD file. It is generically handled within the ADLC.
5600
5601 //----------Conditional Branch Operands----------------------------------------
5602 // Comparison Op - This is the operation of the comparison, and is limited to
5603 // the following set of codes:
5604 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5605 //
5606 // Other attributes of the comparison, such as unsignedness, are specified
5607 // by the comparison instruction that sets a condition code flags register.
5608 // That result is represented by a flags operand whose subtype is appropriate
5609 // to the unsignedness (etc.) of the comparison.
5610 //
5611 // Later, the instruction which matches both the Comparison Op (a Bool) and
5612 // the flags (produced by the Cmp) specifies the coding of the comparison op
5613 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5614
5615 // used for signed integral comparisons and fp comparisons
5616
5617 operand cmpOp()
5618 %{
5619 match(Bool);
5620
5621 format %{ "" %}
5622 interface(COND_INTER) %{
5623 equal(0x0, "eq");
5624 not_equal(0x1, "ne");
5625 less(0xb, "lt");
5626 greater_equal(0xa, "ge");
5627 less_equal(0xd, "le");
5628 greater(0xc, "gt");
5629 overflow(0x6, "vs");
5630 no_overflow(0x7, "vc");
5631 %}
5632 %}
5633
5634 // used for unsigned integral comparisons
5635
5636 operand cmpOpU()
5637 %{
5638 match(Bool);
5639
5640 format %{ "" %}
5641 interface(COND_INTER) %{
5642 equal(0x0, "eq");
5643 not_equal(0x1, "ne");
5644 less(0x3, "lo");
5645 greater_equal(0x2, "hs");
5646 less_equal(0x9, "ls");
5647 greater(0x8, "hi");
5648 overflow(0x6, "vs");
5649 no_overflow(0x7, "vc");
5650 %}
5651 %}
5652
5653 // used for certain integral comparisons which can be
5654 // converted to cbxx or tbxx instructions
5655
5656 operand cmpOpEqNe()
5657 %{
5658 match(Bool);
5659 op_cost(0);
5660 predicate(n->as_Bool()->_test._test == BoolTest::ne
5661 || n->as_Bool()->_test._test == BoolTest::eq);
5662
5663 format %{ "" %}
5664 interface(COND_INTER) %{
5665 equal(0x0, "eq");
5666 not_equal(0x1, "ne");
5667 less(0xb, "lt");
5668 greater_equal(0xa, "ge");
5669 less_equal(0xd, "le");
5670 greater(0xc, "gt");
5671 overflow(0x6, "vs");
5672 no_overflow(0x7, "vc");
5673 %}
5674 %}
5675
5676 // used for certain integral comparisons which can be
5677 // converted to cbxx or tbxx instructions
5678
5679 operand cmpOpLtGe()
5680 %{
5681 match(Bool);
5682 op_cost(0);
5683
5684 predicate(n->as_Bool()->_test._test == BoolTest::lt
5685 || n->as_Bool()->_test._test == BoolTest::ge);
5686
5687 format %{ "" %}
5688 interface(COND_INTER) %{
5689 equal(0x0, "eq");
5690 not_equal(0x1, "ne");
5691 less(0xb, "lt");
5692 greater_equal(0xa, "ge");
5693 less_equal(0xd, "le");
5694 greater(0xc, "gt");
5695 overflow(0x6, "vs");
5696 no_overflow(0x7, "vc");
5697 %}
5698 %}
5699
5700 // used for certain unsigned integral comparisons which can be
5701 // converted to cbxx or tbxx instructions
5702
5703 operand cmpOpUEqNeLeGt()
5704 %{
5705 match(Bool);
5706 op_cost(0);
5707
5708 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5709 n->as_Bool()->_test._test == BoolTest::ne ||
5710 n->as_Bool()->_test._test == BoolTest::le ||
5711 n->as_Bool()->_test._test == BoolTest::gt);
5712
5713 format %{ "" %}
5714 interface(COND_INTER) %{
5715 equal(0x0, "eq");
5716 not_equal(0x1, "ne");
5717 less(0x3, "lo");
5718 greater_equal(0x2, "hs");
5719 less_equal(0x9, "ls");
5720 greater(0x8, "hi");
5721 overflow(0x6, "vs");
5722 no_overflow(0x7, "vc");
5723 %}
5724 %}
5725
5726 // Special operand allowing long args to int ops to be truncated for free
5727
5728 operand iRegL2I(iRegL reg) %{
5729
5730 op_cost(0);
5731
5732 match(ConvL2I reg);
5733
5734 format %{ "l2i($reg)" %}
5735
5736 interface(REG_INTER)
5737 %}
5738
5739 operand iRegL2P(iRegL reg) %{
5740
5741 op_cost(0);
5742
5743 match(CastX2P reg);
5744
5745 format %{ "l2p($reg)" %}
5746
5747 interface(REG_INTER)
5748 %}
5749
5750 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5751 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5752 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5753 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5754
5755 //----------OPERAND CLASSES----------------------------------------------------
5756 // Operand Classes are groups of operands that are used as to simplify
5757 // instruction definitions by not requiring the AD writer to specify
5758 // separate instructions for every form of operand when the
5759 // instruction accepts multiple operand types with the same basic
5760 // encoding and format. The classic case of this is memory operands.
5761
5762 // memory is used to define read/write location for load/store
5763 // instruction defs. we can turn a memory op into an Address
5764
5765 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5766 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5767
5768 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5769 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5770
5771 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5772 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5773
5774 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5775 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5776
5777 // All of the memory operands. For the pipeline description.
5778 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5779 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5780 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5781
5782
5783 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5784 // operations. it allows the src to be either an iRegI or a (ConvL2I
5785 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5786 // can be elided because the 32-bit instruction will just employ the
5787 // lower 32 bits anyway.
5788 //
5789 // n.b. this does not elide all L2I conversions. if the truncated
5790 // value is consumed by more than one operation then the ConvL2I
5791 // cannot be bundled into the consuming nodes so an l2i gets planted
5792 // (actually a movw $dst $src) and the downstream instructions consume
5793 // the result of the l2i as an iRegI input. That's a shame since the
5794 // movw is actually redundant but its not too costly.
5795
5796 opclass iRegIorL2I(iRegI, iRegL2I);
5797 opclass iRegPorL2P(iRegP, iRegL2P);
5798
5799 //----------PIPELINE-----------------------------------------------------------
5800 // Rules which define the behavior of the target architectures pipeline.
5801
5802 // For specific pipelines, eg A53, define the stages of that pipeline
5803 //pipe_desc(ISS, EX1, EX2, WR);
5804 #define ISS S0
5805 #define EX1 S1
5806 #define EX2 S2
5807 #define WR S3
5808
5809 // Integer ALU reg operation
5810 pipeline %{
5811
5812 attributes %{
5813 // ARM instructions are of fixed length
5814 fixed_size_instructions; // Fixed size instructions TODO does
5815 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5816 // ARM instructions come in 32-bit word units
5817 instruction_unit_size = 4; // An instruction is 4 bytes long
5818 instruction_fetch_unit_size = 64; // The processor fetches one line
5819 instruction_fetch_units = 1; // of 64 bytes
5820
5821 // List of nop instructions
5822 nops( MachNop );
5823 %}
5824
5825 // We don't use an actual pipeline model so don't care about resources
5826 // or description. we do use pipeline classes to introduce fixed
5827 // latencies
5828
5829 //----------RESOURCES----------------------------------------------------------
5830 // Resources are the functional units available to the machine
5831
5832 resources( INS0, INS1, INS01 = INS0 | INS1,
5833 ALU0, ALU1, ALU = ALU0 | ALU1,
5834 MAC,
5835 DIV,
5836 BRANCH,
5837 LDST,
5838 NEON_FP);
5839
5840 //----------PIPELINE DESCRIPTION-----------------------------------------------
5841 // Pipeline Description specifies the stages in the machine's pipeline
5842
5843 // Define the pipeline as a generic 6 stage pipeline
5844 pipe_desc(S0, S1, S2, S3, S4, S5);
5845
5846 //----------PIPELINE CLASSES---------------------------------------------------
5847 // Pipeline Classes describe the stages in which input and output are
5848 // referenced by the hardware pipeline.
5849
5850 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5851 %{
5852 single_instruction;
5853 src1 : S1(read);
5854 src2 : S2(read);
5855 dst : S5(write);
5856 INS01 : ISS;
5857 NEON_FP : S5;
5858 %}
5859
5860 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5861 %{
5862 single_instruction;
5863 src1 : S1(read);
5864 src2 : S2(read);
5865 dst : S5(write);
5866 INS01 : ISS;
5867 NEON_FP : S5;
5868 %}
5869
5870 pipe_class fp_uop_s(vRegF dst, vRegF src)
5871 %{
5872 single_instruction;
5873 src : S1(read);
5874 dst : S5(write);
5875 INS01 : ISS;
5876 NEON_FP : S5;
5877 %}
5878
5879 pipe_class fp_uop_d(vRegD dst, vRegD src)
5880 %{
5881 single_instruction;
5882 src : S1(read);
5883 dst : S5(write);
5884 INS01 : ISS;
5885 NEON_FP : S5;
5886 %}
5887
5888 pipe_class fp_d2f(vRegF dst, vRegD src)
5889 %{
5890 single_instruction;
5891 src : S1(read);
5892 dst : S5(write);
5893 INS01 : ISS;
5894 NEON_FP : S5;
5895 %}
5896
5897 pipe_class fp_f2d(vRegD dst, vRegF src)
5898 %{
5899 single_instruction;
5900 src : S1(read);
5901 dst : S5(write);
5902 INS01 : ISS;
5903 NEON_FP : S5;
5904 %}
5905
5906 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5907 %{
5908 single_instruction;
5909 src : S1(read);
5910 dst : S5(write);
5911 INS01 : ISS;
5912 NEON_FP : S5;
5913 %}
5914
5915 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5916 %{
5917 single_instruction;
5918 src : S1(read);
5919 dst : S5(write);
5920 INS01 : ISS;
5921 NEON_FP : S5;
5922 %}
5923
5924 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5925 %{
5926 single_instruction;
5927 src : S1(read);
5928 dst : S5(write);
5929 INS01 : ISS;
5930 NEON_FP : S5;
5931 %}
5932
5933 pipe_class fp_l2f(vRegF dst, iRegL src)
5934 %{
5935 single_instruction;
5936 src : S1(read);
5937 dst : S5(write);
5938 INS01 : ISS;
5939 NEON_FP : S5;
5940 %}
5941
5942 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5943 %{
5944 single_instruction;
5945 src : S1(read);
5946 dst : S5(write);
5947 INS01 : ISS;
5948 NEON_FP : S5;
5949 %}
5950
5951 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5952 %{
5953 single_instruction;
5954 src : S1(read);
5955 dst : S5(write);
5956 INS01 : ISS;
5957 NEON_FP : S5;
5958 %}
5959
5960 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5961 %{
5962 single_instruction;
5963 src : S1(read);
5964 dst : S5(write);
5965 INS01 : ISS;
5966 NEON_FP : S5;
5967 %}
5968
5969 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5970 %{
5971 single_instruction;
5972 src : S1(read);
5973 dst : S5(write);
5974 INS01 : ISS;
5975 NEON_FP : S5;
5976 %}
5977
5978 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5979 %{
5980 single_instruction;
5981 src1 : S1(read);
5982 src2 : S2(read);
5983 dst : S5(write);
5984 INS0 : ISS;
5985 NEON_FP : S5;
5986 %}
5987
5988 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5989 %{
5990 single_instruction;
5991 src1 : S1(read);
5992 src2 : S2(read);
5993 dst : S5(write);
5994 INS0 : ISS;
5995 NEON_FP : S5;
5996 %}
5997
5998 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5999 %{
6000 single_instruction;
6001 cr : S1(read);
6002 src1 : S1(read);
6003 src2 : S1(read);
6004 dst : S3(write);
6005 INS01 : ISS;
6006 NEON_FP : S3;
6007 %}
6008
6009 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6010 %{
6011 single_instruction;
6012 cr : S1(read);
6013 src1 : S1(read);
6014 src2 : S1(read);
6015 dst : S3(write);
6016 INS01 : ISS;
6017 NEON_FP : S3;
6018 %}
6019
6020 pipe_class fp_imm_s(vRegF dst)
6021 %{
6022 single_instruction;
6023 dst : S3(write);
6024 INS01 : ISS;
6025 NEON_FP : S3;
6026 %}
6027
6028 pipe_class fp_imm_d(vRegD dst)
6029 %{
6030 single_instruction;
6031 dst : S3(write);
6032 INS01 : ISS;
6033 NEON_FP : S3;
6034 %}
6035
6036 pipe_class fp_load_constant_s(vRegF dst)
6037 %{
6038 single_instruction;
6039 dst : S4(write);
6040 INS01 : ISS;
6041 NEON_FP : S4;
6042 %}
6043
6044 pipe_class fp_load_constant_d(vRegD dst)
6045 %{
6046 single_instruction;
6047 dst : S4(write);
6048 INS01 : ISS;
6049 NEON_FP : S4;
6050 %}
6051
6052 //------- Integer ALU operations --------------------------
6053
6054 // Integer ALU reg-reg operation
6055 // Operands needed in EX1, result generated in EX2
6056 // Eg. ADD x0, x1, x2
6057 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6058 %{
6059 single_instruction;
6060 dst : EX2(write);
6061 src1 : EX1(read);
6062 src2 : EX1(read);
6063 INS01 : ISS; // Dual issue as instruction 0 or 1
6064 ALU : EX2;
6065 %}
6066
6067 // Integer ALU reg-reg operation with constant shift
6068 // Shifted register must be available in LATE_ISS instead of EX1
6069 // Eg. ADD x0, x1, x2, LSL #2
6070 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6071 %{
6072 single_instruction;
6073 dst : EX2(write);
6074 src1 : EX1(read);
6075 src2 : ISS(read);
6076 INS01 : ISS;
6077 ALU : EX2;
6078 %}
6079
6080 // Integer ALU reg operation with constant shift
6081 // Eg. LSL x0, x1, #shift
6082 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6083 %{
6084 single_instruction;
6085 dst : EX2(write);
6086 src1 : ISS(read);
6087 INS01 : ISS;
6088 ALU : EX2;
6089 %}
6090
6091 // Integer ALU reg-reg operation with variable shift
6092 // Both operands must be available in LATE_ISS instead of EX1
6093 // Result is available in EX1 instead of EX2
6094 // Eg. LSLV x0, x1, x2
6095 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6096 %{
6097 single_instruction;
6098 dst : EX1(write);
6099 src1 : ISS(read);
6100 src2 : ISS(read);
6101 INS01 : ISS;
6102 ALU : EX1;
6103 %}
6104
6105 // Integer ALU reg-reg operation with extract
6106 // As for _vshift above, but result generated in EX2
6107 // Eg. EXTR x0, x1, x2, #N
6108 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6109 %{
6110 single_instruction;
6111 dst : EX2(write);
6112 src1 : ISS(read);
6113 src2 : ISS(read);
6114 INS1 : ISS; // Can only dual issue as Instruction 1
6115 ALU : EX1;
6116 %}
6117
6118 // Integer ALU reg operation
6119 // Eg. NEG x0, x1
6120 pipe_class ialu_reg(iRegI dst, iRegI src)
6121 %{
6122 single_instruction;
6123 dst : EX2(write);
6124 src : EX1(read);
6125 INS01 : ISS;
6126 ALU : EX2;
6127 %}
6128
6129 // Integer ALU reg mmediate operation
6130 // Eg. ADD x0, x1, #N
6131 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6132 %{
6133 single_instruction;
6134 dst : EX2(write);
6135 src1 : EX1(read);
6136 INS01 : ISS;
6137 ALU : EX2;
6138 %}
6139
6140 // Integer ALU immediate operation (no source operands)
6141 // Eg. MOV x0, #N
6142 pipe_class ialu_imm(iRegI dst)
6143 %{
6144 single_instruction;
6145 dst : EX1(write);
6146 INS01 : ISS;
6147 ALU : EX1;
6148 %}
6149
6150 //------- Compare operation -------------------------------
6151
6152 // Compare reg-reg
6153 // Eg. CMP x0, x1
6154 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6155 %{
6156 single_instruction;
6157 // fixed_latency(16);
6158 cr : EX2(write);
6159 op1 : EX1(read);
6160 op2 : EX1(read);
6161 INS01 : ISS;
6162 ALU : EX2;
6163 %}
6164
6165 // Compare reg-reg
6166 // Eg. CMP x0, #N
6167 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6168 %{
6169 single_instruction;
6170 // fixed_latency(16);
6171 cr : EX2(write);
6172 op1 : EX1(read);
6173 INS01 : ISS;
6174 ALU : EX2;
6175 %}
6176
6177 //------- Conditional instructions ------------------------
6178
6179 // Conditional no operands
6180 // Eg. CSINC x0, zr, zr, <cond>
6181 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6182 %{
6183 single_instruction;
6184 cr : EX1(read);
6185 dst : EX2(write);
6186 INS01 : ISS;
6187 ALU : EX2;
6188 %}
6189
6190 // Conditional 2 operand
6191 // EG. CSEL X0, X1, X2, <cond>
6192 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6193 %{
6194 single_instruction;
6195 cr : EX1(read);
6196 src1 : EX1(read);
6197 src2 : EX1(read);
6198 dst : EX2(write);
6199 INS01 : ISS;
6200 ALU : EX2;
6201 %}
6202
6203 // Conditional 2 operand
6204 // EG. CSEL X0, X1, X2, <cond>
6205 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6206 %{
6207 single_instruction;
6208 cr : EX1(read);
6209 src : EX1(read);
6210 dst : EX2(write);
6211 INS01 : ISS;
6212 ALU : EX2;
6213 %}
6214
6215 //------- Multiply pipeline operations --------------------
6216
6217 // Multiply reg-reg
6218 // Eg. MUL w0, w1, w2
6219 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6220 %{
6221 single_instruction;
6222 dst : WR(write);
6223 src1 : ISS(read);
6224 src2 : ISS(read);
6225 INS01 : ISS;
6226 MAC : WR;
6227 %}
6228
6229 // Multiply accumulate
6230 // Eg. MADD w0, w1, w2, w3
6231 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6232 %{
6233 single_instruction;
6234 dst : WR(write);
6235 src1 : ISS(read);
6236 src2 : ISS(read);
6237 src3 : ISS(read);
6238 INS01 : ISS;
6239 MAC : WR;
6240 %}
6241
6242 // Eg. MUL w0, w1, w2
6243 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6244 %{
6245 single_instruction;
6246 fixed_latency(3); // Maximum latency for 64 bit mul
6247 dst : WR(write);
6248 src1 : ISS(read);
6249 src2 : ISS(read);
6250 INS01 : ISS;
6251 MAC : WR;
6252 %}
6253
6254 // Multiply accumulate
6255 // Eg. MADD w0, w1, w2, w3
6256 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6257 %{
6258 single_instruction;
6259 fixed_latency(3); // Maximum latency for 64 bit mul
6260 dst : WR(write);
6261 src1 : ISS(read);
6262 src2 : ISS(read);
6263 src3 : ISS(read);
6264 INS01 : ISS;
6265 MAC : WR;
6266 %}
6267
6268 //------- Divide pipeline operations --------------------
6269
6270 // Eg. SDIV w0, w1, w2
6271 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6272 %{
6273 single_instruction;
6274 fixed_latency(8); // Maximum latency for 32 bit divide
6275 dst : WR(write);
6276 src1 : ISS(read);
6277 src2 : ISS(read);
6278 INS0 : ISS; // Can only dual issue as instruction 0
6279 DIV : WR;
6280 %}
6281
6282 // Eg. SDIV x0, x1, x2
6283 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6284 %{
6285 single_instruction;
6286 fixed_latency(16); // Maximum latency for 64 bit divide
6287 dst : WR(write);
6288 src1 : ISS(read);
6289 src2 : ISS(read);
6290 INS0 : ISS; // Can only dual issue as instruction 0
6291 DIV : WR;
6292 %}
6293
6294 //------- Load pipeline operations ------------------------
6295
6296 // Load - prefetch
6297 // Eg. PFRM <mem>
6298 pipe_class iload_prefetch(memory mem)
6299 %{
6300 single_instruction;
6301 mem : ISS(read);
6302 INS01 : ISS;
6303 LDST : WR;
6304 %}
6305
6306 // Load - reg, mem
6307 // Eg. LDR x0, <mem>
6308 pipe_class iload_reg_mem(iRegI dst, memory mem)
6309 %{
6310 single_instruction;
6311 dst : WR(write);
6312 mem : ISS(read);
6313 INS01 : ISS;
6314 LDST : WR;
6315 %}
6316
6317 // Load - reg, reg
6318 // Eg. LDR x0, [sp, x1]
6319 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6320 %{
6321 single_instruction;
6322 dst : WR(write);
6323 src : ISS(read);
6324 INS01 : ISS;
6325 LDST : WR;
6326 %}
6327
6328 //------- Store pipeline operations -----------------------
6329
6330 // Store - zr, mem
6331 // Eg. STR zr, <mem>
6332 pipe_class istore_mem(memory mem)
6333 %{
6334 single_instruction;
6335 mem : ISS(read);
6336 INS01 : ISS;
6337 LDST : WR;
6338 %}
6339
6340 // Store - reg, mem
6341 // Eg. STR x0, <mem>
6342 pipe_class istore_reg_mem(iRegI src, memory mem)
6343 %{
6344 single_instruction;
6345 mem : ISS(read);
6346 src : EX2(read);
6347 INS01 : ISS;
6348 LDST : WR;
6349 %}
6350
6351 // Store - reg, reg
6352 // Eg. STR x0, [sp, x1]
6353 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6354 %{
6355 single_instruction;
6356 dst : ISS(read);
6357 src : EX2(read);
6358 INS01 : ISS;
6359 LDST : WR;
6360 %}
6361
6362 //------- Store pipeline operations -----------------------
6363
6364 // Branch
6365 pipe_class pipe_branch()
6366 %{
6367 single_instruction;
6368 INS01 : ISS;
6369 BRANCH : EX1;
6370 %}
6371
6372 // Conditional branch
6373 pipe_class pipe_branch_cond(rFlagsReg cr)
6374 %{
6375 single_instruction;
6376 cr : EX1(read);
6377 INS01 : ISS;
6378 BRANCH : EX1;
6379 %}
6380
6381 // Compare & Branch
6382 // EG. CBZ/CBNZ
6383 pipe_class pipe_cmp_branch(iRegI op1)
6384 %{
6385 single_instruction;
6386 op1 : EX1(read);
6387 INS01 : ISS;
6388 BRANCH : EX1;
6389 %}
6390
6391 //------- Synchronisation operations ----------------------
6392
6393 // Any operation requiring serialization.
6394 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6395 pipe_class pipe_serial()
6396 %{
6397 single_instruction;
6398 force_serialization;
6399 fixed_latency(16);
6400 INS01 : ISS(2); // Cannot dual issue with any other instruction
6401 LDST : WR;
6402 %}
6403
6404 // Generic big/slow expanded idiom - also serialized
6405 pipe_class pipe_slow()
6406 %{
6407 instruction_count(10);
6408 multiple_bundles;
6409 force_serialization;
6410 fixed_latency(16);
6411 INS01 : ISS(2); // Cannot dual issue with any other instruction
6412 LDST : WR;
6413 %}
6414
6415 // Empty pipeline class
6416 pipe_class pipe_class_empty()
6417 %{
6418 single_instruction;
6419 fixed_latency(0);
6420 %}
6421
6422 // Default pipeline class.
6423 pipe_class pipe_class_default()
6424 %{
6425 single_instruction;
6426 fixed_latency(2);
6427 %}
6428
6429 // Pipeline class for compares.
6430 pipe_class pipe_class_compare()
6431 %{
6432 single_instruction;
6433 fixed_latency(16);
6434 %}
6435
6436 // Pipeline class for memory operations.
6437 pipe_class pipe_class_memory()
6438 %{
6439 single_instruction;
6440 fixed_latency(16);
6441 %}
6442
6443 // Pipeline class for call.
6444 pipe_class pipe_class_call()
6445 %{
6446 single_instruction;
6447 fixed_latency(100);
6448 %}
6449
6450 // Define the class for the Nop node.
6451 define %{
6452 MachNop = pipe_class_empty;
6453 %}
6454
6455 %}
6456 //----------INSTRUCTIONS-------------------------------------------------------
6457 //
6458 // match -- States which machine-independent subtree may be replaced
6459 // by this instruction.
6460 // ins_cost -- The estimated cost of this instruction is used by instruction
6461 // selection to identify a minimum cost tree of machine
6462 // instructions that matches a tree of machine-independent
6463 // instructions.
6464 // format -- A string providing the disassembly for this instruction.
6465 // The value of an instruction's operand may be inserted
6466 // by referring to it with a '$' prefix.
6467 // opcode -- Three instruction opcodes may be provided. These are referred
6468 // to within an encode class as $primary, $secondary, and $tertiary
6469 // rrspectively. The primary opcode is commonly used to
6470 // indicate the type of machine instruction, while secondary
6471 // and tertiary are often used for prefix options or addressing
6472 // modes.
6473 // ins_encode -- A list of encode classes with parameters. The encode class
6474 // name must have been defined in an 'enc_class' specification
6475 // in the encode section of the architecture description.
6476
6477 // ============================================================================
6478 // Memory (Load/Store) Instructions
6479
6480 // Load Instructions
6481
6482 // Load Byte (8 bit signed)
6483 instruct loadB(iRegINoSp dst, memory1 mem)
6484 %{
6485 match(Set dst (LoadB mem));
6486 predicate(!needs_acquiring_load(n));
6487
6488 ins_cost(4 * INSN_COST);
6489 format %{ "ldrsbw $dst, $mem\t# byte" %}
6490
6491 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6492
6493 ins_pipe(iload_reg_mem);
6494 %}
6495
6496 // Load Byte (8 bit signed) into long
6497 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6498 %{
6499 match(Set dst (ConvI2L (LoadB mem)));
6500 predicate(!needs_acquiring_load(n->in(1)));
6501
6502 ins_cost(4 * INSN_COST);
6503 format %{ "ldrsb $dst, $mem\t# byte" %}
6504
6505 ins_encode(aarch64_enc_ldrsb(dst, mem));
6506
6507 ins_pipe(iload_reg_mem);
6508 %}
6509
6510 // Load Byte (8 bit unsigned)
6511 instruct loadUB(iRegINoSp dst, memory1 mem)
6512 %{
6513 match(Set dst (LoadUB mem));
6514 predicate(!needs_acquiring_load(n));
6515
6516 ins_cost(4 * INSN_COST);
6517 format %{ "ldrbw $dst, $mem\t# byte" %}
6518
6519 ins_encode(aarch64_enc_ldrb(dst, mem));
6520
6521 ins_pipe(iload_reg_mem);
6522 %}
6523
6524 // Load Byte (8 bit unsigned) into long
6525 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6526 %{
6527 match(Set dst (ConvI2L (LoadUB mem)));
6528 predicate(!needs_acquiring_load(n->in(1)));
6529
6530 ins_cost(4 * INSN_COST);
6531 format %{ "ldrb $dst, $mem\t# byte" %}
6532
6533 ins_encode(aarch64_enc_ldrb(dst, mem));
6534
6535 ins_pipe(iload_reg_mem);
6536 %}
6537
6538 // Load Short (16 bit signed)
6539 instruct loadS(iRegINoSp dst, memory2 mem)
6540 %{
6541 match(Set dst (LoadS mem));
6542 predicate(!needs_acquiring_load(n));
6543
6544 ins_cost(4 * INSN_COST);
6545 format %{ "ldrshw $dst, $mem\t# short" %}
6546
6547 ins_encode(aarch64_enc_ldrshw(dst, mem));
6548
6549 ins_pipe(iload_reg_mem);
6550 %}
6551
6552 // Load Short (16 bit signed) into long
6553 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6554 %{
6555 match(Set dst (ConvI2L (LoadS mem)));
6556 predicate(!needs_acquiring_load(n->in(1)));
6557
6558 ins_cost(4 * INSN_COST);
6559 format %{ "ldrsh $dst, $mem\t# short" %}
6560
6561 ins_encode(aarch64_enc_ldrsh(dst, mem));
6562
6563 ins_pipe(iload_reg_mem);
6564 %}
6565
6566 // Load Char (16 bit unsigned)
6567 instruct loadUS(iRegINoSp dst, memory2 mem)
6568 %{
6569 match(Set dst (LoadUS mem));
6570 predicate(!needs_acquiring_load(n));
6571
6572 ins_cost(4 * INSN_COST);
6573 format %{ "ldrh $dst, $mem\t# short" %}
6574
6575 ins_encode(aarch64_enc_ldrh(dst, mem));
6576
6577 ins_pipe(iload_reg_mem);
6578 %}
6579
6580 // Load Short/Char (16 bit unsigned) into long
6581 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6582 %{
6583 match(Set dst (ConvI2L (LoadUS mem)));
6584 predicate(!needs_acquiring_load(n->in(1)));
6585
6586 ins_cost(4 * INSN_COST);
6587 format %{ "ldrh $dst, $mem\t# short" %}
6588
6589 ins_encode(aarch64_enc_ldrh(dst, mem));
6590
6591 ins_pipe(iload_reg_mem);
6592 %}
6593
6594 // Load Integer (32 bit signed)
6595 instruct loadI(iRegINoSp dst, memory4 mem)
6596 %{
6597 match(Set dst (LoadI mem));
6598 predicate(!needs_acquiring_load(n));
6599
6600 ins_cost(4 * INSN_COST);
6601 format %{ "ldrw $dst, $mem\t# int" %}
6602
6603 ins_encode(aarch64_enc_ldrw(dst, mem));
6604
6605 ins_pipe(iload_reg_mem);
6606 %}
6607
6608 // Load Integer (32 bit signed) into long
6609 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6610 %{
6611 match(Set dst (ConvI2L (LoadI mem)));
6612 predicate(!needs_acquiring_load(n->in(1)));
6613
6614 ins_cost(4 * INSN_COST);
6615 format %{ "ldrsw $dst, $mem\t# int" %}
6616
6617 ins_encode(aarch64_enc_ldrsw(dst, mem));
6618
6619 ins_pipe(iload_reg_mem);
6620 %}
6621
6622 // Load Integer (32 bit unsigned) into long
6623 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6624 %{
6625 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6626 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6627
6628 ins_cost(4 * INSN_COST);
6629 format %{ "ldrw $dst, $mem\t# int" %}
6630
6631 ins_encode(aarch64_enc_ldrw(dst, mem));
6632
6633 ins_pipe(iload_reg_mem);
6634 %}
6635
6636 // Load Long (64 bit signed)
6637 instruct loadL(iRegLNoSp dst, memory8 mem)
6638 %{
6639 match(Set dst (LoadL mem));
6640 predicate(!needs_acquiring_load(n));
6641
6642 ins_cost(4 * INSN_COST);
6643 format %{ "ldr $dst, $mem\t# int" %}
6644
6645 ins_encode(aarch64_enc_ldr(dst, mem));
6646
6647 ins_pipe(iload_reg_mem);
6648 %}
6649
6650 // Load Range
6651 instruct loadRange(iRegINoSp dst, memory4 mem)
6652 %{
6653 match(Set dst (LoadRange mem));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrw $dst, $mem\t# range" %}
6657
6658 ins_encode(aarch64_enc_ldrw(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Pointer
6664 instruct loadP(iRegPNoSp dst, memory8 mem)
6665 %{
6666 match(Set dst (LoadP mem));
6667 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldr $dst, $mem\t# ptr" %}
6671
6672 ins_encode(aarch64_enc_ldr(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Compressed Pointer
6678 instruct loadN(iRegNNoSp dst, memory4 mem)
6679 %{
6680 match(Set dst (LoadN mem));
6681 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6682
6683 ins_cost(4 * INSN_COST);
6684 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6685
6686 ins_encode(aarch64_enc_ldrw(dst, mem));
6687
6688 ins_pipe(iload_reg_mem);
6689 %}
6690
6691 // Load Klass Pointer
6692 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6693 %{
6694 match(Set dst (LoadKlass mem));
6695 predicate(!needs_acquiring_load(n));
6696
6697 ins_cost(4 * INSN_COST);
6698 format %{ "ldr $dst, $mem\t# class" %}
6699
6700 ins_encode(aarch64_enc_ldr(dst, mem));
6701
6702 ins_pipe(iload_reg_mem);
6703 %}
6704
6705 // Load Narrow Klass Pointer
6706 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6707 %{
6708 match(Set dst (LoadNKlass mem));
6709 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6713
6714 ins_encode(aarch64_enc_ldrw(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6720 %{
6721 match(Set dst (LoadNKlass mem));
6722 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{
6726 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6727 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6728 %}
6729 ins_encode %{
6730 // inlined aarch64_enc_ldrw
6731 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6732 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6733 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6734 %}
6735 ins_pipe(iload_reg_mem);
6736 %}
6737
6738 // Load Float
6739 instruct loadF(vRegF dst, memory4 mem)
6740 %{
6741 match(Set dst (LoadF mem));
6742 predicate(!needs_acquiring_load(n));
6743
6744 ins_cost(4 * INSN_COST);
6745 format %{ "ldrs $dst, $mem\t# float" %}
6746
6747 ins_encode( aarch64_enc_ldrs(dst, mem) );
6748
6749 ins_pipe(pipe_class_memory);
6750 %}
6751
6752 // Load Double
6753 instruct loadD(vRegD dst, memory8 mem)
6754 %{
6755 match(Set dst (LoadD mem));
6756 predicate(!needs_acquiring_load(n));
6757
6758 ins_cost(4 * INSN_COST);
6759 format %{ "ldrd $dst, $mem\t# double" %}
6760
6761 ins_encode( aarch64_enc_ldrd(dst, mem) );
6762
6763 ins_pipe(pipe_class_memory);
6764 %}
6765
6766
6767 // Load Int Constant
6768 instruct loadConI(iRegINoSp dst, immI src)
6769 %{
6770 match(Set dst src);
6771
6772 ins_cost(INSN_COST);
6773 format %{ "mov $dst, $src\t# int" %}
6774
6775 ins_encode( aarch64_enc_movw_imm(dst, src) );
6776
6777 ins_pipe(ialu_imm);
6778 %}
6779
6780 // Load Long Constant
6781 instruct loadConL(iRegLNoSp dst, immL src)
6782 %{
6783 match(Set dst src);
6784
6785 ins_cost(INSN_COST);
6786 format %{ "mov $dst, $src\t# long" %}
6787
6788 ins_encode( aarch64_enc_mov_imm(dst, src) );
6789
6790 ins_pipe(ialu_imm);
6791 %}
6792
6793 // Load Pointer Constant
6794
6795 instruct loadConP(iRegPNoSp dst, immP con)
6796 %{
6797 match(Set dst con);
6798
6799 ins_cost(INSN_COST * 4);
6800 format %{
6801 "mov $dst, $con\t# ptr\n\t"
6802 %}
6803
6804 ins_encode(aarch64_enc_mov_p(dst, con));
6805
6806 ins_pipe(ialu_imm);
6807 %}
6808
6809 // Load Null Pointer Constant
6810
6811 instruct loadConP0(iRegPNoSp dst, immP0 con)
6812 %{
6813 match(Set dst con);
6814
6815 ins_cost(INSN_COST);
6816 format %{ "mov $dst, $con\t# nullptr ptr" %}
6817
6818 ins_encode(aarch64_enc_mov_p0(dst, con));
6819
6820 ins_pipe(ialu_imm);
6821 %}
6822
6823 // Load Pointer Constant One
6824
6825 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6826 %{
6827 match(Set dst con);
6828
6829 ins_cost(INSN_COST);
6830 format %{ "mov $dst, $con\t# nullptr ptr" %}
6831
6832 ins_encode(aarch64_enc_mov_p1(dst, con));
6833
6834 ins_pipe(ialu_imm);
6835 %}
6836
6837 // Load Byte Map Base Constant
6838
6839 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6840 %{
6841 match(Set dst con);
6842
6843 ins_cost(INSN_COST);
6844 format %{ "adr $dst, $con\t# Byte Map Base" %}
6845
6846 ins_encode %{
6847 __ load_byte_map_base($dst$$Register);
6848 %}
6849
6850 ins_pipe(ialu_imm);
6851 %}
6852
6853 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6854 %{
6855 match(Set dst con);
6856
6857 ins_cost(INSN_COST);
6858 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6859
6860 ins_encode %{
6861 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6862 %}
6863
6864 ins_pipe(ialu_imm);
6865 %}
6866
6867 // Load Narrow Pointer Constant
6868
6869 instruct loadConN(iRegNNoSp dst, immN con)
6870 %{
6871 match(Set dst con);
6872
6873 ins_cost(INSN_COST * 4);
6874 format %{ "mov $dst, $con\t# compressed ptr" %}
6875
6876 ins_encode(aarch64_enc_mov_n(dst, con));
6877
6878 ins_pipe(ialu_imm);
6879 %}
6880
6881 // Load Narrow Null Pointer Constant
6882
6883 instruct loadConN0(iRegNNoSp dst, immN0 con)
6884 %{
6885 match(Set dst con);
6886
6887 ins_cost(INSN_COST);
6888 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6889
6890 ins_encode(aarch64_enc_mov_n0(dst, con));
6891
6892 ins_pipe(ialu_imm);
6893 %}
6894
6895 // Load Narrow Klass Constant
6896
6897 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6898 %{
6899 match(Set dst con);
6900
6901 ins_cost(INSN_COST);
6902 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6903
6904 ins_encode(aarch64_enc_mov_nk(dst, con));
6905
6906 ins_pipe(ialu_imm);
6907 %}
6908
6909 // Load Packed Float Constant
6910
6911 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6912 match(Set dst con);
6913 ins_cost(INSN_COST * 4);
6914 format %{ "fmovs $dst, $con"%}
6915 ins_encode %{
6916 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6917 %}
6918
6919 ins_pipe(fp_imm_s);
6920 %}
6921
6922 // Load Float Constant
6923
6924 instruct loadConF(vRegF dst, immF con) %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST * 4);
6928
6929 format %{
6930 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6931 %}
6932
6933 ins_encode %{
6934 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6935 %}
6936
6937 ins_pipe(fp_load_constant_s);
6938 %}
6939
6940 // Load Packed Double Constant
6941
6942 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6943 match(Set dst con);
6944 ins_cost(INSN_COST);
6945 format %{ "fmovd $dst, $con"%}
6946 ins_encode %{
6947 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6948 %}
6949
6950 ins_pipe(fp_imm_d);
6951 %}
6952
6953 // Load Double Constant
6954
6955 instruct loadConD(vRegD dst, immD con) %{
6956 match(Set dst con);
6957
6958 ins_cost(INSN_COST * 5);
6959 format %{
6960 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6961 %}
6962
6963 ins_encode %{
6964 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6965 %}
6966
6967 ins_pipe(fp_load_constant_d);
6968 %}
6969
6970 // Store Instructions
6971
6972 // Store Byte
6973 instruct storeB(iRegIorL2I src, memory1 mem)
6974 %{
6975 match(Set mem (StoreB mem src));
6976 predicate(!needs_releasing_store(n));
6977
6978 ins_cost(INSN_COST);
6979 format %{ "strb $src, $mem\t# byte" %}
6980
6981 ins_encode(aarch64_enc_strb(src, mem));
6982
6983 ins_pipe(istore_reg_mem);
6984 %}
6985
6986
6987 instruct storeimmB0(immI0 zero, memory1 mem)
6988 %{
6989 match(Set mem (StoreB mem zero));
6990 predicate(!needs_releasing_store(n));
6991
6992 ins_cost(INSN_COST);
6993 format %{ "strb rscractch2, $mem\t# byte" %}
6994
6995 ins_encode(aarch64_enc_strb0(mem));
6996
6997 ins_pipe(istore_mem);
6998 %}
6999
7000 // Store Char/Short
7001 instruct storeC(iRegIorL2I src, memory2 mem)
7002 %{
7003 match(Set mem (StoreC mem src));
7004 predicate(!needs_releasing_store(n));
7005
7006 ins_cost(INSN_COST);
7007 format %{ "strh $src, $mem\t# short" %}
7008
7009 ins_encode(aarch64_enc_strh(src, mem));
7010
7011 ins_pipe(istore_reg_mem);
7012 %}
7013
7014 instruct storeimmC0(immI0 zero, memory2 mem)
7015 %{
7016 match(Set mem (StoreC mem zero));
7017 predicate(!needs_releasing_store(n));
7018
7019 ins_cost(INSN_COST);
7020 format %{ "strh zr, $mem\t# short" %}
7021
7022 ins_encode(aarch64_enc_strh0(mem));
7023
7024 ins_pipe(istore_mem);
7025 %}
7026
7027 // Store Integer
7028
7029 instruct storeI(iRegIorL2I src, memory4 mem)
7030 %{
7031 match(Set mem(StoreI mem src));
7032 predicate(!needs_releasing_store(n));
7033
7034 ins_cost(INSN_COST);
7035 format %{ "strw $src, $mem\t# int" %}
7036
7037 ins_encode(aarch64_enc_strw(src, mem));
7038
7039 ins_pipe(istore_reg_mem);
7040 %}
7041
7042 instruct storeimmI0(immI0 zero, memory4 mem)
7043 %{
7044 match(Set mem(StoreI mem zero));
7045 predicate(!needs_releasing_store(n));
7046
7047 ins_cost(INSN_COST);
7048 format %{ "strw zr, $mem\t# int" %}
7049
7050 ins_encode(aarch64_enc_strw0(mem));
7051
7052 ins_pipe(istore_mem);
7053 %}
7054
7055 // Store Long (64 bit signed)
7056 instruct storeL(iRegL src, memory8 mem)
7057 %{
7058 match(Set mem (StoreL mem src));
7059 predicate(!needs_releasing_store(n));
7060
7061 ins_cost(INSN_COST);
7062 format %{ "str $src, $mem\t# int" %}
7063
7064 ins_encode(aarch64_enc_str(src, mem));
7065
7066 ins_pipe(istore_reg_mem);
7067 %}
7068
7069 // Store Long (64 bit signed)
7070 instruct storeimmL0(immL0 zero, memory8 mem)
7071 %{
7072 match(Set mem (StoreL mem zero));
7073 predicate(!needs_releasing_store(n));
7074
7075 ins_cost(INSN_COST);
7076 format %{ "str zr, $mem\t# int" %}
7077
7078 ins_encode(aarch64_enc_str0(mem));
7079
7080 ins_pipe(istore_mem);
7081 %}
7082
7083 // Store Pointer
7084 instruct storeP(iRegP src, memory8 mem)
7085 %{
7086 match(Set mem (StoreP mem src));
7087 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7088
7089 ins_cost(INSN_COST);
7090 format %{ "str $src, $mem\t# ptr" %}
7091
7092 ins_encode(aarch64_enc_str(src, mem));
7093
7094 ins_pipe(istore_reg_mem);
7095 %}
7096
7097 // Store Pointer
7098 instruct storeimmP0(immP0 zero, memory8 mem)
7099 %{
7100 match(Set mem (StoreP mem zero));
7101 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7102
7103 ins_cost(INSN_COST);
7104 format %{ "str zr, $mem\t# ptr" %}
7105
7106 ins_encode(aarch64_enc_str0(mem));
7107
7108 ins_pipe(istore_mem);
7109 %}
7110
7111 // Store Compressed Pointer
7112 instruct storeN(iRegN src, memory4 mem)
7113 %{
7114 match(Set mem (StoreN mem src));
7115 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7116
7117 ins_cost(INSN_COST);
7118 format %{ "strw $src, $mem\t# compressed ptr" %}
7119
7120 ins_encode(aarch64_enc_strw(src, mem));
7121
7122 ins_pipe(istore_reg_mem);
7123 %}
7124
7125 instruct storeImmN0(immN0 zero, memory4 mem)
7126 %{
7127 match(Set mem (StoreN mem zero));
7128 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7129
7130 ins_cost(INSN_COST);
7131 format %{ "strw zr, $mem\t# compressed ptr" %}
7132
7133 ins_encode(aarch64_enc_strw0(mem));
7134
7135 ins_pipe(istore_mem);
7136 %}
7137
7138 // Store Float
7139 instruct storeF(vRegF src, memory4 mem)
7140 %{
7141 match(Set mem (StoreF mem src));
7142 predicate(!needs_releasing_store(n));
7143
7144 ins_cost(INSN_COST);
7145 format %{ "strs $src, $mem\t# float" %}
7146
7147 ins_encode( aarch64_enc_strs(src, mem) );
7148
7149 ins_pipe(pipe_class_memory);
7150 %}
7151
7152 // TODO
7153 // implement storeImmF0 and storeFImmPacked
7154
7155 // Store Double
7156 instruct storeD(vRegD src, memory8 mem)
7157 %{
7158 match(Set mem (StoreD mem src));
7159 predicate(!needs_releasing_store(n));
7160
7161 ins_cost(INSN_COST);
7162 format %{ "strd $src, $mem\t# double" %}
7163
7164 ins_encode( aarch64_enc_strd(src, mem) );
7165
7166 ins_pipe(pipe_class_memory);
7167 %}
7168
7169 // Store Compressed Klass Pointer
7170 instruct storeNKlass(iRegN src, memory4 mem)
7171 %{
7172 predicate(!needs_releasing_store(n));
7173 match(Set mem (StoreNKlass mem src));
7174
7175 ins_cost(INSN_COST);
7176 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7177
7178 ins_encode(aarch64_enc_strw(src, mem));
7179
7180 ins_pipe(istore_reg_mem);
7181 %}
7182
7183 // TODO
7184 // implement storeImmD0 and storeDImmPacked
7185
7186 // prefetch instructions
7187 // Must be safe to execute with invalid address (cannot fault).
7188
7189 instruct prefetchalloc( memory8 mem ) %{
7190 match(PrefetchAllocation mem);
7191
7192 ins_cost(INSN_COST);
7193 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7194
7195 ins_encode( aarch64_enc_prefetchw(mem) );
7196
7197 ins_pipe(iload_prefetch);
7198 %}
7199
7200 // ---------------- volatile loads and stores ----------------
7201
7202 // Load Byte (8 bit signed)
7203 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7204 %{
7205 match(Set dst (LoadB mem));
7206
7207 ins_cost(VOLATILE_REF_COST);
7208 format %{ "ldarsb $dst, $mem\t# byte" %}
7209
7210 ins_encode(aarch64_enc_ldarsb(dst, mem));
7211
7212 ins_pipe(pipe_serial);
7213 %}
7214
7215 // Load Byte (8 bit signed) into long
7216 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7217 %{
7218 match(Set dst (ConvI2L (LoadB mem)));
7219
7220 ins_cost(VOLATILE_REF_COST);
7221 format %{ "ldarsb $dst, $mem\t# byte" %}
7222
7223 ins_encode(aarch64_enc_ldarsb(dst, mem));
7224
7225 ins_pipe(pipe_serial);
7226 %}
7227
7228 // Load Byte (8 bit unsigned)
7229 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7230 %{
7231 match(Set dst (LoadUB mem));
7232
7233 ins_cost(VOLATILE_REF_COST);
7234 format %{ "ldarb $dst, $mem\t# byte" %}
7235
7236 ins_encode(aarch64_enc_ldarb(dst, mem));
7237
7238 ins_pipe(pipe_serial);
7239 %}
7240
7241 // Load Byte (8 bit unsigned) into long
7242 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7243 %{
7244 match(Set dst (ConvI2L (LoadUB mem)));
7245
7246 ins_cost(VOLATILE_REF_COST);
7247 format %{ "ldarb $dst, $mem\t# byte" %}
7248
7249 ins_encode(aarch64_enc_ldarb(dst, mem));
7250
7251 ins_pipe(pipe_serial);
7252 %}
7253
7254 // Load Short (16 bit signed)
7255 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7256 %{
7257 match(Set dst (LoadS mem));
7258
7259 ins_cost(VOLATILE_REF_COST);
7260 format %{ "ldarshw $dst, $mem\t# short" %}
7261
7262 ins_encode(aarch64_enc_ldarshw(dst, mem));
7263
7264 ins_pipe(pipe_serial);
7265 %}
7266
7267 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7268 %{
7269 match(Set dst (LoadUS mem));
7270
7271 ins_cost(VOLATILE_REF_COST);
7272 format %{ "ldarhw $dst, $mem\t# short" %}
7273
7274 ins_encode(aarch64_enc_ldarhw(dst, mem));
7275
7276 ins_pipe(pipe_serial);
7277 %}
7278
7279 // Load Short/Char (16 bit unsigned) into long
7280 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7281 %{
7282 match(Set dst (ConvI2L (LoadUS mem)));
7283
7284 ins_cost(VOLATILE_REF_COST);
7285 format %{ "ldarh $dst, $mem\t# short" %}
7286
7287 ins_encode(aarch64_enc_ldarh(dst, mem));
7288
7289 ins_pipe(pipe_serial);
7290 %}
7291
7292 // Load Short/Char (16 bit signed) into long
7293 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7294 %{
7295 match(Set dst (ConvI2L (LoadS mem)));
7296
7297 ins_cost(VOLATILE_REF_COST);
7298 format %{ "ldarh $dst, $mem\t# short" %}
7299
7300 ins_encode(aarch64_enc_ldarsh(dst, mem));
7301
7302 ins_pipe(pipe_serial);
7303 %}
7304
7305 // Load Integer (32 bit signed)
7306 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7307 %{
7308 match(Set dst (LoadI mem));
7309
7310 ins_cost(VOLATILE_REF_COST);
7311 format %{ "ldarw $dst, $mem\t# int" %}
7312
7313 ins_encode(aarch64_enc_ldarw(dst, mem));
7314
7315 ins_pipe(pipe_serial);
7316 %}
7317
7318 // Load Integer (32 bit unsigned) into long
7319 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7320 %{
7321 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7322
7323 ins_cost(VOLATILE_REF_COST);
7324 format %{ "ldarw $dst, $mem\t# int" %}
7325
7326 ins_encode(aarch64_enc_ldarw(dst, mem));
7327
7328 ins_pipe(pipe_serial);
7329 %}
7330
7331 // Load Long (64 bit signed)
7332 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7333 %{
7334 match(Set dst (LoadL mem));
7335
7336 ins_cost(VOLATILE_REF_COST);
7337 format %{ "ldar $dst, $mem\t# int" %}
7338
7339 ins_encode(aarch64_enc_ldar(dst, mem));
7340
7341 ins_pipe(pipe_serial);
7342 %}
7343
7344 // Load Pointer
7345 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7346 %{
7347 match(Set dst (LoadP mem));
7348 predicate(n->as_Load()->barrier_data() == 0);
7349
7350 ins_cost(VOLATILE_REF_COST);
7351 format %{ "ldar $dst, $mem\t# ptr" %}
7352
7353 ins_encode(aarch64_enc_ldar(dst, mem));
7354
7355 ins_pipe(pipe_serial);
7356 %}
7357
7358 // Load Compressed Pointer
7359 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7360 %{
7361 match(Set dst (LoadN mem));
7362 predicate(n->as_Load()->barrier_data() == 0);
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7366
7367 ins_encode(aarch64_enc_ldarw(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Float
7373 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (LoadF mem));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldars $dst, $mem\t# float" %}
7379
7380 ins_encode( aarch64_enc_fldars(dst, mem) );
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Double
7386 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set dst (LoadD mem));
7389
7390 ins_cost(VOLATILE_REF_COST);
7391 format %{ "ldard $dst, $mem\t# double" %}
7392
7393 ins_encode( aarch64_enc_fldard(dst, mem) );
7394
7395 ins_pipe(pipe_serial);
7396 %}
7397
7398 // Store Byte
7399 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7400 %{
7401 match(Set mem (StoreB mem src));
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "stlrb $src, $mem\t# byte" %}
7405
7406 ins_encode(aarch64_enc_stlrb(src, mem));
7407
7408 ins_pipe(pipe_class_memory);
7409 %}
7410
7411 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7412 %{
7413 match(Set mem (StoreB mem zero));
7414
7415 ins_cost(VOLATILE_REF_COST);
7416 format %{ "stlrb zr, $mem\t# byte" %}
7417
7418 ins_encode(aarch64_enc_stlrb0(mem));
7419
7420 ins_pipe(pipe_class_memory);
7421 %}
7422
7423 // Store Char/Short
7424 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7425 %{
7426 match(Set mem (StoreC mem src));
7427
7428 ins_cost(VOLATILE_REF_COST);
7429 format %{ "stlrh $src, $mem\t# short" %}
7430
7431 ins_encode(aarch64_enc_stlrh(src, mem));
7432
7433 ins_pipe(pipe_class_memory);
7434 %}
7435
7436 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7437 %{
7438 match(Set mem (StoreC mem zero));
7439
7440 ins_cost(VOLATILE_REF_COST);
7441 format %{ "stlrh zr, $mem\t# short" %}
7442
7443 ins_encode(aarch64_enc_stlrh0(mem));
7444
7445 ins_pipe(pipe_class_memory);
7446 %}
7447
7448 // Store Integer
7449
7450 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7451 %{
7452 match(Set mem(StoreI mem src));
7453
7454 ins_cost(VOLATILE_REF_COST);
7455 format %{ "stlrw $src, $mem\t# int" %}
7456
7457 ins_encode(aarch64_enc_stlrw(src, mem));
7458
7459 ins_pipe(pipe_class_memory);
7460 %}
7461
7462 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7463 %{
7464 match(Set mem(StoreI mem zero));
7465
7466 ins_cost(VOLATILE_REF_COST);
7467 format %{ "stlrw zr, $mem\t# int" %}
7468
7469 ins_encode(aarch64_enc_stlrw0(mem));
7470
7471 ins_pipe(pipe_class_memory);
7472 %}
7473
7474 // Store Long (64 bit signed)
7475 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7476 %{
7477 match(Set mem (StoreL mem src));
7478
7479 ins_cost(VOLATILE_REF_COST);
7480 format %{ "stlr $src, $mem\t# int" %}
7481
7482 ins_encode(aarch64_enc_stlr(src, mem));
7483
7484 ins_pipe(pipe_class_memory);
7485 %}
7486
7487 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7488 %{
7489 match(Set mem (StoreL mem zero));
7490
7491 ins_cost(VOLATILE_REF_COST);
7492 format %{ "stlr zr, $mem\t# int" %}
7493
7494 ins_encode(aarch64_enc_stlr0(mem));
7495
7496 ins_pipe(pipe_class_memory);
7497 %}
7498
7499 // Store Pointer
7500 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7501 %{
7502 match(Set mem (StoreP mem src));
7503 predicate(n->as_Store()->barrier_data() == 0);
7504
7505 ins_cost(VOLATILE_REF_COST);
7506 format %{ "stlr $src, $mem\t# ptr" %}
7507
7508 ins_encode(aarch64_enc_stlr(src, mem));
7509
7510 ins_pipe(pipe_class_memory);
7511 %}
7512
7513 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7514 %{
7515 match(Set mem (StoreP mem zero));
7516 predicate(n->as_Store()->barrier_data() == 0);
7517
7518 ins_cost(VOLATILE_REF_COST);
7519 format %{ "stlr zr, $mem\t# ptr" %}
7520
7521 ins_encode(aarch64_enc_stlr0(mem));
7522
7523 ins_pipe(pipe_class_memory);
7524 %}
7525
7526 // Store Compressed Pointer
7527 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7528 %{
7529 match(Set mem (StoreN mem src));
7530 predicate(n->as_Store()->barrier_data() == 0);
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7534
7535 ins_encode(aarch64_enc_stlrw(src, mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7541 %{
7542 match(Set mem (StoreN mem zero));
7543 predicate(n->as_Store()->barrier_data() == 0);
7544
7545 ins_cost(VOLATILE_REF_COST);
7546 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7547
7548 ins_encode(aarch64_enc_stlrw0(mem));
7549
7550 ins_pipe(pipe_class_memory);
7551 %}
7552
7553 // Store Float
7554 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreF mem src));
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlrs $src, $mem\t# float" %}
7560
7561 ins_encode( aarch64_enc_fstlrs(src, mem) );
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 // TODO
7567 // implement storeImmF0 and storeFImmPacked
7568
7569 // Store Double
7570 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7571 %{
7572 match(Set mem (StoreD mem src));
7573
7574 ins_cost(VOLATILE_REF_COST);
7575 format %{ "stlrd $src, $mem\t# double" %}
7576
7577 ins_encode( aarch64_enc_fstlrd(src, mem) );
7578
7579 ins_pipe(pipe_class_memory);
7580 %}
7581
7582 // ---------------- end of volatile loads and stores ----------------
7583
7584 instruct cacheWB(indirect addr)
7585 %{
7586 predicate(VM_Version::supports_data_cache_line_flush());
7587 match(CacheWB addr);
7588
7589 ins_cost(100);
7590 format %{"cache wb $addr" %}
7591 ins_encode %{
7592 assert($addr->index_position() < 0, "should be");
7593 assert($addr$$disp == 0, "should be");
7594 __ cache_wb(Address($addr$$base$$Register, 0));
7595 %}
7596 ins_pipe(pipe_slow); // XXX
7597 %}
7598
7599 instruct cacheWBPreSync()
7600 %{
7601 predicate(VM_Version::supports_data_cache_line_flush());
7602 match(CacheWBPreSync);
7603
7604 ins_cost(100);
7605 format %{"cache wb presync" %}
7606 ins_encode %{
7607 __ cache_wbsync(true);
7608 %}
7609 ins_pipe(pipe_slow); // XXX
7610 %}
7611
7612 instruct cacheWBPostSync()
7613 %{
7614 predicate(VM_Version::supports_data_cache_line_flush());
7615 match(CacheWBPostSync);
7616
7617 ins_cost(100);
7618 format %{"cache wb postsync" %}
7619 ins_encode %{
7620 __ cache_wbsync(false);
7621 %}
7622 ins_pipe(pipe_slow); // XXX
7623 %}
7624
7625 // ============================================================================
7626 // BSWAP Instructions
7627
7628 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7629 match(Set dst (ReverseBytesI src));
7630
7631 ins_cost(INSN_COST);
7632 format %{ "revw $dst, $src" %}
7633
7634 ins_encode %{
7635 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7636 %}
7637
7638 ins_pipe(ialu_reg);
7639 %}
7640
7641 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7642 match(Set dst (ReverseBytesL src));
7643
7644 ins_cost(INSN_COST);
7645 format %{ "rev $dst, $src" %}
7646
7647 ins_encode %{
7648 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7649 %}
7650
7651 ins_pipe(ialu_reg);
7652 %}
7653
7654 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7655 match(Set dst (ReverseBytesUS src));
7656
7657 ins_cost(INSN_COST);
7658 format %{ "rev16w $dst, $src" %}
7659
7660 ins_encode %{
7661 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7662 %}
7663
7664 ins_pipe(ialu_reg);
7665 %}
7666
7667 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7668 match(Set dst (ReverseBytesS src));
7669
7670 ins_cost(INSN_COST);
7671 format %{ "rev16w $dst, $src\n\t"
7672 "sbfmw $dst, $dst, #0, #15" %}
7673
7674 ins_encode %{
7675 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7676 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7677 %}
7678
7679 ins_pipe(ialu_reg);
7680 %}
7681
7682 // ============================================================================
7683 // Zero Count Instructions
7684
7685 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7686 match(Set dst (CountLeadingZerosI src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "clzw $dst, $src" %}
7690 ins_encode %{
7691 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7692 %}
7693
7694 ins_pipe(ialu_reg);
7695 %}
7696
7697 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7698 match(Set dst (CountLeadingZerosL src));
7699
7700 ins_cost(INSN_COST);
7701 format %{ "clz $dst, $src" %}
7702 ins_encode %{
7703 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7704 %}
7705
7706 ins_pipe(ialu_reg);
7707 %}
7708
7709 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7710 match(Set dst (CountTrailingZerosI src));
7711
7712 ins_cost(INSN_COST * 2);
7713 format %{ "rbitw $dst, $src\n\t"
7714 "clzw $dst, $dst" %}
7715 ins_encode %{
7716 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7717 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7718 %}
7719
7720 ins_pipe(ialu_reg);
7721 %}
7722
7723 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7724 match(Set dst (CountTrailingZerosL src));
7725
7726 ins_cost(INSN_COST * 2);
7727 format %{ "rbit $dst, $src\n\t"
7728 "clz $dst, $dst" %}
7729 ins_encode %{
7730 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7731 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7732 %}
7733
7734 ins_pipe(ialu_reg);
7735 %}
7736
7737 //---------- Population Count Instructions -------------------------------------
7738 //
7739
7740 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7741 match(Set dst (PopCountI src));
7742 effect(TEMP tmp);
7743 ins_cost(INSN_COST * 13);
7744
7745 format %{ "movw $src, $src\n\t"
7746 "mov $tmp, $src\t# vector (1D)\n\t"
7747 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7748 "addv $tmp, $tmp\t# vector (8B)\n\t"
7749 "mov $dst, $tmp\t# vector (1D)" %}
7750 ins_encode %{
7751 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7752 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7753 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7754 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7755 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7756 %}
7757
7758 ins_pipe(pipe_class_default);
7759 %}
7760
7761 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7762 match(Set dst (PopCountI (LoadI mem)));
7763 effect(TEMP tmp);
7764 ins_cost(INSN_COST * 13);
7765
7766 format %{ "ldrs $tmp, $mem\n\t"
7767 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7768 "addv $tmp, $tmp\t# vector (8B)\n\t"
7769 "mov $dst, $tmp\t# vector (1D)" %}
7770 ins_encode %{
7771 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7772 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7773 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7774 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7775 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7776 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7777 %}
7778
7779 ins_pipe(pipe_class_default);
7780 %}
7781
7782 // Note: Long.bitCount(long) returns an int.
7783 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7784 match(Set dst (PopCountL src));
7785 effect(TEMP tmp);
7786 ins_cost(INSN_COST * 13);
7787
7788 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7789 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7790 "addv $tmp, $tmp\t# vector (8B)\n\t"
7791 "mov $dst, $tmp\t# vector (1D)" %}
7792 ins_encode %{
7793 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7794 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7795 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7796 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7797 %}
7798
7799 ins_pipe(pipe_class_default);
7800 %}
7801
7802 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7803 match(Set dst (PopCountL (LoadL mem)));
7804 effect(TEMP tmp);
7805 ins_cost(INSN_COST * 13);
7806
7807 format %{ "ldrd $tmp, $mem\n\t"
7808 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7809 "addv $tmp, $tmp\t# vector (8B)\n\t"
7810 "mov $dst, $tmp\t# vector (1D)" %}
7811 ins_encode %{
7812 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7813 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7814 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7815 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7816 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7817 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7818 %}
7819
7820 ins_pipe(pipe_class_default);
7821 %}
7822
7823 // ============================================================================
7824 // VerifyVectorAlignment Instruction
7825
7826 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7827 match(Set addr (VerifyVectorAlignment addr mask));
7828 effect(KILL cr);
7829 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7830 ins_encode %{
7831 Label Lskip;
7832 // check if masked bits of addr are zero
7833 __ tst($addr$$Register, $mask$$constant);
7834 __ br(Assembler::EQ, Lskip);
7835 __ stop("verify_vector_alignment found a misaligned vector memory access");
7836 __ bind(Lskip);
7837 %}
7838 ins_pipe(pipe_slow);
7839 %}
7840
7841 // ============================================================================
7842 // MemBar Instruction
7843
7844 instruct load_fence() %{
7845 match(LoadFence);
7846 ins_cost(VOLATILE_REF_COST);
7847
7848 format %{ "load_fence" %}
7849
7850 ins_encode %{
7851 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7852 %}
7853 ins_pipe(pipe_serial);
7854 %}
7855
7856 instruct unnecessary_membar_acquire() %{
7857 predicate(unnecessary_acquire(n));
7858 match(MemBarAcquire);
7859 ins_cost(0);
7860
7861 format %{ "membar_acquire (elided)" %}
7862
7863 ins_encode %{
7864 __ block_comment("membar_acquire (elided)");
7865 %}
7866
7867 ins_pipe(pipe_class_empty);
7868 %}
7869
7870 instruct membar_acquire() %{
7871 match(MemBarAcquire);
7872 ins_cost(VOLATILE_REF_COST);
7873
7874 format %{ "membar_acquire\n\t"
7875 "dmb ishld" %}
7876
7877 ins_encode %{
7878 __ block_comment("membar_acquire");
7879 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7880 %}
7881
7882 ins_pipe(pipe_serial);
7883 %}
7884
7885
7886 instruct membar_acquire_lock() %{
7887 match(MemBarAcquireLock);
7888 ins_cost(VOLATILE_REF_COST);
7889
7890 format %{ "membar_acquire_lock (elided)" %}
7891
7892 ins_encode %{
7893 __ block_comment("membar_acquire_lock (elided)");
7894 %}
7895
7896 ins_pipe(pipe_serial);
7897 %}
7898
7899 instruct store_fence() %{
7900 match(StoreFence);
7901 ins_cost(VOLATILE_REF_COST);
7902
7903 format %{ "store_fence" %}
7904
7905 ins_encode %{
7906 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7907 %}
7908 ins_pipe(pipe_serial);
7909 %}
7910
7911 instruct unnecessary_membar_release() %{
7912 predicate(unnecessary_release(n));
7913 match(MemBarRelease);
7914 ins_cost(0);
7915
7916 format %{ "membar_release (elided)" %}
7917
7918 ins_encode %{
7919 __ block_comment("membar_release (elided)");
7920 %}
7921 ins_pipe(pipe_serial);
7922 %}
7923
7924 instruct membar_release() %{
7925 match(MemBarRelease);
7926 ins_cost(VOLATILE_REF_COST);
7927
7928 format %{ "membar_release\n\t"
7929 "dmb ishst\n\tdmb ishld" %}
7930
7931 ins_encode %{
7932 __ block_comment("membar_release");
7933 // These will be merged if AlwaysMergeDMB is enabled.
7934 __ membar(Assembler::StoreStore);
7935 __ membar(Assembler::LoadStore);
7936 %}
7937 ins_pipe(pipe_serial);
7938 %}
7939
7940 instruct membar_storestore() %{
7941 match(MemBarStoreStore);
7942 match(StoreStoreFence);
7943 ins_cost(VOLATILE_REF_COST);
7944
7945 format %{ "MEMBAR-store-store" %}
7946
7947 ins_encode %{
7948 __ membar(Assembler::StoreStore);
7949 %}
7950 ins_pipe(pipe_serial);
7951 %}
7952
7953 instruct membar_release_lock() %{
7954 match(MemBarReleaseLock);
7955 ins_cost(VOLATILE_REF_COST);
7956
7957 format %{ "membar_release_lock (elided)" %}
7958
7959 ins_encode %{
7960 __ block_comment("membar_release_lock (elided)");
7961 %}
7962
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct unnecessary_membar_volatile() %{
7967 predicate(unnecessary_volatile(n));
7968 match(MemBarVolatile);
7969 ins_cost(0);
7970
7971 format %{ "membar_volatile (elided)" %}
7972
7973 ins_encode %{
7974 __ block_comment("membar_volatile (elided)");
7975 %}
7976
7977 ins_pipe(pipe_serial);
7978 %}
7979
7980 instruct membar_volatile() %{
7981 match(MemBarVolatile);
7982 ins_cost(VOLATILE_REF_COST*100);
7983
7984 format %{ "membar_volatile\n\t"
7985 "dmb ish"%}
7986
7987 ins_encode %{
7988 __ block_comment("membar_volatile");
7989 __ membar(Assembler::StoreLoad);
7990 %}
7991
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 // ============================================================================
7996 // Cast/Convert Instructions
7997
7998 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7999 match(Set dst (CastX2P src));
8000
8001 ins_cost(INSN_COST);
8002 format %{ "mov $dst, $src\t# long -> ptr" %}
8003
8004 ins_encode %{
8005 if ($dst$$reg != $src$$reg) {
8006 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8007 }
8008 %}
8009
8010 ins_pipe(ialu_reg);
8011 %}
8012
8013 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8014 match(Set dst (CastP2X src));
8015
8016 ins_cost(INSN_COST);
8017 format %{ "mov $dst, $src\t# ptr -> long" %}
8018
8019 ins_encode %{
8020 if ($dst$$reg != $src$$reg) {
8021 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8022 }
8023 %}
8024
8025 ins_pipe(ialu_reg);
8026 %}
8027
8028 // Convert oop into int for vectors alignment masking
8029 instruct convP2I(iRegINoSp dst, iRegP src) %{
8030 match(Set dst (ConvL2I (CastP2X src)));
8031
8032 ins_cost(INSN_COST);
8033 format %{ "movw $dst, $src\t# ptr -> int" %}
8034 ins_encode %{
8035 __ movw($dst$$Register, $src$$Register);
8036 %}
8037
8038 ins_pipe(ialu_reg);
8039 %}
8040
8041 // Convert compressed oop into int for vectors alignment masking
8042 // in case of 32bit oops (heap < 4Gb).
8043 instruct convN2I(iRegINoSp dst, iRegN src)
8044 %{
8045 predicate(CompressedOops::shift() == 0);
8046 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8047
8048 ins_cost(INSN_COST);
8049 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8050 ins_encode %{
8051 __ movw($dst$$Register, $src$$Register);
8052 %}
8053
8054 ins_pipe(ialu_reg);
8055 %}
8056
8057
8058 // Convert oop pointer into compressed form
8059 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8060 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8061 match(Set dst (EncodeP src));
8062 effect(KILL cr);
8063 ins_cost(INSN_COST * 3);
8064 format %{ "encode_heap_oop $dst, $src" %}
8065 ins_encode %{
8066 Register s = $src$$Register;
8067 Register d = $dst$$Register;
8068 __ encode_heap_oop(d, s);
8069 %}
8070 ins_pipe(ialu_reg);
8071 %}
8072
8073 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8074 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8075 match(Set dst (EncodeP src));
8076 ins_cost(INSN_COST * 3);
8077 format %{ "encode_heap_oop_not_null $dst, $src" %}
8078 ins_encode %{
8079 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8080 %}
8081 ins_pipe(ialu_reg);
8082 %}
8083
8084 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8085 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8086 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8087 match(Set dst (DecodeN src));
8088 ins_cost(INSN_COST * 3);
8089 format %{ "decode_heap_oop $dst, $src" %}
8090 ins_encode %{
8091 Register s = $src$$Register;
8092 Register d = $dst$$Register;
8093 __ decode_heap_oop(d, s);
8094 %}
8095 ins_pipe(ialu_reg);
8096 %}
8097
8098 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8099 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8100 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8101 match(Set dst (DecodeN src));
8102 ins_cost(INSN_COST * 3);
8103 format %{ "decode_heap_oop_not_null $dst, $src" %}
8104 ins_encode %{
8105 Register s = $src$$Register;
8106 Register d = $dst$$Register;
8107 __ decode_heap_oop_not_null(d, s);
8108 %}
8109 ins_pipe(ialu_reg);
8110 %}
8111
8112 // n.b. AArch64 implementations of encode_klass_not_null and
8113 // decode_klass_not_null do not modify the flags register so, unlike
8114 // Intel, we don't kill CR as a side effect here
8115
8116 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8117 match(Set dst (EncodePKlass src));
8118
8119 ins_cost(INSN_COST * 3);
8120 format %{ "encode_klass_not_null $dst,$src" %}
8121
8122 ins_encode %{
8123 Register src_reg = as_Register($src$$reg);
8124 Register dst_reg = as_Register($dst$$reg);
8125 __ encode_klass_not_null(dst_reg, src_reg);
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8132 match(Set dst (DecodeNKlass src));
8133
8134 ins_cost(INSN_COST * 3);
8135 format %{ "decode_klass_not_null $dst,$src" %}
8136
8137 ins_encode %{
8138 Register src_reg = as_Register($src$$reg);
8139 Register dst_reg = as_Register($dst$$reg);
8140 if (dst_reg != src_reg) {
8141 __ decode_klass_not_null(dst_reg, src_reg);
8142 } else {
8143 __ decode_klass_not_null(dst_reg);
8144 }
8145 %}
8146
8147 ins_pipe(ialu_reg);
8148 %}
8149
8150 instruct checkCastPP(iRegPNoSp dst)
8151 %{
8152 match(Set dst (CheckCastPP dst));
8153
8154 size(0);
8155 format %{ "# checkcastPP of $dst" %}
8156 ins_encode(/* empty encoding */);
8157 ins_pipe(pipe_class_empty);
8158 %}
8159
8160 instruct castPP(iRegPNoSp dst)
8161 %{
8162 match(Set dst (CastPP dst));
8163
8164 size(0);
8165 format %{ "# castPP of $dst" %}
8166 ins_encode(/* empty encoding */);
8167 ins_pipe(pipe_class_empty);
8168 %}
8169
8170 instruct castII(iRegI dst)
8171 %{
8172 match(Set dst (CastII dst));
8173
8174 size(0);
8175 format %{ "# castII of $dst" %}
8176 ins_encode(/* empty encoding */);
8177 ins_cost(0);
8178 ins_pipe(pipe_class_empty);
8179 %}
8180
8181 instruct castLL(iRegL dst)
8182 %{
8183 match(Set dst (CastLL dst));
8184
8185 size(0);
8186 format %{ "# castLL of $dst" %}
8187 ins_encode(/* empty encoding */);
8188 ins_cost(0);
8189 ins_pipe(pipe_class_empty);
8190 %}
8191
8192 instruct castFF(vRegF dst)
8193 %{
8194 match(Set dst (CastFF dst));
8195
8196 size(0);
8197 format %{ "# castFF of $dst" %}
8198 ins_encode(/* empty encoding */);
8199 ins_cost(0);
8200 ins_pipe(pipe_class_empty);
8201 %}
8202
8203 instruct castDD(vRegD dst)
8204 %{
8205 match(Set dst (CastDD dst));
8206
8207 size(0);
8208 format %{ "# castDD of $dst" %}
8209 ins_encode(/* empty encoding */);
8210 ins_cost(0);
8211 ins_pipe(pipe_class_empty);
8212 %}
8213
8214 instruct castVV(vReg dst)
8215 %{
8216 match(Set dst (CastVV dst));
8217
8218 size(0);
8219 format %{ "# castVV of $dst" %}
8220 ins_encode(/* empty encoding */);
8221 ins_cost(0);
8222 ins_pipe(pipe_class_empty);
8223 %}
8224
8225 instruct castVVMask(pRegGov dst)
8226 %{
8227 match(Set dst (CastVV dst));
8228
8229 size(0);
8230 format %{ "# castVV of $dst" %}
8231 ins_encode(/* empty encoding */);
8232 ins_cost(0);
8233 ins_pipe(pipe_class_empty);
8234 %}
8235
8236 // ============================================================================
8237 // Atomic operation instructions
8238 //
8239
8240 // standard CompareAndSwapX when we are using barriers
8241 // these have higher priority than the rules selected by a predicate
8242
8243 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8244 // can't match them
8245
8246 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8247
8248 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8249 ins_cost(2 * VOLATILE_REF_COST);
8250
8251 effect(KILL cr);
8252
8253 format %{
8254 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8255 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8256 %}
8257
8258 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8259 aarch64_enc_cset_eq(res));
8260
8261 ins_pipe(pipe_slow);
8262 %}
8263
8264 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8265
8266 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8267 ins_cost(2 * VOLATILE_REF_COST);
8268
8269 effect(KILL cr);
8270
8271 format %{
8272 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8273 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8274 %}
8275
8276 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8277 aarch64_enc_cset_eq(res));
8278
8279 ins_pipe(pipe_slow);
8280 %}
8281
8282 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8283
8284 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8285 ins_cost(2 * VOLATILE_REF_COST);
8286
8287 effect(KILL cr);
8288
8289 format %{
8290 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8291 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8292 %}
8293
8294 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8295 aarch64_enc_cset_eq(res));
8296
8297 ins_pipe(pipe_slow);
8298 %}
8299
8300 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8301
8302 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8303 ins_cost(2 * VOLATILE_REF_COST);
8304
8305 effect(KILL cr);
8306
8307 format %{
8308 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8309 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8310 %}
8311
8312 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8313 aarch64_enc_cset_eq(res));
8314
8315 ins_pipe(pipe_slow);
8316 %}
8317
8318 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8319
8320 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8321 predicate(n->as_LoadStore()->barrier_data() == 0);
8322 ins_cost(2 * VOLATILE_REF_COST);
8323
8324 effect(KILL cr);
8325
8326 format %{
8327 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8328 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8329 %}
8330
8331 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8332 aarch64_enc_cset_eq(res));
8333
8334 ins_pipe(pipe_slow);
8335 %}
8336
8337 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8338
8339 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8340 predicate(n->as_LoadStore()->barrier_data() == 0);
8341 ins_cost(2 * VOLATILE_REF_COST);
8342
8343 effect(KILL cr);
8344
8345 format %{
8346 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8347 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8348 %}
8349
8350 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8351 aarch64_enc_cset_eq(res));
8352
8353 ins_pipe(pipe_slow);
8354 %}
8355
8356 // alternative CompareAndSwapX when we are eliding barriers
8357
8358 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8359
8360 predicate(needs_acquiring_load_exclusive(n));
8361 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8362 ins_cost(VOLATILE_REF_COST);
8363
8364 effect(KILL cr);
8365
8366 format %{
8367 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8368 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8369 %}
8370
8371 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8372 aarch64_enc_cset_eq(res));
8373
8374 ins_pipe(pipe_slow);
8375 %}
8376
8377 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8378
8379 predicate(needs_acquiring_load_exclusive(n));
8380 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8381 ins_cost(VOLATILE_REF_COST);
8382
8383 effect(KILL cr);
8384
8385 format %{
8386 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8387 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8388 %}
8389
8390 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8391 aarch64_enc_cset_eq(res));
8392
8393 ins_pipe(pipe_slow);
8394 %}
8395
8396 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8397
8398 predicate(needs_acquiring_load_exclusive(n));
8399 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8400 ins_cost(VOLATILE_REF_COST);
8401
8402 effect(KILL cr);
8403
8404 format %{
8405 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8406 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8407 %}
8408
8409 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8410 aarch64_enc_cset_eq(res));
8411
8412 ins_pipe(pipe_slow);
8413 %}
8414
8415 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8416
8417 predicate(needs_acquiring_load_exclusive(n));
8418 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8419 ins_cost(VOLATILE_REF_COST);
8420
8421 effect(KILL cr);
8422
8423 format %{
8424 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8425 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8426 %}
8427
8428 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8429 aarch64_enc_cset_eq(res));
8430
8431 ins_pipe(pipe_slow);
8432 %}
8433
8434 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8435
8436 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8437 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8438 ins_cost(VOLATILE_REF_COST);
8439
8440 effect(KILL cr);
8441
8442 format %{
8443 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8444 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8445 %}
8446
8447 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8448 aarch64_enc_cset_eq(res));
8449
8450 ins_pipe(pipe_slow);
8451 %}
8452
8453 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8454
8455 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8456 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8457 ins_cost(VOLATILE_REF_COST);
8458
8459 effect(KILL cr);
8460
8461 format %{
8462 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8463 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8464 %}
8465
8466 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8467 aarch64_enc_cset_eq(res));
8468
8469 ins_pipe(pipe_slow);
8470 %}
8471
8472
8473 // ---------------------------------------------------------------------
8474
8475 // BEGIN This section of the file is automatically generated. Do not edit --------------
8476
8477 // Sundry CAS operations. Note that release is always true,
8478 // regardless of the memory ordering of the CAS. This is because we
8479 // need the volatile case to be sequentially consistent but there is
8480 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8481 // can't check the type of memory ordering here, so we always emit a
8482 // STLXR.
8483
8484 // This section is generated from cas.m4
8485
8486
8487 // This pattern is generated automatically from cas.m4.
8488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8489 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8490 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8491 ins_cost(2 * VOLATILE_REF_COST);
8492 effect(TEMP_DEF res, KILL cr);
8493 format %{
8494 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8495 %}
8496 ins_encode %{
8497 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8498 Assembler::byte, /*acquire*/ false, /*release*/ true,
8499 /*weak*/ false, $res$$Register);
8500 __ sxtbw($res$$Register, $res$$Register);
8501 %}
8502 ins_pipe(pipe_slow);
8503 %}
8504
8505 // This pattern is generated automatically from cas.m4.
8506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8507 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8508 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8509 ins_cost(2 * VOLATILE_REF_COST);
8510 effect(TEMP_DEF res, KILL cr);
8511 format %{
8512 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8513 %}
8514 ins_encode %{
8515 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8516 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8517 /*weak*/ false, $res$$Register);
8518 __ sxthw($res$$Register, $res$$Register);
8519 %}
8520 ins_pipe(pipe_slow);
8521 %}
8522
8523 // This pattern is generated automatically from cas.m4.
8524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8525 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8526 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8527 ins_cost(2 * VOLATILE_REF_COST);
8528 effect(TEMP_DEF res, KILL cr);
8529 format %{
8530 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8531 %}
8532 ins_encode %{
8533 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8534 Assembler::word, /*acquire*/ false, /*release*/ true,
8535 /*weak*/ false, $res$$Register);
8536 %}
8537 ins_pipe(pipe_slow);
8538 %}
8539
8540 // This pattern is generated automatically from cas.m4.
8541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8542 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8543 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8544 ins_cost(2 * VOLATILE_REF_COST);
8545 effect(TEMP_DEF res, KILL cr);
8546 format %{
8547 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8548 %}
8549 ins_encode %{
8550 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8551 Assembler::xword, /*acquire*/ false, /*release*/ true,
8552 /*weak*/ false, $res$$Register);
8553 %}
8554 ins_pipe(pipe_slow);
8555 %}
8556
8557 // This pattern is generated automatically from cas.m4.
8558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8559 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8560 predicate(n->as_LoadStore()->barrier_data() == 0);
8561 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8562 ins_cost(2 * VOLATILE_REF_COST);
8563 effect(TEMP_DEF res, KILL cr);
8564 format %{
8565 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8566 %}
8567 ins_encode %{
8568 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8569 Assembler::word, /*acquire*/ false, /*release*/ true,
8570 /*weak*/ false, $res$$Register);
8571 %}
8572 ins_pipe(pipe_slow);
8573 %}
8574
8575 // This pattern is generated automatically from cas.m4.
8576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8577 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8578 predicate(n->as_LoadStore()->barrier_data() == 0);
8579 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8580 ins_cost(2 * VOLATILE_REF_COST);
8581 effect(TEMP_DEF res, KILL cr);
8582 format %{
8583 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8584 %}
8585 ins_encode %{
8586 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8587 Assembler::xword, /*acquire*/ false, /*release*/ true,
8588 /*weak*/ false, $res$$Register);
8589 %}
8590 ins_pipe(pipe_slow);
8591 %}
8592
8593 // This pattern is generated automatically from cas.m4.
8594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8595 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8596 predicate(needs_acquiring_load_exclusive(n));
8597 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8598 ins_cost(VOLATILE_REF_COST);
8599 effect(TEMP_DEF res, KILL cr);
8600 format %{
8601 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8602 %}
8603 ins_encode %{
8604 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8605 Assembler::byte, /*acquire*/ true, /*release*/ true,
8606 /*weak*/ false, $res$$Register);
8607 __ sxtbw($res$$Register, $res$$Register);
8608 %}
8609 ins_pipe(pipe_slow);
8610 %}
8611
8612 // This pattern is generated automatically from cas.m4.
8613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8614 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8615 predicate(needs_acquiring_load_exclusive(n));
8616 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8617 ins_cost(VOLATILE_REF_COST);
8618 effect(TEMP_DEF res, KILL cr);
8619 format %{
8620 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8621 %}
8622 ins_encode %{
8623 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8624 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8625 /*weak*/ false, $res$$Register);
8626 __ sxthw($res$$Register, $res$$Register);
8627 %}
8628 ins_pipe(pipe_slow);
8629 %}
8630
8631 // This pattern is generated automatically from cas.m4.
8632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8633 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8634 predicate(needs_acquiring_load_exclusive(n));
8635 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8636 ins_cost(VOLATILE_REF_COST);
8637 effect(TEMP_DEF res, KILL cr);
8638 format %{
8639 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8640 %}
8641 ins_encode %{
8642 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8643 Assembler::word, /*acquire*/ true, /*release*/ true,
8644 /*weak*/ false, $res$$Register);
8645 %}
8646 ins_pipe(pipe_slow);
8647 %}
8648
8649 // This pattern is generated automatically from cas.m4.
8650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8651 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8652 predicate(needs_acquiring_load_exclusive(n));
8653 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8654 ins_cost(VOLATILE_REF_COST);
8655 effect(TEMP_DEF res, KILL cr);
8656 format %{
8657 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8658 %}
8659 ins_encode %{
8660 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8661 Assembler::xword, /*acquire*/ true, /*release*/ true,
8662 /*weak*/ false, $res$$Register);
8663 %}
8664 ins_pipe(pipe_slow);
8665 %}
8666
8667 // This pattern is generated automatically from cas.m4.
8668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8669 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8670 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8671 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8672 ins_cost(VOLATILE_REF_COST);
8673 effect(TEMP_DEF res, KILL cr);
8674 format %{
8675 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8676 %}
8677 ins_encode %{
8678 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8679 Assembler::word, /*acquire*/ true, /*release*/ true,
8680 /*weak*/ false, $res$$Register);
8681 %}
8682 ins_pipe(pipe_slow);
8683 %}
8684
8685 // This pattern is generated automatically from cas.m4.
8686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8687 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8688 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8689 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8690 ins_cost(VOLATILE_REF_COST);
8691 effect(TEMP_DEF res, KILL cr);
8692 format %{
8693 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8694 %}
8695 ins_encode %{
8696 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8697 Assembler::xword, /*acquire*/ true, /*release*/ true,
8698 /*weak*/ false, $res$$Register);
8699 %}
8700 ins_pipe(pipe_slow);
8701 %}
8702
8703 // This pattern is generated automatically from cas.m4.
8704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8705 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8706 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8707 ins_cost(2 * VOLATILE_REF_COST);
8708 effect(KILL cr);
8709 format %{
8710 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8711 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8712 %}
8713 ins_encode %{
8714 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8715 Assembler::byte, /*acquire*/ false, /*release*/ true,
8716 /*weak*/ true, noreg);
8717 __ csetw($res$$Register, Assembler::EQ);
8718 %}
8719 ins_pipe(pipe_slow);
8720 %}
8721
8722 // This pattern is generated automatically from cas.m4.
8723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8724 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8725 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8726 ins_cost(2 * VOLATILE_REF_COST);
8727 effect(KILL cr);
8728 format %{
8729 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8730 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8731 %}
8732 ins_encode %{
8733 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8734 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8735 /*weak*/ true, noreg);
8736 __ csetw($res$$Register, Assembler::EQ);
8737 %}
8738 ins_pipe(pipe_slow);
8739 %}
8740
8741 // This pattern is generated automatically from cas.m4.
8742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8743 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8744 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8745 ins_cost(2 * VOLATILE_REF_COST);
8746 effect(KILL cr);
8747 format %{
8748 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8749 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8750 %}
8751 ins_encode %{
8752 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8753 Assembler::word, /*acquire*/ false, /*release*/ true,
8754 /*weak*/ true, noreg);
8755 __ csetw($res$$Register, Assembler::EQ);
8756 %}
8757 ins_pipe(pipe_slow);
8758 %}
8759
8760 // This pattern is generated automatically from cas.m4.
8761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8762 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8763 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8764 ins_cost(2 * VOLATILE_REF_COST);
8765 effect(KILL cr);
8766 format %{
8767 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8768 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8769 %}
8770 ins_encode %{
8771 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8772 Assembler::xword, /*acquire*/ false, /*release*/ true,
8773 /*weak*/ true, noreg);
8774 __ csetw($res$$Register, Assembler::EQ);
8775 %}
8776 ins_pipe(pipe_slow);
8777 %}
8778
8779 // This pattern is generated automatically from cas.m4.
8780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8781 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8782 predicate(n->as_LoadStore()->barrier_data() == 0);
8783 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8784 ins_cost(2 * VOLATILE_REF_COST);
8785 effect(KILL cr);
8786 format %{
8787 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8788 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8789 %}
8790 ins_encode %{
8791 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8792 Assembler::word, /*acquire*/ false, /*release*/ true,
8793 /*weak*/ true, noreg);
8794 __ csetw($res$$Register, Assembler::EQ);
8795 %}
8796 ins_pipe(pipe_slow);
8797 %}
8798
8799 // This pattern is generated automatically from cas.m4.
8800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8801 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8802 predicate(n->as_LoadStore()->barrier_data() == 0);
8803 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8804 ins_cost(2 * VOLATILE_REF_COST);
8805 effect(KILL cr);
8806 format %{
8807 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8808 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8809 %}
8810 ins_encode %{
8811 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8812 Assembler::xword, /*acquire*/ false, /*release*/ true,
8813 /*weak*/ true, noreg);
8814 __ csetw($res$$Register, Assembler::EQ);
8815 %}
8816 ins_pipe(pipe_slow);
8817 %}
8818
8819 // This pattern is generated automatically from cas.m4.
8820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8821 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8822 predicate(needs_acquiring_load_exclusive(n));
8823 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8824 ins_cost(VOLATILE_REF_COST);
8825 effect(KILL cr);
8826 format %{
8827 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8828 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8829 %}
8830 ins_encode %{
8831 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8832 Assembler::byte, /*acquire*/ true, /*release*/ true,
8833 /*weak*/ true, noreg);
8834 __ csetw($res$$Register, Assembler::EQ);
8835 %}
8836 ins_pipe(pipe_slow);
8837 %}
8838
8839 // This pattern is generated automatically from cas.m4.
8840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8841 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8842 predicate(needs_acquiring_load_exclusive(n));
8843 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8844 ins_cost(VOLATILE_REF_COST);
8845 effect(KILL cr);
8846 format %{
8847 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8848 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8849 %}
8850 ins_encode %{
8851 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8852 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8853 /*weak*/ true, noreg);
8854 __ csetw($res$$Register, Assembler::EQ);
8855 %}
8856 ins_pipe(pipe_slow);
8857 %}
8858
8859 // This pattern is generated automatically from cas.m4.
8860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8861 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8862 predicate(needs_acquiring_load_exclusive(n));
8863 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8864 ins_cost(VOLATILE_REF_COST);
8865 effect(KILL cr);
8866 format %{
8867 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8868 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8869 %}
8870 ins_encode %{
8871 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8872 Assembler::word, /*acquire*/ true, /*release*/ true,
8873 /*weak*/ true, noreg);
8874 __ csetw($res$$Register, Assembler::EQ);
8875 %}
8876 ins_pipe(pipe_slow);
8877 %}
8878
8879 // This pattern is generated automatically from cas.m4.
8880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8881 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8882 predicate(needs_acquiring_load_exclusive(n));
8883 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8884 ins_cost(VOLATILE_REF_COST);
8885 effect(KILL cr);
8886 format %{
8887 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8888 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8889 %}
8890 ins_encode %{
8891 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8892 Assembler::xword, /*acquire*/ true, /*release*/ true,
8893 /*weak*/ true, noreg);
8894 __ csetw($res$$Register, Assembler::EQ);
8895 %}
8896 ins_pipe(pipe_slow);
8897 %}
8898
8899 // This pattern is generated automatically from cas.m4.
8900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8901 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8902 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8903 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8904 ins_cost(VOLATILE_REF_COST);
8905 effect(KILL cr);
8906 format %{
8907 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8908 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8909 %}
8910 ins_encode %{
8911 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8912 Assembler::word, /*acquire*/ true, /*release*/ true,
8913 /*weak*/ true, noreg);
8914 __ csetw($res$$Register, Assembler::EQ);
8915 %}
8916 ins_pipe(pipe_slow);
8917 %}
8918
8919 // This pattern is generated automatically from cas.m4.
8920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8921 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8922 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8923 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8924 ins_cost(VOLATILE_REF_COST);
8925 effect(KILL cr);
8926 format %{
8927 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8928 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8929 %}
8930 ins_encode %{
8931 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8932 Assembler::xword, /*acquire*/ true, /*release*/ true,
8933 /*weak*/ true, noreg);
8934 __ csetw($res$$Register, Assembler::EQ);
8935 %}
8936 ins_pipe(pipe_slow);
8937 %}
8938
8939 // END This section of the file is automatically generated. Do not edit --------------
8940 // ---------------------------------------------------------------------
8941
8942 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8943 match(Set prev (GetAndSetI mem newv));
8944 ins_cost(2 * VOLATILE_REF_COST);
8945 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8946 ins_encode %{
8947 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8948 %}
8949 ins_pipe(pipe_serial);
8950 %}
8951
8952 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8953 match(Set prev (GetAndSetL mem newv));
8954 ins_cost(2 * VOLATILE_REF_COST);
8955 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8956 ins_encode %{
8957 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8958 %}
8959 ins_pipe(pipe_serial);
8960 %}
8961
8962 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8963 predicate(n->as_LoadStore()->barrier_data() == 0);
8964 match(Set prev (GetAndSetN mem newv));
8965 ins_cost(2 * VOLATILE_REF_COST);
8966 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8967 ins_encode %{
8968 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8969 %}
8970 ins_pipe(pipe_serial);
8971 %}
8972
8973 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8974 predicate(n->as_LoadStore()->barrier_data() == 0);
8975 match(Set prev (GetAndSetP mem newv));
8976 ins_cost(2 * VOLATILE_REF_COST);
8977 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8978 ins_encode %{
8979 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8980 %}
8981 ins_pipe(pipe_serial);
8982 %}
8983
8984 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8985 predicate(needs_acquiring_load_exclusive(n));
8986 match(Set prev (GetAndSetI mem newv));
8987 ins_cost(VOLATILE_REF_COST);
8988 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8989 ins_encode %{
8990 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8991 %}
8992 ins_pipe(pipe_serial);
8993 %}
8994
8995 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8996 predicate(needs_acquiring_load_exclusive(n));
8997 match(Set prev (GetAndSetL mem newv));
8998 ins_cost(VOLATILE_REF_COST);
8999 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9000 ins_encode %{
9001 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9002 %}
9003 ins_pipe(pipe_serial);
9004 %}
9005
9006 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9007 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9008 match(Set prev (GetAndSetN mem newv));
9009 ins_cost(VOLATILE_REF_COST);
9010 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9011 ins_encode %{
9012 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9013 %}
9014 ins_pipe(pipe_serial);
9015 %}
9016
9017 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9018 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9019 match(Set prev (GetAndSetP mem newv));
9020 ins_cost(VOLATILE_REF_COST);
9021 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9022 ins_encode %{
9023 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9024 %}
9025 ins_pipe(pipe_serial);
9026 %}
9027
9028
9029 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9030 match(Set newval (GetAndAddL mem incr));
9031 ins_cost(2 * VOLATILE_REF_COST + 1);
9032 format %{ "get_and_addL $newval, [$mem], $incr" %}
9033 ins_encode %{
9034 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9035 %}
9036 ins_pipe(pipe_serial);
9037 %}
9038
9039 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9040 predicate(n->as_LoadStore()->result_not_used());
9041 match(Set dummy (GetAndAddL mem incr));
9042 ins_cost(2 * VOLATILE_REF_COST);
9043 format %{ "get_and_addL [$mem], $incr" %}
9044 ins_encode %{
9045 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9046 %}
9047 ins_pipe(pipe_serial);
9048 %}
9049
9050 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9051 match(Set newval (GetAndAddL mem incr));
9052 ins_cost(2 * VOLATILE_REF_COST + 1);
9053 format %{ "get_and_addL $newval, [$mem], $incr" %}
9054 ins_encode %{
9055 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9056 %}
9057 ins_pipe(pipe_serial);
9058 %}
9059
9060 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9061 predicate(n->as_LoadStore()->result_not_used());
9062 match(Set dummy (GetAndAddL mem incr));
9063 ins_cost(2 * VOLATILE_REF_COST);
9064 format %{ "get_and_addL [$mem], $incr" %}
9065 ins_encode %{
9066 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9067 %}
9068 ins_pipe(pipe_serial);
9069 %}
9070
9071 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9072 match(Set newval (GetAndAddI mem incr));
9073 ins_cost(2 * VOLATILE_REF_COST + 1);
9074 format %{ "get_and_addI $newval, [$mem], $incr" %}
9075 ins_encode %{
9076 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9077 %}
9078 ins_pipe(pipe_serial);
9079 %}
9080
9081 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9082 predicate(n->as_LoadStore()->result_not_used());
9083 match(Set dummy (GetAndAddI mem incr));
9084 ins_cost(2 * VOLATILE_REF_COST);
9085 format %{ "get_and_addI [$mem], $incr" %}
9086 ins_encode %{
9087 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9088 %}
9089 ins_pipe(pipe_serial);
9090 %}
9091
9092 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9093 match(Set newval (GetAndAddI mem incr));
9094 ins_cost(2 * VOLATILE_REF_COST + 1);
9095 format %{ "get_and_addI $newval, [$mem], $incr" %}
9096 ins_encode %{
9097 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9098 %}
9099 ins_pipe(pipe_serial);
9100 %}
9101
9102 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9103 predicate(n->as_LoadStore()->result_not_used());
9104 match(Set dummy (GetAndAddI mem incr));
9105 ins_cost(2 * VOLATILE_REF_COST);
9106 format %{ "get_and_addI [$mem], $incr" %}
9107 ins_encode %{
9108 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9109 %}
9110 ins_pipe(pipe_serial);
9111 %}
9112
9113 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9114 predicate(needs_acquiring_load_exclusive(n));
9115 match(Set newval (GetAndAddL mem incr));
9116 ins_cost(VOLATILE_REF_COST + 1);
9117 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9118 ins_encode %{
9119 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9120 %}
9121 ins_pipe(pipe_serial);
9122 %}
9123
9124 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9125 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9126 match(Set dummy (GetAndAddL mem incr));
9127 ins_cost(VOLATILE_REF_COST);
9128 format %{ "get_and_addL_acq [$mem], $incr" %}
9129 ins_encode %{
9130 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9131 %}
9132 ins_pipe(pipe_serial);
9133 %}
9134
9135 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9136 predicate(needs_acquiring_load_exclusive(n));
9137 match(Set newval (GetAndAddL mem incr));
9138 ins_cost(VOLATILE_REF_COST + 1);
9139 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9140 ins_encode %{
9141 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9142 %}
9143 ins_pipe(pipe_serial);
9144 %}
9145
9146 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9147 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9148 match(Set dummy (GetAndAddL mem incr));
9149 ins_cost(VOLATILE_REF_COST);
9150 format %{ "get_and_addL_acq [$mem], $incr" %}
9151 ins_encode %{
9152 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9153 %}
9154 ins_pipe(pipe_serial);
9155 %}
9156
9157 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9158 predicate(needs_acquiring_load_exclusive(n));
9159 match(Set newval (GetAndAddI mem incr));
9160 ins_cost(VOLATILE_REF_COST + 1);
9161 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9162 ins_encode %{
9163 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9164 %}
9165 ins_pipe(pipe_serial);
9166 %}
9167
9168 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9169 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9170 match(Set dummy (GetAndAddI mem incr));
9171 ins_cost(VOLATILE_REF_COST);
9172 format %{ "get_and_addI_acq [$mem], $incr" %}
9173 ins_encode %{
9174 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9175 %}
9176 ins_pipe(pipe_serial);
9177 %}
9178
9179 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9180 predicate(needs_acquiring_load_exclusive(n));
9181 match(Set newval (GetAndAddI mem incr));
9182 ins_cost(VOLATILE_REF_COST + 1);
9183 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9184 ins_encode %{
9185 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9186 %}
9187 ins_pipe(pipe_serial);
9188 %}
9189
9190 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9191 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9192 match(Set dummy (GetAndAddI mem incr));
9193 ins_cost(VOLATILE_REF_COST);
9194 format %{ "get_and_addI_acq [$mem], $incr" %}
9195 ins_encode %{
9196 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9197 %}
9198 ins_pipe(pipe_serial);
9199 %}
9200
9201 // Manifest a CmpU result in an integer register.
9202 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9203 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9204 %{
9205 match(Set dst (CmpU3 src1 src2));
9206 effect(KILL flags);
9207
9208 ins_cost(INSN_COST * 3);
9209 format %{
9210 "cmpw $src1, $src2\n\t"
9211 "csetw $dst, ne\n\t"
9212 "cnegw $dst, lo\t# CmpU3(reg)"
9213 %}
9214 ins_encode %{
9215 __ cmpw($src1$$Register, $src2$$Register);
9216 __ csetw($dst$$Register, Assembler::NE);
9217 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9218 %}
9219
9220 ins_pipe(pipe_class_default);
9221 %}
9222
9223 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9224 %{
9225 match(Set dst (CmpU3 src1 src2));
9226 effect(KILL flags);
9227
9228 ins_cost(INSN_COST * 3);
9229 format %{
9230 "subsw zr, $src1, $src2\n\t"
9231 "csetw $dst, ne\n\t"
9232 "cnegw $dst, lo\t# CmpU3(imm)"
9233 %}
9234 ins_encode %{
9235 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9236 __ csetw($dst$$Register, Assembler::NE);
9237 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9238 %}
9239
9240 ins_pipe(pipe_class_default);
9241 %}
9242
9243 // Manifest a CmpUL result in an integer register.
9244 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9245 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9246 %{
9247 match(Set dst (CmpUL3 src1 src2));
9248 effect(KILL flags);
9249
9250 ins_cost(INSN_COST * 3);
9251 format %{
9252 "cmp $src1, $src2\n\t"
9253 "csetw $dst, ne\n\t"
9254 "cnegw $dst, lo\t# CmpUL3(reg)"
9255 %}
9256 ins_encode %{
9257 __ cmp($src1$$Register, $src2$$Register);
9258 __ csetw($dst$$Register, Assembler::NE);
9259 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9260 %}
9261
9262 ins_pipe(pipe_class_default);
9263 %}
9264
9265 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9266 %{
9267 match(Set dst (CmpUL3 src1 src2));
9268 effect(KILL flags);
9269
9270 ins_cost(INSN_COST * 3);
9271 format %{
9272 "subs zr, $src1, $src2\n\t"
9273 "csetw $dst, ne\n\t"
9274 "cnegw $dst, lo\t# CmpUL3(imm)"
9275 %}
9276 ins_encode %{
9277 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9278 __ csetw($dst$$Register, Assembler::NE);
9279 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9280 %}
9281
9282 ins_pipe(pipe_class_default);
9283 %}
9284
9285 // Manifest a CmpL result in an integer register.
9286 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9287 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9288 %{
9289 match(Set dst (CmpL3 src1 src2));
9290 effect(KILL flags);
9291
9292 ins_cost(INSN_COST * 3);
9293 format %{
9294 "cmp $src1, $src2\n\t"
9295 "csetw $dst, ne\n\t"
9296 "cnegw $dst, lt\t# CmpL3(reg)"
9297 %}
9298 ins_encode %{
9299 __ cmp($src1$$Register, $src2$$Register);
9300 __ csetw($dst$$Register, Assembler::NE);
9301 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9302 %}
9303
9304 ins_pipe(pipe_class_default);
9305 %}
9306
9307 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9308 %{
9309 match(Set dst (CmpL3 src1 src2));
9310 effect(KILL flags);
9311
9312 ins_cost(INSN_COST * 3);
9313 format %{
9314 "subs zr, $src1, $src2\n\t"
9315 "csetw $dst, ne\n\t"
9316 "cnegw $dst, lt\t# CmpL3(imm)"
9317 %}
9318 ins_encode %{
9319 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9320 __ csetw($dst$$Register, Assembler::NE);
9321 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9322 %}
9323
9324 ins_pipe(pipe_class_default);
9325 %}
9326
9327 // ============================================================================
9328 // Conditional Move Instructions
9329
9330 // n.b. we have identical rules for both a signed compare op (cmpOp)
9331 // and an unsigned compare op (cmpOpU). it would be nice if we could
9332 // define an op class which merged both inputs and use it to type the
9333 // argument to a single rule. unfortunatelyt his fails because the
9334 // opclass does not live up to the COND_INTER interface of its
9335 // component operands. When the generic code tries to negate the
9336 // operand it ends up running the generci Machoper::negate method
9337 // which throws a ShouldNotHappen. So, we have to provide two flavours
9338 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9339
9340 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9341 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9342
9343 ins_cost(INSN_COST * 2);
9344 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9345
9346 ins_encode %{
9347 __ cselw(as_Register($dst$$reg),
9348 as_Register($src2$$reg),
9349 as_Register($src1$$reg),
9350 (Assembler::Condition)$cmp$$cmpcode);
9351 %}
9352
9353 ins_pipe(icond_reg_reg);
9354 %}
9355
9356 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9357 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9358
9359 ins_cost(INSN_COST * 2);
9360 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9361
9362 ins_encode %{
9363 __ cselw(as_Register($dst$$reg),
9364 as_Register($src2$$reg),
9365 as_Register($src1$$reg),
9366 (Assembler::Condition)$cmp$$cmpcode);
9367 %}
9368
9369 ins_pipe(icond_reg_reg);
9370 %}
9371
9372 // special cases where one arg is zero
9373
9374 // n.b. this is selected in preference to the rule above because it
9375 // avoids loading constant 0 into a source register
9376
9377 // TODO
9378 // we ought only to be able to cull one of these variants as the ideal
9379 // transforms ought always to order the zero consistently (to left/right?)
9380
9381 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9382 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9383
9384 ins_cost(INSN_COST * 2);
9385 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9386
9387 ins_encode %{
9388 __ cselw(as_Register($dst$$reg),
9389 as_Register($src$$reg),
9390 zr,
9391 (Assembler::Condition)$cmp$$cmpcode);
9392 %}
9393
9394 ins_pipe(icond_reg);
9395 %}
9396
9397 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9398 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9399
9400 ins_cost(INSN_COST * 2);
9401 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9402
9403 ins_encode %{
9404 __ cselw(as_Register($dst$$reg),
9405 as_Register($src$$reg),
9406 zr,
9407 (Assembler::Condition)$cmp$$cmpcode);
9408 %}
9409
9410 ins_pipe(icond_reg);
9411 %}
9412
9413 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9414 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9415
9416 ins_cost(INSN_COST * 2);
9417 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9418
9419 ins_encode %{
9420 __ cselw(as_Register($dst$$reg),
9421 zr,
9422 as_Register($src$$reg),
9423 (Assembler::Condition)$cmp$$cmpcode);
9424 %}
9425
9426 ins_pipe(icond_reg);
9427 %}
9428
9429 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9430 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9431
9432 ins_cost(INSN_COST * 2);
9433 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9434
9435 ins_encode %{
9436 __ cselw(as_Register($dst$$reg),
9437 zr,
9438 as_Register($src$$reg),
9439 (Assembler::Condition)$cmp$$cmpcode);
9440 %}
9441
9442 ins_pipe(icond_reg);
9443 %}
9444
9445 // special case for creating a boolean 0 or 1
9446
9447 // n.b. this is selected in preference to the rule above because it
9448 // avoids loading constants 0 and 1 into a source register
9449
9450 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9451 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9452
9453 ins_cost(INSN_COST * 2);
9454 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9455
9456 ins_encode %{
9457 // equivalently
9458 // cset(as_Register($dst$$reg),
9459 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9460 __ csincw(as_Register($dst$$reg),
9461 zr,
9462 zr,
9463 (Assembler::Condition)$cmp$$cmpcode);
9464 %}
9465
9466 ins_pipe(icond_none);
9467 %}
9468
9469 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9470 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9471
9472 ins_cost(INSN_COST * 2);
9473 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9474
9475 ins_encode %{
9476 // equivalently
9477 // cset(as_Register($dst$$reg),
9478 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9479 __ csincw(as_Register($dst$$reg),
9480 zr,
9481 zr,
9482 (Assembler::Condition)$cmp$$cmpcode);
9483 %}
9484
9485 ins_pipe(icond_none);
9486 %}
9487
9488 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9489 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9490
9491 ins_cost(INSN_COST * 2);
9492 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9493
9494 ins_encode %{
9495 __ csel(as_Register($dst$$reg),
9496 as_Register($src2$$reg),
9497 as_Register($src1$$reg),
9498 (Assembler::Condition)$cmp$$cmpcode);
9499 %}
9500
9501 ins_pipe(icond_reg_reg);
9502 %}
9503
9504 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9505 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9506
9507 ins_cost(INSN_COST * 2);
9508 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9509
9510 ins_encode %{
9511 __ csel(as_Register($dst$$reg),
9512 as_Register($src2$$reg),
9513 as_Register($src1$$reg),
9514 (Assembler::Condition)$cmp$$cmpcode);
9515 %}
9516
9517 ins_pipe(icond_reg_reg);
9518 %}
9519
9520 // special cases where one arg is zero
9521
9522 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9523 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9524
9525 ins_cost(INSN_COST * 2);
9526 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9527
9528 ins_encode %{
9529 __ csel(as_Register($dst$$reg),
9530 zr,
9531 as_Register($src$$reg),
9532 (Assembler::Condition)$cmp$$cmpcode);
9533 %}
9534
9535 ins_pipe(icond_reg);
9536 %}
9537
9538 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9539 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9540
9541 ins_cost(INSN_COST * 2);
9542 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9543
9544 ins_encode %{
9545 __ csel(as_Register($dst$$reg),
9546 zr,
9547 as_Register($src$$reg),
9548 (Assembler::Condition)$cmp$$cmpcode);
9549 %}
9550
9551 ins_pipe(icond_reg);
9552 %}
9553
9554 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9555 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9556
9557 ins_cost(INSN_COST * 2);
9558 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9559
9560 ins_encode %{
9561 __ csel(as_Register($dst$$reg),
9562 as_Register($src$$reg),
9563 zr,
9564 (Assembler::Condition)$cmp$$cmpcode);
9565 %}
9566
9567 ins_pipe(icond_reg);
9568 %}
9569
9570 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9571 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9572
9573 ins_cost(INSN_COST * 2);
9574 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9575
9576 ins_encode %{
9577 __ csel(as_Register($dst$$reg),
9578 as_Register($src$$reg),
9579 zr,
9580 (Assembler::Condition)$cmp$$cmpcode);
9581 %}
9582
9583 ins_pipe(icond_reg);
9584 %}
9585
9586 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9587 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9588
9589 ins_cost(INSN_COST * 2);
9590 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9591
9592 ins_encode %{
9593 __ csel(as_Register($dst$$reg),
9594 as_Register($src2$$reg),
9595 as_Register($src1$$reg),
9596 (Assembler::Condition)$cmp$$cmpcode);
9597 %}
9598
9599 ins_pipe(icond_reg_reg);
9600 %}
9601
9602 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9603 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9604
9605 ins_cost(INSN_COST * 2);
9606 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9607
9608 ins_encode %{
9609 __ csel(as_Register($dst$$reg),
9610 as_Register($src2$$reg),
9611 as_Register($src1$$reg),
9612 (Assembler::Condition)$cmp$$cmpcode);
9613 %}
9614
9615 ins_pipe(icond_reg_reg);
9616 %}
9617
9618 // special cases where one arg is zero
9619
9620 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9621 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9622
9623 ins_cost(INSN_COST * 2);
9624 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9625
9626 ins_encode %{
9627 __ csel(as_Register($dst$$reg),
9628 zr,
9629 as_Register($src$$reg),
9630 (Assembler::Condition)$cmp$$cmpcode);
9631 %}
9632
9633 ins_pipe(icond_reg);
9634 %}
9635
9636 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9637 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9638
9639 ins_cost(INSN_COST * 2);
9640 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9641
9642 ins_encode %{
9643 __ csel(as_Register($dst$$reg),
9644 zr,
9645 as_Register($src$$reg),
9646 (Assembler::Condition)$cmp$$cmpcode);
9647 %}
9648
9649 ins_pipe(icond_reg);
9650 %}
9651
9652 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9653 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9654
9655 ins_cost(INSN_COST * 2);
9656 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9657
9658 ins_encode %{
9659 __ csel(as_Register($dst$$reg),
9660 as_Register($src$$reg),
9661 zr,
9662 (Assembler::Condition)$cmp$$cmpcode);
9663 %}
9664
9665 ins_pipe(icond_reg);
9666 %}
9667
9668 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9669 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9670
9671 ins_cost(INSN_COST * 2);
9672 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9673
9674 ins_encode %{
9675 __ csel(as_Register($dst$$reg),
9676 as_Register($src$$reg),
9677 zr,
9678 (Assembler::Condition)$cmp$$cmpcode);
9679 %}
9680
9681 ins_pipe(icond_reg);
9682 %}
9683
9684 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9685 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9686
9687 ins_cost(INSN_COST * 2);
9688 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9689
9690 ins_encode %{
9691 __ cselw(as_Register($dst$$reg),
9692 as_Register($src2$$reg),
9693 as_Register($src1$$reg),
9694 (Assembler::Condition)$cmp$$cmpcode);
9695 %}
9696
9697 ins_pipe(icond_reg_reg);
9698 %}
9699
9700 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9701 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9702
9703 ins_cost(INSN_COST * 2);
9704 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9705
9706 ins_encode %{
9707 __ cselw(as_Register($dst$$reg),
9708 as_Register($src2$$reg),
9709 as_Register($src1$$reg),
9710 (Assembler::Condition)$cmp$$cmpcode);
9711 %}
9712
9713 ins_pipe(icond_reg_reg);
9714 %}
9715
9716 // special cases where one arg is zero
9717
9718 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9719 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9720
9721 ins_cost(INSN_COST * 2);
9722 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9723
9724 ins_encode %{
9725 __ cselw(as_Register($dst$$reg),
9726 zr,
9727 as_Register($src$$reg),
9728 (Assembler::Condition)$cmp$$cmpcode);
9729 %}
9730
9731 ins_pipe(icond_reg);
9732 %}
9733
9734 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9735 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9736
9737 ins_cost(INSN_COST * 2);
9738 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9739
9740 ins_encode %{
9741 __ cselw(as_Register($dst$$reg),
9742 zr,
9743 as_Register($src$$reg),
9744 (Assembler::Condition)$cmp$$cmpcode);
9745 %}
9746
9747 ins_pipe(icond_reg);
9748 %}
9749
9750 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9751 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9752
9753 ins_cost(INSN_COST * 2);
9754 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9755
9756 ins_encode %{
9757 __ cselw(as_Register($dst$$reg),
9758 as_Register($src$$reg),
9759 zr,
9760 (Assembler::Condition)$cmp$$cmpcode);
9761 %}
9762
9763 ins_pipe(icond_reg);
9764 %}
9765
9766 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9767 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9768
9769 ins_cost(INSN_COST * 2);
9770 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9771
9772 ins_encode %{
9773 __ cselw(as_Register($dst$$reg),
9774 as_Register($src$$reg),
9775 zr,
9776 (Assembler::Condition)$cmp$$cmpcode);
9777 %}
9778
9779 ins_pipe(icond_reg);
9780 %}
9781
9782 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9783 %{
9784 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9785
9786 ins_cost(INSN_COST * 3);
9787
9788 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9789 ins_encode %{
9790 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9791 __ fcsels(as_FloatRegister($dst$$reg),
9792 as_FloatRegister($src2$$reg),
9793 as_FloatRegister($src1$$reg),
9794 cond);
9795 %}
9796
9797 ins_pipe(fp_cond_reg_reg_s);
9798 %}
9799
9800 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9801 %{
9802 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9803
9804 ins_cost(INSN_COST * 3);
9805
9806 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9807 ins_encode %{
9808 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9809 __ fcsels(as_FloatRegister($dst$$reg),
9810 as_FloatRegister($src2$$reg),
9811 as_FloatRegister($src1$$reg),
9812 cond);
9813 %}
9814
9815 ins_pipe(fp_cond_reg_reg_s);
9816 %}
9817
9818 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9819 %{
9820 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9821
9822 ins_cost(INSN_COST * 3);
9823
9824 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9825 ins_encode %{
9826 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9827 __ fcseld(as_FloatRegister($dst$$reg),
9828 as_FloatRegister($src2$$reg),
9829 as_FloatRegister($src1$$reg),
9830 cond);
9831 %}
9832
9833 ins_pipe(fp_cond_reg_reg_d);
9834 %}
9835
9836 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9837 %{
9838 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9839
9840 ins_cost(INSN_COST * 3);
9841
9842 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9843 ins_encode %{
9844 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9845 __ fcseld(as_FloatRegister($dst$$reg),
9846 as_FloatRegister($src2$$reg),
9847 as_FloatRegister($src1$$reg),
9848 cond);
9849 %}
9850
9851 ins_pipe(fp_cond_reg_reg_d);
9852 %}
9853
9854 // ============================================================================
9855 // Arithmetic Instructions
9856 //
9857
9858 // Integer Addition
9859
9860 // TODO
9861 // these currently employ operations which do not set CR and hence are
9862 // not flagged as killing CR but we would like to isolate the cases
9863 // where we want to set flags from those where we don't. need to work
9864 // out how to do that.
9865
9866 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9867 match(Set dst (AddI src1 src2));
9868
9869 ins_cost(INSN_COST);
9870 format %{ "addw $dst, $src1, $src2" %}
9871
9872 ins_encode %{
9873 __ addw(as_Register($dst$$reg),
9874 as_Register($src1$$reg),
9875 as_Register($src2$$reg));
9876 %}
9877
9878 ins_pipe(ialu_reg_reg);
9879 %}
9880
9881 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9882 match(Set dst (AddI src1 src2));
9883
9884 ins_cost(INSN_COST);
9885 format %{ "addw $dst, $src1, $src2" %}
9886
9887 // use opcode to indicate that this is an add not a sub
9888 opcode(0x0);
9889
9890 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9891
9892 ins_pipe(ialu_reg_imm);
9893 %}
9894
9895 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9896 match(Set dst (AddI (ConvL2I src1) src2));
9897
9898 ins_cost(INSN_COST);
9899 format %{ "addw $dst, $src1, $src2" %}
9900
9901 // use opcode to indicate that this is an add not a sub
9902 opcode(0x0);
9903
9904 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9905
9906 ins_pipe(ialu_reg_imm);
9907 %}
9908
9909 // Pointer Addition
9910 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9911 match(Set dst (AddP src1 src2));
9912
9913 ins_cost(INSN_COST);
9914 format %{ "add $dst, $src1, $src2\t# ptr" %}
9915
9916 ins_encode %{
9917 __ add(as_Register($dst$$reg),
9918 as_Register($src1$$reg),
9919 as_Register($src2$$reg));
9920 %}
9921
9922 ins_pipe(ialu_reg_reg);
9923 %}
9924
9925 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9926 match(Set dst (AddP src1 (ConvI2L src2)));
9927
9928 ins_cost(1.9 * INSN_COST);
9929 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9930
9931 ins_encode %{
9932 __ add(as_Register($dst$$reg),
9933 as_Register($src1$$reg),
9934 as_Register($src2$$reg), ext::sxtw);
9935 %}
9936
9937 ins_pipe(ialu_reg_reg);
9938 %}
9939
9940 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9941 match(Set dst (AddP src1 (LShiftL src2 scale)));
9942
9943 ins_cost(1.9 * INSN_COST);
9944 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9945
9946 ins_encode %{
9947 __ lea(as_Register($dst$$reg),
9948 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9949 Address::lsl($scale$$constant)));
9950 %}
9951
9952 ins_pipe(ialu_reg_reg_shift);
9953 %}
9954
9955 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9956 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9957
9958 ins_cost(1.9 * INSN_COST);
9959 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9960
9961 ins_encode %{
9962 __ lea(as_Register($dst$$reg),
9963 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9964 Address::sxtw($scale$$constant)));
9965 %}
9966
9967 ins_pipe(ialu_reg_reg_shift);
9968 %}
9969
9970 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9971 match(Set dst (LShiftL (ConvI2L src) scale));
9972
9973 ins_cost(INSN_COST);
9974 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9975
9976 ins_encode %{
9977 __ sbfiz(as_Register($dst$$reg),
9978 as_Register($src$$reg),
9979 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9980 %}
9981
9982 ins_pipe(ialu_reg_shift);
9983 %}
9984
9985 // Pointer Immediate Addition
9986 // n.b. this needs to be more expensive than using an indirect memory
9987 // operand
9988 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9989 match(Set dst (AddP src1 src2));
9990
9991 ins_cost(INSN_COST);
9992 format %{ "add $dst, $src1, $src2\t# ptr" %}
9993
9994 // use opcode to indicate that this is an add not a sub
9995 opcode(0x0);
9996
9997 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9998
9999 ins_pipe(ialu_reg_imm);
10000 %}
10001
10002 // Long Addition
10003 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10004
10005 match(Set dst (AddL src1 src2));
10006
10007 ins_cost(INSN_COST);
10008 format %{ "add $dst, $src1, $src2" %}
10009
10010 ins_encode %{
10011 __ add(as_Register($dst$$reg),
10012 as_Register($src1$$reg),
10013 as_Register($src2$$reg));
10014 %}
10015
10016 ins_pipe(ialu_reg_reg);
10017 %}
10018
10019 // No constant pool entries requiredLong Immediate Addition.
10020 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10021 match(Set dst (AddL src1 src2));
10022
10023 ins_cost(INSN_COST);
10024 format %{ "add $dst, $src1, $src2" %}
10025
10026 // use opcode to indicate that this is an add not a sub
10027 opcode(0x0);
10028
10029 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10030
10031 ins_pipe(ialu_reg_imm);
10032 %}
10033
10034 // Integer Subtraction
10035 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10036 match(Set dst (SubI src1 src2));
10037
10038 ins_cost(INSN_COST);
10039 format %{ "subw $dst, $src1, $src2" %}
10040
10041 ins_encode %{
10042 __ subw(as_Register($dst$$reg),
10043 as_Register($src1$$reg),
10044 as_Register($src2$$reg));
10045 %}
10046
10047 ins_pipe(ialu_reg_reg);
10048 %}
10049
10050 // Immediate Subtraction
10051 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10052 match(Set dst (SubI src1 src2));
10053
10054 ins_cost(INSN_COST);
10055 format %{ "subw $dst, $src1, $src2" %}
10056
10057 // use opcode to indicate that this is a sub not an add
10058 opcode(0x1);
10059
10060 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10061
10062 ins_pipe(ialu_reg_imm);
10063 %}
10064
10065 // Long Subtraction
10066 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10067
10068 match(Set dst (SubL src1 src2));
10069
10070 ins_cost(INSN_COST);
10071 format %{ "sub $dst, $src1, $src2" %}
10072
10073 ins_encode %{
10074 __ sub(as_Register($dst$$reg),
10075 as_Register($src1$$reg),
10076 as_Register($src2$$reg));
10077 %}
10078
10079 ins_pipe(ialu_reg_reg);
10080 %}
10081
10082 // No constant pool entries requiredLong Immediate Subtraction.
10083 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10084 match(Set dst (SubL src1 src2));
10085
10086 ins_cost(INSN_COST);
10087 format %{ "sub$dst, $src1, $src2" %}
10088
10089 // use opcode to indicate that this is a sub not an add
10090 opcode(0x1);
10091
10092 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10093
10094 ins_pipe(ialu_reg_imm);
10095 %}
10096
10097 // Integer Negation (special case for sub)
10098
10099 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10100 match(Set dst (SubI zero src));
10101
10102 ins_cost(INSN_COST);
10103 format %{ "negw $dst, $src\t# int" %}
10104
10105 ins_encode %{
10106 __ negw(as_Register($dst$$reg),
10107 as_Register($src$$reg));
10108 %}
10109
10110 ins_pipe(ialu_reg);
10111 %}
10112
10113 // Long Negation
10114
10115 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10116 match(Set dst (SubL zero src));
10117
10118 ins_cost(INSN_COST);
10119 format %{ "neg $dst, $src\t# long" %}
10120
10121 ins_encode %{
10122 __ neg(as_Register($dst$$reg),
10123 as_Register($src$$reg));
10124 %}
10125
10126 ins_pipe(ialu_reg);
10127 %}
10128
10129 // Integer Multiply
10130
10131 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10132 match(Set dst (MulI src1 src2));
10133
10134 ins_cost(INSN_COST * 3);
10135 format %{ "mulw $dst, $src1, $src2" %}
10136
10137 ins_encode %{
10138 __ mulw(as_Register($dst$$reg),
10139 as_Register($src1$$reg),
10140 as_Register($src2$$reg));
10141 %}
10142
10143 ins_pipe(imul_reg_reg);
10144 %}
10145
10146 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10147 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10148
10149 ins_cost(INSN_COST * 3);
10150 format %{ "smull $dst, $src1, $src2" %}
10151
10152 ins_encode %{
10153 __ smull(as_Register($dst$$reg),
10154 as_Register($src1$$reg),
10155 as_Register($src2$$reg));
10156 %}
10157
10158 ins_pipe(imul_reg_reg);
10159 %}
10160
10161 // Long Multiply
10162
10163 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10164 match(Set dst (MulL src1 src2));
10165
10166 ins_cost(INSN_COST * 5);
10167 format %{ "mul $dst, $src1, $src2" %}
10168
10169 ins_encode %{
10170 __ mul(as_Register($dst$$reg),
10171 as_Register($src1$$reg),
10172 as_Register($src2$$reg));
10173 %}
10174
10175 ins_pipe(lmul_reg_reg);
10176 %}
10177
10178 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10179 %{
10180 match(Set dst (MulHiL src1 src2));
10181
10182 ins_cost(INSN_COST * 7);
10183 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10184
10185 ins_encode %{
10186 __ smulh(as_Register($dst$$reg),
10187 as_Register($src1$$reg),
10188 as_Register($src2$$reg));
10189 %}
10190
10191 ins_pipe(lmul_reg_reg);
10192 %}
10193
10194 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10195 %{
10196 match(Set dst (UMulHiL src1 src2));
10197
10198 ins_cost(INSN_COST * 7);
10199 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10200
10201 ins_encode %{
10202 __ umulh(as_Register($dst$$reg),
10203 as_Register($src1$$reg),
10204 as_Register($src2$$reg));
10205 %}
10206
10207 ins_pipe(lmul_reg_reg);
10208 %}
10209
10210 // Combined Integer Multiply & Add/Sub
10211
10212 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10213 match(Set dst (AddI src3 (MulI src1 src2)));
10214
10215 ins_cost(INSN_COST * 3);
10216 format %{ "madd $dst, $src1, $src2, $src3" %}
10217
10218 ins_encode %{
10219 __ maddw(as_Register($dst$$reg),
10220 as_Register($src1$$reg),
10221 as_Register($src2$$reg),
10222 as_Register($src3$$reg));
10223 %}
10224
10225 ins_pipe(imac_reg_reg);
10226 %}
10227
10228 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10229 match(Set dst (SubI src3 (MulI src1 src2)));
10230
10231 ins_cost(INSN_COST * 3);
10232 format %{ "msub $dst, $src1, $src2, $src3" %}
10233
10234 ins_encode %{
10235 __ msubw(as_Register($dst$$reg),
10236 as_Register($src1$$reg),
10237 as_Register($src2$$reg),
10238 as_Register($src3$$reg));
10239 %}
10240
10241 ins_pipe(imac_reg_reg);
10242 %}
10243
10244 // Combined Integer Multiply & Neg
10245
10246 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10247 match(Set dst (MulI (SubI zero src1) src2));
10248
10249 ins_cost(INSN_COST * 3);
10250 format %{ "mneg $dst, $src1, $src2" %}
10251
10252 ins_encode %{
10253 __ mnegw(as_Register($dst$$reg),
10254 as_Register($src1$$reg),
10255 as_Register($src2$$reg));
10256 %}
10257
10258 ins_pipe(imac_reg_reg);
10259 %}
10260
10261 // Combined Long Multiply & Add/Sub
10262
10263 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10264 match(Set dst (AddL src3 (MulL src1 src2)));
10265
10266 ins_cost(INSN_COST * 5);
10267 format %{ "madd $dst, $src1, $src2, $src3" %}
10268
10269 ins_encode %{
10270 __ madd(as_Register($dst$$reg),
10271 as_Register($src1$$reg),
10272 as_Register($src2$$reg),
10273 as_Register($src3$$reg));
10274 %}
10275
10276 ins_pipe(lmac_reg_reg);
10277 %}
10278
10279 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10280 match(Set dst (SubL src3 (MulL src1 src2)));
10281
10282 ins_cost(INSN_COST * 5);
10283 format %{ "msub $dst, $src1, $src2, $src3" %}
10284
10285 ins_encode %{
10286 __ msub(as_Register($dst$$reg),
10287 as_Register($src1$$reg),
10288 as_Register($src2$$reg),
10289 as_Register($src3$$reg));
10290 %}
10291
10292 ins_pipe(lmac_reg_reg);
10293 %}
10294
10295 // Combined Long Multiply & Neg
10296
10297 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10298 match(Set dst (MulL (SubL zero src1) src2));
10299
10300 ins_cost(INSN_COST * 5);
10301 format %{ "mneg $dst, $src1, $src2" %}
10302
10303 ins_encode %{
10304 __ mneg(as_Register($dst$$reg),
10305 as_Register($src1$$reg),
10306 as_Register($src2$$reg));
10307 %}
10308
10309 ins_pipe(lmac_reg_reg);
10310 %}
10311
10312 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10313
10314 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10315 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10316
10317 ins_cost(INSN_COST * 3);
10318 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10319
10320 ins_encode %{
10321 __ smaddl(as_Register($dst$$reg),
10322 as_Register($src1$$reg),
10323 as_Register($src2$$reg),
10324 as_Register($src3$$reg));
10325 %}
10326
10327 ins_pipe(imac_reg_reg);
10328 %}
10329
10330 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10331 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10332
10333 ins_cost(INSN_COST * 3);
10334 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10335
10336 ins_encode %{
10337 __ smsubl(as_Register($dst$$reg),
10338 as_Register($src1$$reg),
10339 as_Register($src2$$reg),
10340 as_Register($src3$$reg));
10341 %}
10342
10343 ins_pipe(imac_reg_reg);
10344 %}
10345
10346 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10347 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10348
10349 ins_cost(INSN_COST * 3);
10350 format %{ "smnegl $dst, $src1, $src2" %}
10351
10352 ins_encode %{
10353 __ smnegl(as_Register($dst$$reg),
10354 as_Register($src1$$reg),
10355 as_Register($src2$$reg));
10356 %}
10357
10358 ins_pipe(imac_reg_reg);
10359 %}
10360
10361 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10362
10363 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10364 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10365
10366 ins_cost(INSN_COST * 5);
10367 format %{ "mulw rscratch1, $src1, $src2\n\t"
10368 "maddw $dst, $src3, $src4, rscratch1" %}
10369
10370 ins_encode %{
10371 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10372 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10373
10374 ins_pipe(imac_reg_reg);
10375 %}
10376
10377 // Integer Divide
10378
10379 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10380 match(Set dst (DivI src1 src2));
10381
10382 ins_cost(INSN_COST * 19);
10383 format %{ "sdivw $dst, $src1, $src2" %}
10384
10385 ins_encode(aarch64_enc_divw(dst, src1, src2));
10386 ins_pipe(idiv_reg_reg);
10387 %}
10388
10389 // Long Divide
10390
10391 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10392 match(Set dst (DivL src1 src2));
10393
10394 ins_cost(INSN_COST * 35);
10395 format %{ "sdiv $dst, $src1, $src2" %}
10396
10397 ins_encode(aarch64_enc_div(dst, src1, src2));
10398 ins_pipe(ldiv_reg_reg);
10399 %}
10400
10401 // Integer Remainder
10402
10403 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10404 match(Set dst (ModI src1 src2));
10405
10406 ins_cost(INSN_COST * 22);
10407 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10408 "msubw $dst, rscratch1, $src2, $src1" %}
10409
10410 ins_encode(aarch64_enc_modw(dst, src1, src2));
10411 ins_pipe(idiv_reg_reg);
10412 %}
10413
10414 // Long Remainder
10415
10416 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10417 match(Set dst (ModL src1 src2));
10418
10419 ins_cost(INSN_COST * 38);
10420 format %{ "sdiv rscratch1, $src1, $src2\n"
10421 "msub $dst, rscratch1, $src2, $src1" %}
10422
10423 ins_encode(aarch64_enc_mod(dst, src1, src2));
10424 ins_pipe(ldiv_reg_reg);
10425 %}
10426
10427 // Unsigned Integer Divide
10428
10429 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10430 match(Set dst (UDivI src1 src2));
10431
10432 ins_cost(INSN_COST * 19);
10433 format %{ "udivw $dst, $src1, $src2" %}
10434
10435 ins_encode %{
10436 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10437 %}
10438
10439 ins_pipe(idiv_reg_reg);
10440 %}
10441
10442 // Unsigned Long Divide
10443
10444 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10445 match(Set dst (UDivL src1 src2));
10446
10447 ins_cost(INSN_COST * 35);
10448 format %{ "udiv $dst, $src1, $src2" %}
10449
10450 ins_encode %{
10451 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10452 %}
10453
10454 ins_pipe(ldiv_reg_reg);
10455 %}
10456
10457 // Unsigned Integer Remainder
10458
10459 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10460 match(Set dst (UModI src1 src2));
10461
10462 ins_cost(INSN_COST * 22);
10463 format %{ "udivw rscratch1, $src1, $src2\n\t"
10464 "msubw $dst, rscratch1, $src2, $src1" %}
10465
10466 ins_encode %{
10467 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10468 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10469 %}
10470
10471 ins_pipe(idiv_reg_reg);
10472 %}
10473
10474 // Unsigned Long Remainder
10475
10476 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10477 match(Set dst (UModL src1 src2));
10478
10479 ins_cost(INSN_COST * 38);
10480 format %{ "udiv rscratch1, $src1, $src2\n"
10481 "msub $dst, rscratch1, $src2, $src1" %}
10482
10483 ins_encode %{
10484 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10485 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10486 %}
10487
10488 ins_pipe(ldiv_reg_reg);
10489 %}
10490
10491 // Integer Shifts
10492
10493 // Shift Left Register
10494 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10495 match(Set dst (LShiftI src1 src2));
10496
10497 ins_cost(INSN_COST * 2);
10498 format %{ "lslvw $dst, $src1, $src2" %}
10499
10500 ins_encode %{
10501 __ lslvw(as_Register($dst$$reg),
10502 as_Register($src1$$reg),
10503 as_Register($src2$$reg));
10504 %}
10505
10506 ins_pipe(ialu_reg_reg_vshift);
10507 %}
10508
10509 // Shift Left Immediate
10510 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10511 match(Set dst (LShiftI src1 src2));
10512
10513 ins_cost(INSN_COST);
10514 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10515
10516 ins_encode %{
10517 __ lslw(as_Register($dst$$reg),
10518 as_Register($src1$$reg),
10519 $src2$$constant & 0x1f);
10520 %}
10521
10522 ins_pipe(ialu_reg_shift);
10523 %}
10524
10525 // Shift Right Logical Register
10526 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10527 match(Set dst (URShiftI src1 src2));
10528
10529 ins_cost(INSN_COST * 2);
10530 format %{ "lsrvw $dst, $src1, $src2" %}
10531
10532 ins_encode %{
10533 __ lsrvw(as_Register($dst$$reg),
10534 as_Register($src1$$reg),
10535 as_Register($src2$$reg));
10536 %}
10537
10538 ins_pipe(ialu_reg_reg_vshift);
10539 %}
10540
10541 // Shift Right Logical Immediate
10542 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10543 match(Set dst (URShiftI src1 src2));
10544
10545 ins_cost(INSN_COST);
10546 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10547
10548 ins_encode %{
10549 __ lsrw(as_Register($dst$$reg),
10550 as_Register($src1$$reg),
10551 $src2$$constant & 0x1f);
10552 %}
10553
10554 ins_pipe(ialu_reg_shift);
10555 %}
10556
10557 // Shift Right Arithmetic Register
10558 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10559 match(Set dst (RShiftI src1 src2));
10560
10561 ins_cost(INSN_COST * 2);
10562 format %{ "asrvw $dst, $src1, $src2" %}
10563
10564 ins_encode %{
10565 __ asrvw(as_Register($dst$$reg),
10566 as_Register($src1$$reg),
10567 as_Register($src2$$reg));
10568 %}
10569
10570 ins_pipe(ialu_reg_reg_vshift);
10571 %}
10572
10573 // Shift Right Arithmetic Immediate
10574 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10575 match(Set dst (RShiftI src1 src2));
10576
10577 ins_cost(INSN_COST);
10578 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10579
10580 ins_encode %{
10581 __ asrw(as_Register($dst$$reg),
10582 as_Register($src1$$reg),
10583 $src2$$constant & 0x1f);
10584 %}
10585
10586 ins_pipe(ialu_reg_shift);
10587 %}
10588
10589 // Combined Int Mask and Right Shift (using UBFM)
10590 // TODO
10591
10592 // Long Shifts
10593
10594 // Shift Left Register
10595 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10596 match(Set dst (LShiftL src1 src2));
10597
10598 ins_cost(INSN_COST * 2);
10599 format %{ "lslv $dst, $src1, $src2" %}
10600
10601 ins_encode %{
10602 __ lslv(as_Register($dst$$reg),
10603 as_Register($src1$$reg),
10604 as_Register($src2$$reg));
10605 %}
10606
10607 ins_pipe(ialu_reg_reg_vshift);
10608 %}
10609
10610 // Shift Left Immediate
10611 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10612 match(Set dst (LShiftL src1 src2));
10613
10614 ins_cost(INSN_COST);
10615 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10616
10617 ins_encode %{
10618 __ lsl(as_Register($dst$$reg),
10619 as_Register($src1$$reg),
10620 $src2$$constant & 0x3f);
10621 %}
10622
10623 ins_pipe(ialu_reg_shift);
10624 %}
10625
10626 // Shift Right Logical Register
10627 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10628 match(Set dst (URShiftL src1 src2));
10629
10630 ins_cost(INSN_COST * 2);
10631 format %{ "lsrv $dst, $src1, $src2" %}
10632
10633 ins_encode %{
10634 __ lsrv(as_Register($dst$$reg),
10635 as_Register($src1$$reg),
10636 as_Register($src2$$reg));
10637 %}
10638
10639 ins_pipe(ialu_reg_reg_vshift);
10640 %}
10641
10642 // Shift Right Logical Immediate
10643 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10644 match(Set dst (URShiftL src1 src2));
10645
10646 ins_cost(INSN_COST);
10647 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10648
10649 ins_encode %{
10650 __ lsr(as_Register($dst$$reg),
10651 as_Register($src1$$reg),
10652 $src2$$constant & 0x3f);
10653 %}
10654
10655 ins_pipe(ialu_reg_shift);
10656 %}
10657
10658 // A special-case pattern for card table stores.
10659 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10660 match(Set dst (URShiftL (CastP2X src1) src2));
10661
10662 ins_cost(INSN_COST);
10663 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10664
10665 ins_encode %{
10666 __ lsr(as_Register($dst$$reg),
10667 as_Register($src1$$reg),
10668 $src2$$constant & 0x3f);
10669 %}
10670
10671 ins_pipe(ialu_reg_shift);
10672 %}
10673
10674 // Shift Right Arithmetic Register
10675 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10676 match(Set dst (RShiftL src1 src2));
10677
10678 ins_cost(INSN_COST * 2);
10679 format %{ "asrv $dst, $src1, $src2" %}
10680
10681 ins_encode %{
10682 __ asrv(as_Register($dst$$reg),
10683 as_Register($src1$$reg),
10684 as_Register($src2$$reg));
10685 %}
10686
10687 ins_pipe(ialu_reg_reg_vshift);
10688 %}
10689
10690 // Shift Right Arithmetic Immediate
10691 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10692 match(Set dst (RShiftL src1 src2));
10693
10694 ins_cost(INSN_COST);
10695 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10696
10697 ins_encode %{
10698 __ asr(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 $src2$$constant & 0x3f);
10701 %}
10702
10703 ins_pipe(ialu_reg_shift);
10704 %}
10705
10706 // BEGIN This section of the file is automatically generated. Do not edit --------------
10707 // This section is generated from aarch64_ad.m4
10708
10709 // This pattern is automatically generated from aarch64_ad.m4
10710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10711 instruct regL_not_reg(iRegLNoSp dst,
10712 iRegL src1, immL_M1 m1,
10713 rFlagsReg cr) %{
10714 match(Set dst (XorL src1 m1));
10715 ins_cost(INSN_COST);
10716 format %{ "eon $dst, $src1, zr" %}
10717
10718 ins_encode %{
10719 __ eon(as_Register($dst$$reg),
10720 as_Register($src1$$reg),
10721 zr,
10722 Assembler::LSL, 0);
10723 %}
10724
10725 ins_pipe(ialu_reg);
10726 %}
10727
10728 // This pattern is automatically generated from aarch64_ad.m4
10729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10730 instruct regI_not_reg(iRegINoSp dst,
10731 iRegIorL2I src1, immI_M1 m1,
10732 rFlagsReg cr) %{
10733 match(Set dst (XorI src1 m1));
10734 ins_cost(INSN_COST);
10735 format %{ "eonw $dst, $src1, zr" %}
10736
10737 ins_encode %{
10738 __ eonw(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 zr,
10741 Assembler::LSL, 0);
10742 %}
10743
10744 ins_pipe(ialu_reg);
10745 %}
10746
10747 // This pattern is automatically generated from aarch64_ad.m4
10748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10749 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10750 immI0 zero, iRegIorL2I src1, immI src2) %{
10751 match(Set dst (SubI zero (URShiftI src1 src2)));
10752
10753 ins_cost(1.9 * INSN_COST);
10754 format %{ "negw $dst, $src1, LSR $src2" %}
10755
10756 ins_encode %{
10757 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10758 Assembler::LSR, $src2$$constant & 0x1f);
10759 %}
10760
10761 ins_pipe(ialu_reg_shift);
10762 %}
10763
10764 // This pattern is automatically generated from aarch64_ad.m4
10765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10766 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10767 immI0 zero, iRegIorL2I src1, immI src2) %{
10768 match(Set dst (SubI zero (RShiftI src1 src2)));
10769
10770 ins_cost(1.9 * INSN_COST);
10771 format %{ "negw $dst, $src1, ASR $src2" %}
10772
10773 ins_encode %{
10774 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10775 Assembler::ASR, $src2$$constant & 0x1f);
10776 %}
10777
10778 ins_pipe(ialu_reg_shift);
10779 %}
10780
10781 // This pattern is automatically generated from aarch64_ad.m4
10782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10783 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10784 immI0 zero, iRegIorL2I src1, immI src2) %{
10785 match(Set dst (SubI zero (LShiftI src1 src2)));
10786
10787 ins_cost(1.9 * INSN_COST);
10788 format %{ "negw $dst, $src1, LSL $src2" %}
10789
10790 ins_encode %{
10791 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10792 Assembler::LSL, $src2$$constant & 0x1f);
10793 %}
10794
10795 ins_pipe(ialu_reg_shift);
10796 %}
10797
10798 // This pattern is automatically generated from aarch64_ad.m4
10799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10800 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10801 immL0 zero, iRegL src1, immI src2) %{
10802 match(Set dst (SubL zero (URShiftL src1 src2)));
10803
10804 ins_cost(1.9 * INSN_COST);
10805 format %{ "neg $dst, $src1, LSR $src2" %}
10806
10807 ins_encode %{
10808 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10809 Assembler::LSR, $src2$$constant & 0x3f);
10810 %}
10811
10812 ins_pipe(ialu_reg_shift);
10813 %}
10814
10815 // This pattern is automatically generated from aarch64_ad.m4
10816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10817 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10818 immL0 zero, iRegL src1, immI src2) %{
10819 match(Set dst (SubL zero (RShiftL src1 src2)));
10820
10821 ins_cost(1.9 * INSN_COST);
10822 format %{ "neg $dst, $src1, ASR $src2" %}
10823
10824 ins_encode %{
10825 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10826 Assembler::ASR, $src2$$constant & 0x3f);
10827 %}
10828
10829 ins_pipe(ialu_reg_shift);
10830 %}
10831
10832 // This pattern is automatically generated from aarch64_ad.m4
10833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10834 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10835 immL0 zero, iRegL src1, immI src2) %{
10836 match(Set dst (SubL zero (LShiftL src1 src2)));
10837
10838 ins_cost(1.9 * INSN_COST);
10839 format %{ "neg $dst, $src1, LSL $src2" %}
10840
10841 ins_encode %{
10842 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10843 Assembler::LSL, $src2$$constant & 0x3f);
10844 %}
10845
10846 ins_pipe(ialu_reg_shift);
10847 %}
10848
10849 // This pattern is automatically generated from aarch64_ad.m4
10850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10851 instruct AndI_reg_not_reg(iRegINoSp dst,
10852 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10853 match(Set dst (AndI src1 (XorI src2 m1)));
10854 ins_cost(INSN_COST);
10855 format %{ "bicw $dst, $src1, $src2" %}
10856
10857 ins_encode %{
10858 __ bicw(as_Register($dst$$reg),
10859 as_Register($src1$$reg),
10860 as_Register($src2$$reg),
10861 Assembler::LSL, 0);
10862 %}
10863
10864 ins_pipe(ialu_reg_reg);
10865 %}
10866
10867 // This pattern is automatically generated from aarch64_ad.m4
10868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10869 instruct AndL_reg_not_reg(iRegLNoSp dst,
10870 iRegL src1, iRegL src2, immL_M1 m1) %{
10871 match(Set dst (AndL src1 (XorL src2 m1)));
10872 ins_cost(INSN_COST);
10873 format %{ "bic $dst, $src1, $src2" %}
10874
10875 ins_encode %{
10876 __ bic(as_Register($dst$$reg),
10877 as_Register($src1$$reg),
10878 as_Register($src2$$reg),
10879 Assembler::LSL, 0);
10880 %}
10881
10882 ins_pipe(ialu_reg_reg);
10883 %}
10884
10885 // This pattern is automatically generated from aarch64_ad.m4
10886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10887 instruct OrI_reg_not_reg(iRegINoSp dst,
10888 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10889 match(Set dst (OrI src1 (XorI src2 m1)));
10890 ins_cost(INSN_COST);
10891 format %{ "ornw $dst, $src1, $src2" %}
10892
10893 ins_encode %{
10894 __ ornw(as_Register($dst$$reg),
10895 as_Register($src1$$reg),
10896 as_Register($src2$$reg),
10897 Assembler::LSL, 0);
10898 %}
10899
10900 ins_pipe(ialu_reg_reg);
10901 %}
10902
10903 // This pattern is automatically generated from aarch64_ad.m4
10904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10905 instruct OrL_reg_not_reg(iRegLNoSp dst,
10906 iRegL src1, iRegL src2, immL_M1 m1) %{
10907 match(Set dst (OrL src1 (XorL src2 m1)));
10908 ins_cost(INSN_COST);
10909 format %{ "orn $dst, $src1, $src2" %}
10910
10911 ins_encode %{
10912 __ orn(as_Register($dst$$reg),
10913 as_Register($src1$$reg),
10914 as_Register($src2$$reg),
10915 Assembler::LSL, 0);
10916 %}
10917
10918 ins_pipe(ialu_reg_reg);
10919 %}
10920
10921 // This pattern is automatically generated from aarch64_ad.m4
10922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10923 instruct XorI_reg_not_reg(iRegINoSp dst,
10924 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10925 match(Set dst (XorI m1 (XorI src2 src1)));
10926 ins_cost(INSN_COST);
10927 format %{ "eonw $dst, $src1, $src2" %}
10928
10929 ins_encode %{
10930 __ eonw(as_Register($dst$$reg),
10931 as_Register($src1$$reg),
10932 as_Register($src2$$reg),
10933 Assembler::LSL, 0);
10934 %}
10935
10936 ins_pipe(ialu_reg_reg);
10937 %}
10938
10939 // This pattern is automatically generated from aarch64_ad.m4
10940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10941 instruct XorL_reg_not_reg(iRegLNoSp dst,
10942 iRegL src1, iRegL src2, immL_M1 m1) %{
10943 match(Set dst (XorL m1 (XorL src2 src1)));
10944 ins_cost(INSN_COST);
10945 format %{ "eon $dst, $src1, $src2" %}
10946
10947 ins_encode %{
10948 __ eon(as_Register($dst$$reg),
10949 as_Register($src1$$reg),
10950 as_Register($src2$$reg),
10951 Assembler::LSL, 0);
10952 %}
10953
10954 ins_pipe(ialu_reg_reg);
10955 %}
10956
10957 // This pattern is automatically generated from aarch64_ad.m4
10958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10959 // val & (-1 ^ (val >>> shift)) ==> bicw
10960 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10961 iRegIorL2I src1, iRegIorL2I src2,
10962 immI src3, immI_M1 src4) %{
10963 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10964 ins_cost(1.9 * INSN_COST);
10965 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10966
10967 ins_encode %{
10968 __ bicw(as_Register($dst$$reg),
10969 as_Register($src1$$reg),
10970 as_Register($src2$$reg),
10971 Assembler::LSR,
10972 $src3$$constant & 0x1f);
10973 %}
10974
10975 ins_pipe(ialu_reg_reg_shift);
10976 %}
10977
10978 // This pattern is automatically generated from aarch64_ad.m4
10979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10980 // val & (-1 ^ (val >>> shift)) ==> bic
10981 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10982 iRegL src1, iRegL src2,
10983 immI src3, immL_M1 src4) %{
10984 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10985 ins_cost(1.9 * INSN_COST);
10986 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10987
10988 ins_encode %{
10989 __ bic(as_Register($dst$$reg),
10990 as_Register($src1$$reg),
10991 as_Register($src2$$reg),
10992 Assembler::LSR,
10993 $src3$$constant & 0x3f);
10994 %}
10995
10996 ins_pipe(ialu_reg_reg_shift);
10997 %}
10998
10999 // This pattern is automatically generated from aarch64_ad.m4
11000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11001 // val & (-1 ^ (val >> shift)) ==> bicw
11002 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11003 iRegIorL2I src1, iRegIorL2I src2,
11004 immI src3, immI_M1 src4) %{
11005 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11006 ins_cost(1.9 * INSN_COST);
11007 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11008
11009 ins_encode %{
11010 __ bicw(as_Register($dst$$reg),
11011 as_Register($src1$$reg),
11012 as_Register($src2$$reg),
11013 Assembler::ASR,
11014 $src3$$constant & 0x1f);
11015 %}
11016
11017 ins_pipe(ialu_reg_reg_shift);
11018 %}
11019
11020 // This pattern is automatically generated from aarch64_ad.m4
11021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11022 // val & (-1 ^ (val >> shift)) ==> bic
11023 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11024 iRegL src1, iRegL src2,
11025 immI src3, immL_M1 src4) %{
11026 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11027 ins_cost(1.9 * INSN_COST);
11028 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11029
11030 ins_encode %{
11031 __ bic(as_Register($dst$$reg),
11032 as_Register($src1$$reg),
11033 as_Register($src2$$reg),
11034 Assembler::ASR,
11035 $src3$$constant & 0x3f);
11036 %}
11037
11038 ins_pipe(ialu_reg_reg_shift);
11039 %}
11040
11041 // This pattern is automatically generated from aarch64_ad.m4
11042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11043 // val & (-1 ^ (val ror shift)) ==> bicw
11044 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11045 iRegIorL2I src1, iRegIorL2I src2,
11046 immI src3, immI_M1 src4) %{
11047 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11048 ins_cost(1.9 * INSN_COST);
11049 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11050
11051 ins_encode %{
11052 __ bicw(as_Register($dst$$reg),
11053 as_Register($src1$$reg),
11054 as_Register($src2$$reg),
11055 Assembler::ROR,
11056 $src3$$constant & 0x1f);
11057 %}
11058
11059 ins_pipe(ialu_reg_reg_shift);
11060 %}
11061
11062 // This pattern is automatically generated from aarch64_ad.m4
11063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11064 // val & (-1 ^ (val ror shift)) ==> bic
11065 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11066 iRegL src1, iRegL src2,
11067 immI src3, immL_M1 src4) %{
11068 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11069 ins_cost(1.9 * INSN_COST);
11070 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11071
11072 ins_encode %{
11073 __ bic(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg),
11076 Assembler::ROR,
11077 $src3$$constant & 0x3f);
11078 %}
11079
11080 ins_pipe(ialu_reg_reg_shift);
11081 %}
11082
11083 // This pattern is automatically generated from aarch64_ad.m4
11084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11085 // val & (-1 ^ (val << shift)) ==> bicw
11086 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11087 iRegIorL2I src1, iRegIorL2I src2,
11088 immI src3, immI_M1 src4) %{
11089 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11090 ins_cost(1.9 * INSN_COST);
11091 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11092
11093 ins_encode %{
11094 __ bicw(as_Register($dst$$reg),
11095 as_Register($src1$$reg),
11096 as_Register($src2$$reg),
11097 Assembler::LSL,
11098 $src3$$constant & 0x1f);
11099 %}
11100
11101 ins_pipe(ialu_reg_reg_shift);
11102 %}
11103
11104 // This pattern is automatically generated from aarch64_ad.m4
11105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11106 // val & (-1 ^ (val << shift)) ==> bic
11107 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11108 iRegL src1, iRegL src2,
11109 immI src3, immL_M1 src4) %{
11110 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11111 ins_cost(1.9 * INSN_COST);
11112 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11113
11114 ins_encode %{
11115 __ bic(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg),
11118 Assembler::LSL,
11119 $src3$$constant & 0x3f);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_shift);
11123 %}
11124
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11128 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11129 iRegIorL2I src1, iRegIorL2I src2,
11130 immI src3, immI_M1 src4) %{
11131 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11132 ins_cost(1.9 * INSN_COST);
11133 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11134
11135 ins_encode %{
11136 __ eonw(as_Register($dst$$reg),
11137 as_Register($src1$$reg),
11138 as_Register($src2$$reg),
11139 Assembler::LSR,
11140 $src3$$constant & 0x1f);
11141 %}
11142
11143 ins_pipe(ialu_reg_reg_shift);
11144 %}
11145
11146 // This pattern is automatically generated from aarch64_ad.m4
11147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11148 // val ^ (-1 ^ (val >>> shift)) ==> eon
11149 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11150 iRegL src1, iRegL src2,
11151 immI src3, immL_M1 src4) %{
11152 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11153 ins_cost(1.9 * INSN_COST);
11154 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11155
11156 ins_encode %{
11157 __ eon(as_Register($dst$$reg),
11158 as_Register($src1$$reg),
11159 as_Register($src2$$reg),
11160 Assembler::LSR,
11161 $src3$$constant & 0x3f);
11162 %}
11163
11164 ins_pipe(ialu_reg_reg_shift);
11165 %}
11166
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 // val ^ (-1 ^ (val >> shift)) ==> eonw
11170 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11171 iRegIorL2I src1, iRegIorL2I src2,
11172 immI src3, immI_M1 src4) %{
11173 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11174 ins_cost(1.9 * INSN_COST);
11175 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11176
11177 ins_encode %{
11178 __ eonw(as_Register($dst$$reg),
11179 as_Register($src1$$reg),
11180 as_Register($src2$$reg),
11181 Assembler::ASR,
11182 $src3$$constant & 0x1f);
11183 %}
11184
11185 ins_pipe(ialu_reg_reg_shift);
11186 %}
11187
11188 // This pattern is automatically generated from aarch64_ad.m4
11189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11190 // val ^ (-1 ^ (val >> shift)) ==> eon
11191 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11192 iRegL src1, iRegL src2,
11193 immI src3, immL_M1 src4) %{
11194 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11195 ins_cost(1.9 * INSN_COST);
11196 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11197
11198 ins_encode %{
11199 __ eon(as_Register($dst$$reg),
11200 as_Register($src1$$reg),
11201 as_Register($src2$$reg),
11202 Assembler::ASR,
11203 $src3$$constant & 0x3f);
11204 %}
11205
11206 ins_pipe(ialu_reg_reg_shift);
11207 %}
11208
11209 // This pattern is automatically generated from aarch64_ad.m4
11210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11211 // val ^ (-1 ^ (val ror shift)) ==> eonw
11212 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11213 iRegIorL2I src1, iRegIorL2I src2,
11214 immI src3, immI_M1 src4) %{
11215 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11216 ins_cost(1.9 * INSN_COST);
11217 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11218
11219 ins_encode %{
11220 __ eonw(as_Register($dst$$reg),
11221 as_Register($src1$$reg),
11222 as_Register($src2$$reg),
11223 Assembler::ROR,
11224 $src3$$constant & 0x1f);
11225 %}
11226
11227 ins_pipe(ialu_reg_reg_shift);
11228 %}
11229
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 // val ^ (-1 ^ (val ror shift)) ==> eon
11233 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11234 iRegL src1, iRegL src2,
11235 immI src3, immL_M1 src4) %{
11236 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11237 ins_cost(1.9 * INSN_COST);
11238 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11239
11240 ins_encode %{
11241 __ eon(as_Register($dst$$reg),
11242 as_Register($src1$$reg),
11243 as_Register($src2$$reg),
11244 Assembler::ROR,
11245 $src3$$constant & 0x3f);
11246 %}
11247
11248 ins_pipe(ialu_reg_reg_shift);
11249 %}
11250
11251 // This pattern is automatically generated from aarch64_ad.m4
11252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11253 // val ^ (-1 ^ (val << shift)) ==> eonw
11254 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11255 iRegIorL2I src1, iRegIorL2I src2,
11256 immI src3, immI_M1 src4) %{
11257 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11258 ins_cost(1.9 * INSN_COST);
11259 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11260
11261 ins_encode %{
11262 __ eonw(as_Register($dst$$reg),
11263 as_Register($src1$$reg),
11264 as_Register($src2$$reg),
11265 Assembler::LSL,
11266 $src3$$constant & 0x1f);
11267 %}
11268
11269 ins_pipe(ialu_reg_reg_shift);
11270 %}
11271
11272 // This pattern is automatically generated from aarch64_ad.m4
11273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11274 // val ^ (-1 ^ (val << shift)) ==> eon
11275 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11276 iRegL src1, iRegL src2,
11277 immI src3, immL_M1 src4) %{
11278 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11279 ins_cost(1.9 * INSN_COST);
11280 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11281
11282 ins_encode %{
11283 __ eon(as_Register($dst$$reg),
11284 as_Register($src1$$reg),
11285 as_Register($src2$$reg),
11286 Assembler::LSL,
11287 $src3$$constant & 0x3f);
11288 %}
11289
11290 ins_pipe(ialu_reg_reg_shift);
11291 %}
11292
11293 // This pattern is automatically generated from aarch64_ad.m4
11294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11295 // val | (-1 ^ (val >>> shift)) ==> ornw
11296 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11297 iRegIorL2I src1, iRegIorL2I src2,
11298 immI src3, immI_M1 src4) %{
11299 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11300 ins_cost(1.9 * INSN_COST);
11301 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11302
11303 ins_encode %{
11304 __ ornw(as_Register($dst$$reg),
11305 as_Register($src1$$reg),
11306 as_Register($src2$$reg),
11307 Assembler::LSR,
11308 $src3$$constant & 0x1f);
11309 %}
11310
11311 ins_pipe(ialu_reg_reg_shift);
11312 %}
11313
11314 // This pattern is automatically generated from aarch64_ad.m4
11315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11316 // val | (-1 ^ (val >>> shift)) ==> orn
11317 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11318 iRegL src1, iRegL src2,
11319 immI src3, immL_M1 src4) %{
11320 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11321 ins_cost(1.9 * INSN_COST);
11322 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11323
11324 ins_encode %{
11325 __ orn(as_Register($dst$$reg),
11326 as_Register($src1$$reg),
11327 as_Register($src2$$reg),
11328 Assembler::LSR,
11329 $src3$$constant & 0x3f);
11330 %}
11331
11332 ins_pipe(ialu_reg_reg_shift);
11333 %}
11334
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 // val | (-1 ^ (val >> shift)) ==> ornw
11338 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11339 iRegIorL2I src1, iRegIorL2I src2,
11340 immI src3, immI_M1 src4) %{
11341 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11342 ins_cost(1.9 * INSN_COST);
11343 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11344
11345 ins_encode %{
11346 __ ornw(as_Register($dst$$reg),
11347 as_Register($src1$$reg),
11348 as_Register($src2$$reg),
11349 Assembler::ASR,
11350 $src3$$constant & 0x1f);
11351 %}
11352
11353 ins_pipe(ialu_reg_reg_shift);
11354 %}
11355
11356 // This pattern is automatically generated from aarch64_ad.m4
11357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11358 // val | (-1 ^ (val >> shift)) ==> orn
11359 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11360 iRegL src1, iRegL src2,
11361 immI src3, immL_M1 src4) %{
11362 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11363 ins_cost(1.9 * INSN_COST);
11364 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11365
11366 ins_encode %{
11367 __ orn(as_Register($dst$$reg),
11368 as_Register($src1$$reg),
11369 as_Register($src2$$reg),
11370 Assembler::ASR,
11371 $src3$$constant & 0x3f);
11372 %}
11373
11374 ins_pipe(ialu_reg_reg_shift);
11375 %}
11376
11377 // This pattern is automatically generated from aarch64_ad.m4
11378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11379 // val | (-1 ^ (val ror shift)) ==> ornw
11380 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11381 iRegIorL2I src1, iRegIorL2I src2,
11382 immI src3, immI_M1 src4) %{
11383 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11384 ins_cost(1.9 * INSN_COST);
11385 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11386
11387 ins_encode %{
11388 __ ornw(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg),
11391 Assembler::ROR,
11392 $src3$$constant & 0x1f);
11393 %}
11394
11395 ins_pipe(ialu_reg_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400 // val | (-1 ^ (val ror shift)) ==> orn
11401 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11402 iRegL src1, iRegL src2,
11403 immI src3, immL_M1 src4) %{
11404 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11405 ins_cost(1.9 * INSN_COST);
11406 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11407
11408 ins_encode %{
11409 __ orn(as_Register($dst$$reg),
11410 as_Register($src1$$reg),
11411 as_Register($src2$$reg),
11412 Assembler::ROR,
11413 $src3$$constant & 0x3f);
11414 %}
11415
11416 ins_pipe(ialu_reg_reg_shift);
11417 %}
11418
11419 // This pattern is automatically generated from aarch64_ad.m4
11420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11421 // val | (-1 ^ (val << shift)) ==> ornw
11422 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11423 iRegIorL2I src1, iRegIorL2I src2,
11424 immI src3, immI_M1 src4) %{
11425 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11426 ins_cost(1.9 * INSN_COST);
11427 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11428
11429 ins_encode %{
11430 __ ornw(as_Register($dst$$reg),
11431 as_Register($src1$$reg),
11432 as_Register($src2$$reg),
11433 Assembler::LSL,
11434 $src3$$constant & 0x1f);
11435 %}
11436
11437 ins_pipe(ialu_reg_reg_shift);
11438 %}
11439
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442 // val | (-1 ^ (val << shift)) ==> orn
11443 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11444 iRegL src1, iRegL src2,
11445 immI src3, immL_M1 src4) %{
11446 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11447 ins_cost(1.9 * INSN_COST);
11448 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11449
11450 ins_encode %{
11451 __ orn(as_Register($dst$$reg),
11452 as_Register($src1$$reg),
11453 as_Register($src2$$reg),
11454 Assembler::LSL,
11455 $src3$$constant & 0x3f);
11456 %}
11457
11458 ins_pipe(ialu_reg_reg_shift);
11459 %}
11460
11461 // This pattern is automatically generated from aarch64_ad.m4
11462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11463 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11464 iRegIorL2I src1, iRegIorL2I src2,
11465 immI src3) %{
11466 match(Set dst (AndI src1 (URShiftI src2 src3)));
11467
11468 ins_cost(1.9 * INSN_COST);
11469 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11470
11471 ins_encode %{
11472 __ andw(as_Register($dst$$reg),
11473 as_Register($src1$$reg),
11474 as_Register($src2$$reg),
11475 Assembler::LSR,
11476 $src3$$constant & 0x1f);
11477 %}
11478
11479 ins_pipe(ialu_reg_reg_shift);
11480 %}
11481
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11485 iRegL src1, iRegL src2,
11486 immI src3) %{
11487 match(Set dst (AndL src1 (URShiftL src2 src3)));
11488
11489 ins_cost(1.9 * INSN_COST);
11490 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11491
11492 ins_encode %{
11493 __ andr(as_Register($dst$$reg),
11494 as_Register($src1$$reg),
11495 as_Register($src2$$reg),
11496 Assembler::LSR,
11497 $src3$$constant & 0x3f);
11498 %}
11499
11500 ins_pipe(ialu_reg_reg_shift);
11501 %}
11502
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11506 iRegIorL2I src1, iRegIorL2I src2,
11507 immI src3) %{
11508 match(Set dst (AndI src1 (RShiftI src2 src3)));
11509
11510 ins_cost(1.9 * INSN_COST);
11511 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11512
11513 ins_encode %{
11514 __ andw(as_Register($dst$$reg),
11515 as_Register($src1$$reg),
11516 as_Register($src2$$reg),
11517 Assembler::ASR,
11518 $src3$$constant & 0x1f);
11519 %}
11520
11521 ins_pipe(ialu_reg_reg_shift);
11522 %}
11523
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11527 iRegL src1, iRegL src2,
11528 immI src3) %{
11529 match(Set dst (AndL src1 (RShiftL src2 src3)));
11530
11531 ins_cost(1.9 * INSN_COST);
11532 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11533
11534 ins_encode %{
11535 __ andr(as_Register($dst$$reg),
11536 as_Register($src1$$reg),
11537 as_Register($src2$$reg),
11538 Assembler::ASR,
11539 $src3$$constant & 0x3f);
11540 %}
11541
11542 ins_pipe(ialu_reg_reg_shift);
11543 %}
11544
11545 // This pattern is automatically generated from aarch64_ad.m4
11546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11547 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11548 iRegIorL2I src1, iRegIorL2I src2,
11549 immI src3) %{
11550 match(Set dst (AndI src1 (LShiftI src2 src3)));
11551
11552 ins_cost(1.9 * INSN_COST);
11553 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11554
11555 ins_encode %{
11556 __ andw(as_Register($dst$$reg),
11557 as_Register($src1$$reg),
11558 as_Register($src2$$reg),
11559 Assembler::LSL,
11560 $src3$$constant & 0x1f);
11561 %}
11562
11563 ins_pipe(ialu_reg_reg_shift);
11564 %}
11565
11566 // This pattern is automatically generated from aarch64_ad.m4
11567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11568 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11569 iRegL src1, iRegL src2,
11570 immI src3) %{
11571 match(Set dst (AndL src1 (LShiftL src2 src3)));
11572
11573 ins_cost(1.9 * INSN_COST);
11574 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11575
11576 ins_encode %{
11577 __ andr(as_Register($dst$$reg),
11578 as_Register($src1$$reg),
11579 as_Register($src2$$reg),
11580 Assembler::LSL,
11581 $src3$$constant & 0x3f);
11582 %}
11583
11584 ins_pipe(ialu_reg_reg_shift);
11585 %}
11586
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11590 iRegIorL2I src1, iRegIorL2I src2,
11591 immI src3) %{
11592 match(Set dst (AndI src1 (RotateRight src2 src3)));
11593
11594 ins_cost(1.9 * INSN_COST);
11595 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11596
11597 ins_encode %{
11598 __ andw(as_Register($dst$$reg),
11599 as_Register($src1$$reg),
11600 as_Register($src2$$reg),
11601 Assembler::ROR,
11602 $src3$$constant & 0x1f);
11603 %}
11604
11605 ins_pipe(ialu_reg_reg_shift);
11606 %}
11607
11608 // This pattern is automatically generated from aarch64_ad.m4
11609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11610 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11611 iRegL src1, iRegL src2,
11612 immI src3) %{
11613 match(Set dst (AndL src1 (RotateRight src2 src3)));
11614
11615 ins_cost(1.9 * INSN_COST);
11616 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11617
11618 ins_encode %{
11619 __ andr(as_Register($dst$$reg),
11620 as_Register($src1$$reg),
11621 as_Register($src2$$reg),
11622 Assembler::ROR,
11623 $src3$$constant & 0x3f);
11624 %}
11625
11626 ins_pipe(ialu_reg_reg_shift);
11627 %}
11628
11629 // This pattern is automatically generated from aarch64_ad.m4
11630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11631 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11632 iRegIorL2I src1, iRegIorL2I src2,
11633 immI src3) %{
11634 match(Set dst (XorI src1 (URShiftI src2 src3)));
11635
11636 ins_cost(1.9 * INSN_COST);
11637 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11638
11639 ins_encode %{
11640 __ eorw(as_Register($dst$$reg),
11641 as_Register($src1$$reg),
11642 as_Register($src2$$reg),
11643 Assembler::LSR,
11644 $src3$$constant & 0x1f);
11645 %}
11646
11647 ins_pipe(ialu_reg_reg_shift);
11648 %}
11649
11650 // This pattern is automatically generated from aarch64_ad.m4
11651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11652 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11653 iRegL src1, iRegL src2,
11654 immI src3) %{
11655 match(Set dst (XorL src1 (URShiftL src2 src3)));
11656
11657 ins_cost(1.9 * INSN_COST);
11658 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11659
11660 ins_encode %{
11661 __ eor(as_Register($dst$$reg),
11662 as_Register($src1$$reg),
11663 as_Register($src2$$reg),
11664 Assembler::LSR,
11665 $src3$$constant & 0x3f);
11666 %}
11667
11668 ins_pipe(ialu_reg_reg_shift);
11669 %}
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11674 iRegIorL2I src1, iRegIorL2I src2,
11675 immI src3) %{
11676 match(Set dst (XorI src1 (RShiftI src2 src3)));
11677
11678 ins_cost(1.9 * INSN_COST);
11679 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11680
11681 ins_encode %{
11682 __ eorw(as_Register($dst$$reg),
11683 as_Register($src1$$reg),
11684 as_Register($src2$$reg),
11685 Assembler::ASR,
11686 $src3$$constant & 0x1f);
11687 %}
11688
11689 ins_pipe(ialu_reg_reg_shift);
11690 %}
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11695 iRegL src1, iRegL src2,
11696 immI src3) %{
11697 match(Set dst (XorL src1 (RShiftL src2 src3)));
11698
11699 ins_cost(1.9 * INSN_COST);
11700 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11701
11702 ins_encode %{
11703 __ eor(as_Register($dst$$reg),
11704 as_Register($src1$$reg),
11705 as_Register($src2$$reg),
11706 Assembler::ASR,
11707 $src3$$constant & 0x3f);
11708 %}
11709
11710 ins_pipe(ialu_reg_reg_shift);
11711 %}
11712
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11716 iRegIorL2I src1, iRegIorL2I src2,
11717 immI src3) %{
11718 match(Set dst (XorI src1 (LShiftI src2 src3)));
11719
11720 ins_cost(1.9 * INSN_COST);
11721 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11722
11723 ins_encode %{
11724 __ eorw(as_Register($dst$$reg),
11725 as_Register($src1$$reg),
11726 as_Register($src2$$reg),
11727 Assembler::LSL,
11728 $src3$$constant & 0x1f);
11729 %}
11730
11731 ins_pipe(ialu_reg_reg_shift);
11732 %}
11733
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11737 iRegL src1, iRegL src2,
11738 immI src3) %{
11739 match(Set dst (XorL src1 (LShiftL src2 src3)));
11740
11741 ins_cost(1.9 * INSN_COST);
11742 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11743
11744 ins_encode %{
11745 __ eor(as_Register($dst$$reg),
11746 as_Register($src1$$reg),
11747 as_Register($src2$$reg),
11748 Assembler::LSL,
11749 $src3$$constant & 0x3f);
11750 %}
11751
11752 ins_pipe(ialu_reg_reg_shift);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11758 iRegIorL2I src1, iRegIorL2I src2,
11759 immI src3) %{
11760 match(Set dst (XorI src1 (RotateRight src2 src3)));
11761
11762 ins_cost(1.9 * INSN_COST);
11763 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11764
11765 ins_encode %{
11766 __ eorw(as_Register($dst$$reg),
11767 as_Register($src1$$reg),
11768 as_Register($src2$$reg),
11769 Assembler::ROR,
11770 $src3$$constant & 0x1f);
11771 %}
11772
11773 ins_pipe(ialu_reg_reg_shift);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11779 iRegL src1, iRegL src2,
11780 immI src3) %{
11781 match(Set dst (XorL src1 (RotateRight src2 src3)));
11782
11783 ins_cost(1.9 * INSN_COST);
11784 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11785
11786 ins_encode %{
11787 __ eor(as_Register($dst$$reg),
11788 as_Register($src1$$reg),
11789 as_Register($src2$$reg),
11790 Assembler::ROR,
11791 $src3$$constant & 0x3f);
11792 %}
11793
11794 ins_pipe(ialu_reg_reg_shift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11800 iRegIorL2I src1, iRegIorL2I src2,
11801 immI src3) %{
11802 match(Set dst (OrI src1 (URShiftI src2 src3)));
11803
11804 ins_cost(1.9 * INSN_COST);
11805 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11806
11807 ins_encode %{
11808 __ orrw(as_Register($dst$$reg),
11809 as_Register($src1$$reg),
11810 as_Register($src2$$reg),
11811 Assembler::LSR,
11812 $src3$$constant & 0x1f);
11813 %}
11814
11815 ins_pipe(ialu_reg_reg_shift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11821 iRegL src1, iRegL src2,
11822 immI src3) %{
11823 match(Set dst (OrL src1 (URShiftL src2 src3)));
11824
11825 ins_cost(1.9 * INSN_COST);
11826 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11827
11828 ins_encode %{
11829 __ orr(as_Register($dst$$reg),
11830 as_Register($src1$$reg),
11831 as_Register($src2$$reg),
11832 Assembler::LSR,
11833 $src3$$constant & 0x3f);
11834 %}
11835
11836 ins_pipe(ialu_reg_reg_shift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11842 iRegIorL2I src1, iRegIorL2I src2,
11843 immI src3) %{
11844 match(Set dst (OrI src1 (RShiftI src2 src3)));
11845
11846 ins_cost(1.9 * INSN_COST);
11847 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11848
11849 ins_encode %{
11850 __ orrw(as_Register($dst$$reg),
11851 as_Register($src1$$reg),
11852 as_Register($src2$$reg),
11853 Assembler::ASR,
11854 $src3$$constant & 0x1f);
11855 %}
11856
11857 ins_pipe(ialu_reg_reg_shift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11863 iRegL src1, iRegL src2,
11864 immI src3) %{
11865 match(Set dst (OrL src1 (RShiftL src2 src3)));
11866
11867 ins_cost(1.9 * INSN_COST);
11868 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11869
11870 ins_encode %{
11871 __ orr(as_Register($dst$$reg),
11872 as_Register($src1$$reg),
11873 as_Register($src2$$reg),
11874 Assembler::ASR,
11875 $src3$$constant & 0x3f);
11876 %}
11877
11878 ins_pipe(ialu_reg_reg_shift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11884 iRegIorL2I src1, iRegIorL2I src2,
11885 immI src3) %{
11886 match(Set dst (OrI src1 (LShiftI src2 src3)));
11887
11888 ins_cost(1.9 * INSN_COST);
11889 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11890
11891 ins_encode %{
11892 __ orrw(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg),
11895 Assembler::LSL,
11896 $src3$$constant & 0x1f);
11897 %}
11898
11899 ins_pipe(ialu_reg_reg_shift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11905 iRegL src1, iRegL src2,
11906 immI src3) %{
11907 match(Set dst (OrL src1 (LShiftL src2 src3)));
11908
11909 ins_cost(1.9 * INSN_COST);
11910 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11911
11912 ins_encode %{
11913 __ orr(as_Register($dst$$reg),
11914 as_Register($src1$$reg),
11915 as_Register($src2$$reg),
11916 Assembler::LSL,
11917 $src3$$constant & 0x3f);
11918 %}
11919
11920 ins_pipe(ialu_reg_reg_shift);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11926 iRegIorL2I src1, iRegIorL2I src2,
11927 immI src3) %{
11928 match(Set dst (OrI src1 (RotateRight src2 src3)));
11929
11930 ins_cost(1.9 * INSN_COST);
11931 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11932
11933 ins_encode %{
11934 __ orrw(as_Register($dst$$reg),
11935 as_Register($src1$$reg),
11936 as_Register($src2$$reg),
11937 Assembler::ROR,
11938 $src3$$constant & 0x1f);
11939 %}
11940
11941 ins_pipe(ialu_reg_reg_shift);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11947 iRegL src1, iRegL src2,
11948 immI src3) %{
11949 match(Set dst (OrL src1 (RotateRight src2 src3)));
11950
11951 ins_cost(1.9 * INSN_COST);
11952 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11953
11954 ins_encode %{
11955 __ orr(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 as_Register($src2$$reg),
11958 Assembler::ROR,
11959 $src3$$constant & 0x3f);
11960 %}
11961
11962 ins_pipe(ialu_reg_reg_shift);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11968 iRegIorL2I src1, iRegIorL2I src2,
11969 immI src3) %{
11970 match(Set dst (AddI src1 (URShiftI src2 src3)));
11971
11972 ins_cost(1.9 * INSN_COST);
11973 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11974
11975 ins_encode %{
11976 __ addw(as_Register($dst$$reg),
11977 as_Register($src1$$reg),
11978 as_Register($src2$$reg),
11979 Assembler::LSR,
11980 $src3$$constant & 0x1f);
11981 %}
11982
11983 ins_pipe(ialu_reg_reg_shift);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11989 iRegL src1, iRegL src2,
11990 immI src3) %{
11991 match(Set dst (AddL src1 (URShiftL src2 src3)));
11992
11993 ins_cost(1.9 * INSN_COST);
11994 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11995
11996 ins_encode %{
11997 __ add(as_Register($dst$$reg),
11998 as_Register($src1$$reg),
11999 as_Register($src2$$reg),
12000 Assembler::LSR,
12001 $src3$$constant & 0x3f);
12002 %}
12003
12004 ins_pipe(ialu_reg_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12010 iRegIorL2I src1, iRegIorL2I src2,
12011 immI src3) %{
12012 match(Set dst (AddI src1 (RShiftI src2 src3)));
12013
12014 ins_cost(1.9 * INSN_COST);
12015 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12016
12017 ins_encode %{
12018 __ addw(as_Register($dst$$reg),
12019 as_Register($src1$$reg),
12020 as_Register($src2$$reg),
12021 Assembler::ASR,
12022 $src3$$constant & 0x1f);
12023 %}
12024
12025 ins_pipe(ialu_reg_reg_shift);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12031 iRegL src1, iRegL src2,
12032 immI src3) %{
12033 match(Set dst (AddL src1 (RShiftL src2 src3)));
12034
12035 ins_cost(1.9 * INSN_COST);
12036 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12037
12038 ins_encode %{
12039 __ add(as_Register($dst$$reg),
12040 as_Register($src1$$reg),
12041 as_Register($src2$$reg),
12042 Assembler::ASR,
12043 $src3$$constant & 0x3f);
12044 %}
12045
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12052 iRegIorL2I src1, iRegIorL2I src2,
12053 immI src3) %{
12054 match(Set dst (AddI src1 (LShiftI src2 src3)));
12055
12056 ins_cost(1.9 * INSN_COST);
12057 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12058
12059 ins_encode %{
12060 __ addw(as_Register($dst$$reg),
12061 as_Register($src1$$reg),
12062 as_Register($src2$$reg),
12063 Assembler::LSL,
12064 $src3$$constant & 0x1f);
12065 %}
12066
12067 ins_pipe(ialu_reg_reg_shift);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12073 iRegL src1, iRegL src2,
12074 immI src3) %{
12075 match(Set dst (AddL src1 (LShiftL src2 src3)));
12076
12077 ins_cost(1.9 * INSN_COST);
12078 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12079
12080 ins_encode %{
12081 __ add(as_Register($dst$$reg),
12082 as_Register($src1$$reg),
12083 as_Register($src2$$reg),
12084 Assembler::LSL,
12085 $src3$$constant & 0x3f);
12086 %}
12087
12088 ins_pipe(ialu_reg_reg_shift);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12094 iRegIorL2I src1, iRegIorL2I src2,
12095 immI src3) %{
12096 match(Set dst (SubI src1 (URShiftI src2 src3)));
12097
12098 ins_cost(1.9 * INSN_COST);
12099 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12100
12101 ins_encode %{
12102 __ subw(as_Register($dst$$reg),
12103 as_Register($src1$$reg),
12104 as_Register($src2$$reg),
12105 Assembler::LSR,
12106 $src3$$constant & 0x1f);
12107 %}
12108
12109 ins_pipe(ialu_reg_reg_shift);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12115 iRegL src1, iRegL src2,
12116 immI src3) %{
12117 match(Set dst (SubL src1 (URShiftL src2 src3)));
12118
12119 ins_cost(1.9 * INSN_COST);
12120 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12121
12122 ins_encode %{
12123 __ sub(as_Register($dst$$reg),
12124 as_Register($src1$$reg),
12125 as_Register($src2$$reg),
12126 Assembler::LSR,
12127 $src3$$constant & 0x3f);
12128 %}
12129
12130 ins_pipe(ialu_reg_reg_shift);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12136 iRegIorL2I src1, iRegIorL2I src2,
12137 immI src3) %{
12138 match(Set dst (SubI src1 (RShiftI src2 src3)));
12139
12140 ins_cost(1.9 * INSN_COST);
12141 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12142
12143 ins_encode %{
12144 __ subw(as_Register($dst$$reg),
12145 as_Register($src1$$reg),
12146 as_Register($src2$$reg),
12147 Assembler::ASR,
12148 $src3$$constant & 0x1f);
12149 %}
12150
12151 ins_pipe(ialu_reg_reg_shift);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12157 iRegL src1, iRegL src2,
12158 immI src3) %{
12159 match(Set dst (SubL src1 (RShiftL src2 src3)));
12160
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12163
12164 ins_encode %{
12165 __ sub(as_Register($dst$$reg),
12166 as_Register($src1$$reg),
12167 as_Register($src2$$reg),
12168 Assembler::ASR,
12169 $src3$$constant & 0x3f);
12170 %}
12171
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12178 iRegIorL2I src1, iRegIorL2I src2,
12179 immI src3) %{
12180 match(Set dst (SubI src1 (LShiftI src2 src3)));
12181
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12184
12185 ins_encode %{
12186 __ subw(as_Register($dst$$reg),
12187 as_Register($src1$$reg),
12188 as_Register($src2$$reg),
12189 Assembler::LSL,
12190 $src3$$constant & 0x1f);
12191 %}
12192
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12199 iRegL src1, iRegL src2,
12200 immI src3) %{
12201 match(Set dst (SubL src1 (LShiftL src2 src3)));
12202
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12205
12206 ins_encode %{
12207 __ sub(as_Register($dst$$reg),
12208 as_Register($src1$$reg),
12209 as_Register($src2$$reg),
12210 Assembler::LSL,
12211 $src3$$constant & 0x3f);
12212 %}
12213
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219
12220 // Shift Left followed by Shift Right.
12221 // This idiom is used by the compiler for the i2b bytecode etc.
12222 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12223 %{
12224 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12225 ins_cost(INSN_COST * 2);
12226 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12227 ins_encode %{
12228 int lshift = $lshift_count$$constant & 63;
12229 int rshift = $rshift_count$$constant & 63;
12230 int s = 63 - lshift;
12231 int r = (rshift - lshift) & 63;
12232 __ sbfm(as_Register($dst$$reg),
12233 as_Register($src$$reg),
12234 r, s);
12235 %}
12236
12237 ins_pipe(ialu_reg_shift);
12238 %}
12239
12240 // This pattern is automatically generated from aarch64_ad.m4
12241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12242
12243 // Shift Left followed by Shift Right.
12244 // This idiom is used by the compiler for the i2b bytecode etc.
12245 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12246 %{
12247 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12248 ins_cost(INSN_COST * 2);
12249 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12250 ins_encode %{
12251 int lshift = $lshift_count$$constant & 31;
12252 int rshift = $rshift_count$$constant & 31;
12253 int s = 31 - lshift;
12254 int r = (rshift - lshift) & 31;
12255 __ sbfmw(as_Register($dst$$reg),
12256 as_Register($src$$reg),
12257 r, s);
12258 %}
12259
12260 ins_pipe(ialu_reg_shift);
12261 %}
12262
12263 // This pattern is automatically generated from aarch64_ad.m4
12264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12265
12266 // Shift Left followed by Shift Right.
12267 // This idiom is used by the compiler for the i2b bytecode etc.
12268 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12269 %{
12270 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12271 ins_cost(INSN_COST * 2);
12272 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12273 ins_encode %{
12274 int lshift = $lshift_count$$constant & 63;
12275 int rshift = $rshift_count$$constant & 63;
12276 int s = 63 - lshift;
12277 int r = (rshift - lshift) & 63;
12278 __ ubfm(as_Register($dst$$reg),
12279 as_Register($src$$reg),
12280 r, s);
12281 %}
12282
12283 ins_pipe(ialu_reg_shift);
12284 %}
12285
12286 // This pattern is automatically generated from aarch64_ad.m4
12287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12288
12289 // Shift Left followed by Shift Right.
12290 // This idiom is used by the compiler for the i2b bytecode etc.
12291 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12292 %{
12293 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12294 ins_cost(INSN_COST * 2);
12295 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12296 ins_encode %{
12297 int lshift = $lshift_count$$constant & 31;
12298 int rshift = $rshift_count$$constant & 31;
12299 int s = 31 - lshift;
12300 int r = (rshift - lshift) & 31;
12301 __ ubfmw(as_Register($dst$$reg),
12302 as_Register($src$$reg),
12303 r, s);
12304 %}
12305
12306 ins_pipe(ialu_reg_shift);
12307 %}
12308
12309 // Bitfield extract with shift & mask
12310
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12314 %{
12315 match(Set dst (AndI (URShiftI src rshift) mask));
12316 // Make sure we are not going to exceed what ubfxw can do.
12317 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12318
12319 ins_cost(INSN_COST);
12320 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12321 ins_encode %{
12322 int rshift = $rshift$$constant & 31;
12323 intptr_t mask = $mask$$constant;
12324 int width = exact_log2(mask+1);
12325 __ ubfxw(as_Register($dst$$reg),
12326 as_Register($src$$reg), rshift, width);
12327 %}
12328 ins_pipe(ialu_reg_shift);
12329 %}
12330
12331 // This pattern is automatically generated from aarch64_ad.m4
12332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12333 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12334 %{
12335 match(Set dst (AndL (URShiftL src rshift) mask));
12336 // Make sure we are not going to exceed what ubfx can do.
12337 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12338
12339 ins_cost(INSN_COST);
12340 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12341 ins_encode %{
12342 int rshift = $rshift$$constant & 63;
12343 intptr_t mask = $mask$$constant;
12344 int width = exact_log2_long(mask+1);
12345 __ ubfx(as_Register($dst$$reg),
12346 as_Register($src$$reg), rshift, width);
12347 %}
12348 ins_pipe(ialu_reg_shift);
12349 %}
12350
12351
12352 // This pattern is automatically generated from aarch64_ad.m4
12353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12354
12355 // We can use ubfx when extending an And with a mask when we know mask
12356 // is positive. We know that because immI_bitmask guarantees it.
12357 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12358 %{
12359 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12360 // Make sure we are not going to exceed what ubfxw can do.
12361 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12362
12363 ins_cost(INSN_COST * 2);
12364 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12365 ins_encode %{
12366 int rshift = $rshift$$constant & 31;
12367 intptr_t mask = $mask$$constant;
12368 int width = exact_log2(mask+1);
12369 __ ubfx(as_Register($dst$$reg),
12370 as_Register($src$$reg), rshift, width);
12371 %}
12372 ins_pipe(ialu_reg_shift);
12373 %}
12374
12375
12376 // This pattern is automatically generated from aarch64_ad.m4
12377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12378
12379 // We can use ubfiz when masking by a positive number and then left shifting the result.
12380 // We know that the mask is positive because immI_bitmask guarantees it.
12381 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12382 %{
12383 match(Set dst (LShiftI (AndI src mask) lshift));
12384 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12385
12386 ins_cost(INSN_COST);
12387 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12388 ins_encode %{
12389 int lshift = $lshift$$constant & 31;
12390 intptr_t mask = $mask$$constant;
12391 int width = exact_log2(mask+1);
12392 __ ubfizw(as_Register($dst$$reg),
12393 as_Register($src$$reg), lshift, width);
12394 %}
12395 ins_pipe(ialu_reg_shift);
12396 %}
12397
12398 // This pattern is automatically generated from aarch64_ad.m4
12399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12400
12401 // We can use ubfiz when masking by a positive number and then left shifting the result.
12402 // We know that the mask is positive because immL_bitmask guarantees it.
12403 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12404 %{
12405 match(Set dst (LShiftL (AndL src mask) lshift));
12406 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12407
12408 ins_cost(INSN_COST);
12409 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12410 ins_encode %{
12411 int lshift = $lshift$$constant & 63;
12412 intptr_t mask = $mask$$constant;
12413 int width = exact_log2_long(mask+1);
12414 __ ubfiz(as_Register($dst$$reg),
12415 as_Register($src$$reg), lshift, width);
12416 %}
12417 ins_pipe(ialu_reg_shift);
12418 %}
12419
12420 // This pattern is automatically generated from aarch64_ad.m4
12421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12422
12423 // We can use ubfiz when masking by a positive number and then left shifting the result.
12424 // We know that the mask is positive because immI_bitmask guarantees it.
12425 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12426 %{
12427 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12428 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12429
12430 ins_cost(INSN_COST);
12431 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12432 ins_encode %{
12433 int lshift = $lshift$$constant & 31;
12434 intptr_t mask = $mask$$constant;
12435 int width = exact_log2(mask+1);
12436 __ ubfizw(as_Register($dst$$reg),
12437 as_Register($src$$reg), lshift, width);
12438 %}
12439 ins_pipe(ialu_reg_shift);
12440 %}
12441
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444
12445 // We can use ubfiz when masking by a positive number and then left shifting the result.
12446 // We know that the mask is positive because immL_bitmask guarantees it.
12447 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12448 %{
12449 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12450 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12451
12452 ins_cost(INSN_COST);
12453 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12454 ins_encode %{
12455 int lshift = $lshift$$constant & 63;
12456 intptr_t mask = $mask$$constant;
12457 int width = exact_log2_long(mask+1);
12458 __ ubfiz(as_Register($dst$$reg),
12459 as_Register($src$$reg), lshift, width);
12460 %}
12461 ins_pipe(ialu_reg_shift);
12462 %}
12463
12464
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467
12468 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12469 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12470 %{
12471 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12472 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12473
12474 ins_cost(INSN_COST);
12475 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12476 ins_encode %{
12477 int lshift = $lshift$$constant & 63;
12478 intptr_t mask = $mask$$constant;
12479 int width = exact_log2(mask+1);
12480 __ ubfiz(as_Register($dst$$reg),
12481 as_Register($src$$reg), lshift, width);
12482 %}
12483 ins_pipe(ialu_reg_shift);
12484 %}
12485
12486 // This pattern is automatically generated from aarch64_ad.m4
12487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12488
12489 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12490 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12491 %{
12492 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12493 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12494
12495 ins_cost(INSN_COST);
12496 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12497 ins_encode %{
12498 int lshift = $lshift$$constant & 31;
12499 intptr_t mask = $mask$$constant;
12500 int width = exact_log2(mask+1);
12501 __ ubfiz(as_Register($dst$$reg),
12502 as_Register($src$$reg), lshift, width);
12503 %}
12504 ins_pipe(ialu_reg_shift);
12505 %}
12506
12507 // This pattern is automatically generated from aarch64_ad.m4
12508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12509
12510 // Can skip int2long conversions after AND with small bitmask
12511 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12512 %{
12513 match(Set dst (ConvI2L (AndI src msk)));
12514 ins_cost(INSN_COST);
12515 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12516 ins_encode %{
12517 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12518 %}
12519 ins_pipe(ialu_reg_shift);
12520 %}
12521
12522
12523 // Rotations
12524
12525 // This pattern is automatically generated from aarch64_ad.m4
12526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12527 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12528 %{
12529 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12530 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12531
12532 ins_cost(INSN_COST);
12533 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12534
12535 ins_encode %{
12536 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12537 $rshift$$constant & 63);
12538 %}
12539 ins_pipe(ialu_reg_reg_extr);
12540 %}
12541
12542
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12546 %{
12547 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12548 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12549
12550 ins_cost(INSN_COST);
12551 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12552
12553 ins_encode %{
12554 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12555 $rshift$$constant & 31);
12556 %}
12557 ins_pipe(ialu_reg_reg_extr);
12558 %}
12559
12560
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12564 %{
12565 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12566 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12567
12568 ins_cost(INSN_COST);
12569 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12570
12571 ins_encode %{
12572 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12573 $rshift$$constant & 63);
12574 %}
12575 ins_pipe(ialu_reg_reg_extr);
12576 %}
12577
12578
12579 // This pattern is automatically generated from aarch64_ad.m4
12580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12581 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12582 %{
12583 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12584 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12585
12586 ins_cost(INSN_COST);
12587 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12588
12589 ins_encode %{
12590 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12591 $rshift$$constant & 31);
12592 %}
12593 ins_pipe(ialu_reg_reg_extr);
12594 %}
12595
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12599 %{
12600 match(Set dst (RotateRight src shift));
12601
12602 ins_cost(INSN_COST);
12603 format %{ "ror $dst, $src, $shift" %}
12604
12605 ins_encode %{
12606 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12607 $shift$$constant & 0x1f);
12608 %}
12609 ins_pipe(ialu_reg_reg_vshift);
12610 %}
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12615 %{
12616 match(Set dst (RotateRight src shift));
12617
12618 ins_cost(INSN_COST);
12619 format %{ "ror $dst, $src, $shift" %}
12620
12621 ins_encode %{
12622 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12623 $shift$$constant & 0x3f);
12624 %}
12625 ins_pipe(ialu_reg_reg_vshift);
12626 %}
12627
12628 // This pattern is automatically generated from aarch64_ad.m4
12629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12630 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12631 %{
12632 match(Set dst (RotateRight src shift));
12633
12634 ins_cost(INSN_COST);
12635 format %{ "ror $dst, $src, $shift" %}
12636
12637 ins_encode %{
12638 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12639 %}
12640 ins_pipe(ialu_reg_reg_vshift);
12641 %}
12642
12643 // This pattern is automatically generated from aarch64_ad.m4
12644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12645 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12646 %{
12647 match(Set dst (RotateRight src shift));
12648
12649 ins_cost(INSN_COST);
12650 format %{ "ror $dst, $src, $shift" %}
12651
12652 ins_encode %{
12653 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12654 %}
12655 ins_pipe(ialu_reg_reg_vshift);
12656 %}
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12661 %{
12662 match(Set dst (RotateLeft src shift));
12663
12664 ins_cost(INSN_COST);
12665 format %{ "rol $dst, $src, $shift" %}
12666
12667 ins_encode %{
12668 __ subw(rscratch1, zr, as_Register($shift$$reg));
12669 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12670 %}
12671 ins_pipe(ialu_reg_reg_vshift);
12672 %}
12673
12674 // This pattern is automatically generated from aarch64_ad.m4
12675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12676 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12677 %{
12678 match(Set dst (RotateLeft src shift));
12679
12680 ins_cost(INSN_COST);
12681 format %{ "rol $dst, $src, $shift" %}
12682
12683 ins_encode %{
12684 __ subw(rscratch1, zr, as_Register($shift$$reg));
12685 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12686 %}
12687 ins_pipe(ialu_reg_reg_vshift);
12688 %}
12689
12690
12691 // Add/subtract (extended)
12692
12693 // This pattern is automatically generated from aarch64_ad.m4
12694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12695 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12696 %{
12697 match(Set dst (AddL src1 (ConvI2L src2)));
12698 ins_cost(INSN_COST);
12699 format %{ "add $dst, $src1, $src2, sxtw" %}
12700
12701 ins_encode %{
12702 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12703 as_Register($src2$$reg), ext::sxtw);
12704 %}
12705 ins_pipe(ialu_reg_reg);
12706 %}
12707
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12711 %{
12712 match(Set dst (SubL src1 (ConvI2L src2)));
12713 ins_cost(INSN_COST);
12714 format %{ "sub $dst, $src1, $src2, sxtw" %}
12715
12716 ins_encode %{
12717 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12718 as_Register($src2$$reg), ext::sxtw);
12719 %}
12720 ins_pipe(ialu_reg_reg);
12721 %}
12722
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12726 %{
12727 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12728 ins_cost(INSN_COST);
12729 format %{ "add $dst, $src1, $src2, sxth" %}
12730
12731 ins_encode %{
12732 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12733 as_Register($src2$$reg), ext::sxth);
12734 %}
12735 ins_pipe(ialu_reg_reg);
12736 %}
12737
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12741 %{
12742 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12743 ins_cost(INSN_COST);
12744 format %{ "add $dst, $src1, $src2, sxtb" %}
12745
12746 ins_encode %{
12747 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12748 as_Register($src2$$reg), ext::sxtb);
12749 %}
12750 ins_pipe(ialu_reg_reg);
12751 %}
12752
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12756 %{
12757 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12758 ins_cost(INSN_COST);
12759 format %{ "add $dst, $src1, $src2, uxtb" %}
12760
12761 ins_encode %{
12762 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12763 as_Register($src2$$reg), ext::uxtb);
12764 %}
12765 ins_pipe(ialu_reg_reg);
12766 %}
12767
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12771 %{
12772 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12773 ins_cost(INSN_COST);
12774 format %{ "add $dst, $src1, $src2, sxth" %}
12775
12776 ins_encode %{
12777 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12778 as_Register($src2$$reg), ext::sxth);
12779 %}
12780 ins_pipe(ialu_reg_reg);
12781 %}
12782
12783 // This pattern is automatically generated from aarch64_ad.m4
12784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12785 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12786 %{
12787 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12788 ins_cost(INSN_COST);
12789 format %{ "add $dst, $src1, $src2, sxtw" %}
12790
12791 ins_encode %{
12792 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12793 as_Register($src2$$reg), ext::sxtw);
12794 %}
12795 ins_pipe(ialu_reg_reg);
12796 %}
12797
12798 // This pattern is automatically generated from aarch64_ad.m4
12799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12800 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12801 %{
12802 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12803 ins_cost(INSN_COST);
12804 format %{ "add $dst, $src1, $src2, sxtb" %}
12805
12806 ins_encode %{
12807 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12808 as_Register($src2$$reg), ext::sxtb);
12809 %}
12810 ins_pipe(ialu_reg_reg);
12811 %}
12812
12813 // This pattern is automatically generated from aarch64_ad.m4
12814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12815 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12816 %{
12817 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12818 ins_cost(INSN_COST);
12819 format %{ "add $dst, $src1, $src2, uxtb" %}
12820
12821 ins_encode %{
12822 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12823 as_Register($src2$$reg), ext::uxtb);
12824 %}
12825 ins_pipe(ialu_reg_reg);
12826 %}
12827
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12831 %{
12832 match(Set dst (AddI src1 (AndI src2 mask)));
12833 ins_cost(INSN_COST);
12834 format %{ "addw $dst, $src1, $src2, uxtb" %}
12835
12836 ins_encode %{
12837 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12838 as_Register($src2$$reg), ext::uxtb);
12839 %}
12840 ins_pipe(ialu_reg_reg);
12841 %}
12842
12843 // This pattern is automatically generated from aarch64_ad.m4
12844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12845 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12846 %{
12847 match(Set dst (AddI src1 (AndI src2 mask)));
12848 ins_cost(INSN_COST);
12849 format %{ "addw $dst, $src1, $src2, uxth" %}
12850
12851 ins_encode %{
12852 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12853 as_Register($src2$$reg), ext::uxth);
12854 %}
12855 ins_pipe(ialu_reg_reg);
12856 %}
12857
12858 // This pattern is automatically generated from aarch64_ad.m4
12859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12860 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12861 %{
12862 match(Set dst (AddL src1 (AndL src2 mask)));
12863 ins_cost(INSN_COST);
12864 format %{ "add $dst, $src1, $src2, uxtb" %}
12865
12866 ins_encode %{
12867 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12868 as_Register($src2$$reg), ext::uxtb);
12869 %}
12870 ins_pipe(ialu_reg_reg);
12871 %}
12872
12873 // This pattern is automatically generated from aarch64_ad.m4
12874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12875 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12876 %{
12877 match(Set dst (AddL src1 (AndL src2 mask)));
12878 ins_cost(INSN_COST);
12879 format %{ "add $dst, $src1, $src2, uxth" %}
12880
12881 ins_encode %{
12882 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12883 as_Register($src2$$reg), ext::uxth);
12884 %}
12885 ins_pipe(ialu_reg_reg);
12886 %}
12887
12888 // This pattern is automatically generated from aarch64_ad.m4
12889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12890 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12891 %{
12892 match(Set dst (AddL src1 (AndL src2 mask)));
12893 ins_cost(INSN_COST);
12894 format %{ "add $dst, $src1, $src2, uxtw" %}
12895
12896 ins_encode %{
12897 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12898 as_Register($src2$$reg), ext::uxtw);
12899 %}
12900 ins_pipe(ialu_reg_reg);
12901 %}
12902
12903 // This pattern is automatically generated from aarch64_ad.m4
12904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12905 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12906 %{
12907 match(Set dst (SubI src1 (AndI src2 mask)));
12908 ins_cost(INSN_COST);
12909 format %{ "subw $dst, $src1, $src2, uxtb" %}
12910
12911 ins_encode %{
12912 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12913 as_Register($src2$$reg), ext::uxtb);
12914 %}
12915 ins_pipe(ialu_reg_reg);
12916 %}
12917
12918 // This pattern is automatically generated from aarch64_ad.m4
12919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12920 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12921 %{
12922 match(Set dst (SubI src1 (AndI src2 mask)));
12923 ins_cost(INSN_COST);
12924 format %{ "subw $dst, $src1, $src2, uxth" %}
12925
12926 ins_encode %{
12927 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12928 as_Register($src2$$reg), ext::uxth);
12929 %}
12930 ins_pipe(ialu_reg_reg);
12931 %}
12932
12933 // This pattern is automatically generated from aarch64_ad.m4
12934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12935 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12936 %{
12937 match(Set dst (SubL src1 (AndL src2 mask)));
12938 ins_cost(INSN_COST);
12939 format %{ "sub $dst, $src1, $src2, uxtb" %}
12940
12941 ins_encode %{
12942 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12943 as_Register($src2$$reg), ext::uxtb);
12944 %}
12945 ins_pipe(ialu_reg_reg);
12946 %}
12947
12948 // This pattern is automatically generated from aarch64_ad.m4
12949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12950 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12951 %{
12952 match(Set dst (SubL src1 (AndL src2 mask)));
12953 ins_cost(INSN_COST);
12954 format %{ "sub $dst, $src1, $src2, uxth" %}
12955
12956 ins_encode %{
12957 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12958 as_Register($src2$$reg), ext::uxth);
12959 %}
12960 ins_pipe(ialu_reg_reg);
12961 %}
12962
12963 // This pattern is automatically generated from aarch64_ad.m4
12964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12965 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12966 %{
12967 match(Set dst (SubL src1 (AndL src2 mask)));
12968 ins_cost(INSN_COST);
12969 format %{ "sub $dst, $src1, $src2, uxtw" %}
12970
12971 ins_encode %{
12972 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12973 as_Register($src2$$reg), ext::uxtw);
12974 %}
12975 ins_pipe(ialu_reg_reg);
12976 %}
12977
12978
12979 // This pattern is automatically generated from aarch64_ad.m4
12980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12981 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12982 %{
12983 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12984 ins_cost(1.9 * INSN_COST);
12985 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12986
12987 ins_encode %{
12988 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12989 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12990 %}
12991 ins_pipe(ialu_reg_reg_shift);
12992 %}
12993
12994 // This pattern is automatically generated from aarch64_ad.m4
12995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12996 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12997 %{
12998 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12999 ins_cost(1.9 * INSN_COST);
13000 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13001
13002 ins_encode %{
13003 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13004 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13005 %}
13006 ins_pipe(ialu_reg_reg_shift);
13007 %}
13008
13009 // This pattern is automatically generated from aarch64_ad.m4
13010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13011 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13012 %{
13013 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13014 ins_cost(1.9 * INSN_COST);
13015 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13016
13017 ins_encode %{
13018 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13019 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13020 %}
13021 ins_pipe(ialu_reg_reg_shift);
13022 %}
13023
13024 // This pattern is automatically generated from aarch64_ad.m4
13025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13026 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13027 %{
13028 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13029 ins_cost(1.9 * INSN_COST);
13030 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13031
13032 ins_encode %{
13033 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13034 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13035 %}
13036 ins_pipe(ialu_reg_reg_shift);
13037 %}
13038
13039 // This pattern is automatically generated from aarch64_ad.m4
13040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13041 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13042 %{
13043 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13044 ins_cost(1.9 * INSN_COST);
13045 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13046
13047 ins_encode %{
13048 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13049 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13050 %}
13051 ins_pipe(ialu_reg_reg_shift);
13052 %}
13053
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13057 %{
13058 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13059 ins_cost(1.9 * INSN_COST);
13060 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13061
13062 ins_encode %{
13063 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13064 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13065 %}
13066 ins_pipe(ialu_reg_reg_shift);
13067 %}
13068
13069 // This pattern is automatically generated from aarch64_ad.m4
13070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13071 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13072 %{
13073 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13074 ins_cost(1.9 * INSN_COST);
13075 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13076
13077 ins_encode %{
13078 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13079 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13080 %}
13081 ins_pipe(ialu_reg_reg_shift);
13082 %}
13083
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13087 %{
13088 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13089 ins_cost(1.9 * INSN_COST);
13090 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13091
13092 ins_encode %{
13093 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13094 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13095 %}
13096 ins_pipe(ialu_reg_reg_shift);
13097 %}
13098
13099 // This pattern is automatically generated from aarch64_ad.m4
13100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13101 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13102 %{
13103 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13104 ins_cost(1.9 * INSN_COST);
13105 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13106
13107 ins_encode %{
13108 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13109 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13110 %}
13111 ins_pipe(ialu_reg_reg_shift);
13112 %}
13113
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13117 %{
13118 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13119 ins_cost(1.9 * INSN_COST);
13120 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13121
13122 ins_encode %{
13123 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13124 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13125 %}
13126 ins_pipe(ialu_reg_reg_shift);
13127 %}
13128
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13132 %{
13133 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13134 ins_cost(1.9 * INSN_COST);
13135 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13136
13137 ins_encode %{
13138 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13139 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13140 %}
13141 ins_pipe(ialu_reg_reg_shift);
13142 %}
13143
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13147 %{
13148 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13149 ins_cost(1.9 * INSN_COST);
13150 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13151
13152 ins_encode %{
13153 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13154 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13155 %}
13156 ins_pipe(ialu_reg_reg_shift);
13157 %}
13158
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13162 %{
13163 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13164 ins_cost(1.9 * INSN_COST);
13165 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13166
13167 ins_encode %{
13168 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13170 %}
13171 ins_pipe(ialu_reg_reg_shift);
13172 %}
13173
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13177 %{
13178 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13179 ins_cost(1.9 * INSN_COST);
13180 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13181
13182 ins_encode %{
13183 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13184 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13185 %}
13186 ins_pipe(ialu_reg_reg_shift);
13187 %}
13188
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13192 %{
13193 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13194 ins_cost(1.9 * INSN_COST);
13195 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13196
13197 ins_encode %{
13198 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13199 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13200 %}
13201 ins_pipe(ialu_reg_reg_shift);
13202 %}
13203
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13207 %{
13208 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13209 ins_cost(1.9 * INSN_COST);
13210 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13211
13212 ins_encode %{
13213 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13214 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13215 %}
13216 ins_pipe(ialu_reg_reg_shift);
13217 %}
13218
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13222 %{
13223 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13224 ins_cost(1.9 * INSN_COST);
13225 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13226
13227 ins_encode %{
13228 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13229 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13230 %}
13231 ins_pipe(ialu_reg_reg_shift);
13232 %}
13233
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13237 %{
13238 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13239 ins_cost(1.9 * INSN_COST);
13240 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13241
13242 ins_encode %{
13243 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13244 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13245 %}
13246 ins_pipe(ialu_reg_reg_shift);
13247 %}
13248
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13252 %{
13253 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13254 ins_cost(1.9 * INSN_COST);
13255 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13256
13257 ins_encode %{
13258 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13259 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13260 %}
13261 ins_pipe(ialu_reg_reg_shift);
13262 %}
13263
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13267 %{
13268 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13269 ins_cost(1.9 * INSN_COST);
13270 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13271
13272 ins_encode %{
13273 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13274 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13275 %}
13276 ins_pipe(ialu_reg_reg_shift);
13277 %}
13278
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13282 %{
13283 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13284 ins_cost(1.9 * INSN_COST);
13285 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13286
13287 ins_encode %{
13288 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13289 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13290 %}
13291 ins_pipe(ialu_reg_reg_shift);
13292 %}
13293
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13297 %{
13298 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13299 ins_cost(1.9 * INSN_COST);
13300 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13301
13302 ins_encode %{
13303 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13304 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13305 %}
13306 ins_pipe(ialu_reg_reg_shift);
13307 %}
13308
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13312 %{
13313 effect(DEF dst, USE src1, USE src2, USE cr);
13314 ins_cost(INSN_COST * 2);
13315 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13316
13317 ins_encode %{
13318 __ cselw($dst$$Register,
13319 $src1$$Register,
13320 $src2$$Register,
13321 Assembler::LT);
13322 %}
13323 ins_pipe(icond_reg_reg);
13324 %}
13325
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13329 %{
13330 effect(DEF dst, USE src1, USE src2, USE cr);
13331 ins_cost(INSN_COST * 2);
13332 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13333
13334 ins_encode %{
13335 __ cselw($dst$$Register,
13336 $src1$$Register,
13337 $src2$$Register,
13338 Assembler::GT);
13339 %}
13340 ins_pipe(icond_reg_reg);
13341 %}
13342
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13346 %{
13347 effect(DEF dst, USE src1, USE cr);
13348 ins_cost(INSN_COST * 2);
13349 format %{ "cselw $dst, $src1, zr lt\t" %}
13350
13351 ins_encode %{
13352 __ cselw($dst$$Register,
13353 $src1$$Register,
13354 zr,
13355 Assembler::LT);
13356 %}
13357 ins_pipe(icond_reg);
13358 %}
13359
13360 // This pattern is automatically generated from aarch64_ad.m4
13361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13362 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13363 %{
13364 effect(DEF dst, USE src1, USE cr);
13365 ins_cost(INSN_COST * 2);
13366 format %{ "cselw $dst, $src1, zr gt\t" %}
13367
13368 ins_encode %{
13369 __ cselw($dst$$Register,
13370 $src1$$Register,
13371 zr,
13372 Assembler::GT);
13373 %}
13374 ins_pipe(icond_reg);
13375 %}
13376
13377 // This pattern is automatically generated from aarch64_ad.m4
13378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13379 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13380 %{
13381 effect(DEF dst, USE src1, USE cr);
13382 ins_cost(INSN_COST * 2);
13383 format %{ "csincw $dst, $src1, zr le\t" %}
13384
13385 ins_encode %{
13386 __ csincw($dst$$Register,
13387 $src1$$Register,
13388 zr,
13389 Assembler::LE);
13390 %}
13391 ins_pipe(icond_reg);
13392 %}
13393
13394 // This pattern is automatically generated from aarch64_ad.m4
13395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13396 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13397 %{
13398 effect(DEF dst, USE src1, USE cr);
13399 ins_cost(INSN_COST * 2);
13400 format %{ "csincw $dst, $src1, zr gt\t" %}
13401
13402 ins_encode %{
13403 __ csincw($dst$$Register,
13404 $src1$$Register,
13405 zr,
13406 Assembler::GT);
13407 %}
13408 ins_pipe(icond_reg);
13409 %}
13410
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13414 %{
13415 effect(DEF dst, USE src1, USE cr);
13416 ins_cost(INSN_COST * 2);
13417 format %{ "csinvw $dst, $src1, zr lt\t" %}
13418
13419 ins_encode %{
13420 __ csinvw($dst$$Register,
13421 $src1$$Register,
13422 zr,
13423 Assembler::LT);
13424 %}
13425 ins_pipe(icond_reg);
13426 %}
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13431 %{
13432 effect(DEF dst, USE src1, USE cr);
13433 ins_cost(INSN_COST * 2);
13434 format %{ "csinvw $dst, $src1, zr ge\t" %}
13435
13436 ins_encode %{
13437 __ csinvw($dst$$Register,
13438 $src1$$Register,
13439 zr,
13440 Assembler::GE);
13441 %}
13442 ins_pipe(icond_reg);
13443 %}
13444
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13448 %{
13449 match(Set dst (MinI src imm));
13450 ins_cost(INSN_COST * 3);
13451 expand %{
13452 rFlagsReg cr;
13453 compI_reg_imm0(cr, src);
13454 cmovI_reg_imm0_lt(dst, src, cr);
13455 %}
13456 %}
13457
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13461 %{
13462 match(Set dst (MinI imm src));
13463 ins_cost(INSN_COST * 3);
13464 expand %{
13465 rFlagsReg cr;
13466 compI_reg_imm0(cr, src);
13467 cmovI_reg_imm0_lt(dst, src, cr);
13468 %}
13469 %}
13470
13471 // This pattern is automatically generated from aarch64_ad.m4
13472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13473 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13474 %{
13475 match(Set dst (MinI src imm));
13476 ins_cost(INSN_COST * 3);
13477 expand %{
13478 rFlagsReg cr;
13479 compI_reg_imm0(cr, src);
13480 cmovI_reg_imm1_le(dst, src, cr);
13481 %}
13482 %}
13483
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13487 %{
13488 match(Set dst (MinI imm src));
13489 ins_cost(INSN_COST * 3);
13490 expand %{
13491 rFlagsReg cr;
13492 compI_reg_imm0(cr, src);
13493 cmovI_reg_imm1_le(dst, src, cr);
13494 %}
13495 %}
13496
13497 // This pattern is automatically generated from aarch64_ad.m4
13498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13499 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13500 %{
13501 match(Set dst (MinI src imm));
13502 ins_cost(INSN_COST * 3);
13503 expand %{
13504 rFlagsReg cr;
13505 compI_reg_imm0(cr, src);
13506 cmovI_reg_immM1_lt(dst, src, cr);
13507 %}
13508 %}
13509
13510 // This pattern is automatically generated from aarch64_ad.m4
13511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13512 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13513 %{
13514 match(Set dst (MinI imm src));
13515 ins_cost(INSN_COST * 3);
13516 expand %{
13517 rFlagsReg cr;
13518 compI_reg_imm0(cr, src);
13519 cmovI_reg_immM1_lt(dst, src, cr);
13520 %}
13521 %}
13522
13523 // This pattern is automatically generated from aarch64_ad.m4
13524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13525 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13526 %{
13527 match(Set dst (MaxI src imm));
13528 ins_cost(INSN_COST * 3);
13529 expand %{
13530 rFlagsReg cr;
13531 compI_reg_imm0(cr, src);
13532 cmovI_reg_imm0_gt(dst, src, cr);
13533 %}
13534 %}
13535
13536 // This pattern is automatically generated from aarch64_ad.m4
13537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13538 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13539 %{
13540 match(Set dst (MaxI imm src));
13541 ins_cost(INSN_COST * 3);
13542 expand %{
13543 rFlagsReg cr;
13544 compI_reg_imm0(cr, src);
13545 cmovI_reg_imm0_gt(dst, src, cr);
13546 %}
13547 %}
13548
13549 // This pattern is automatically generated from aarch64_ad.m4
13550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13551 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13552 %{
13553 match(Set dst (MaxI src imm));
13554 ins_cost(INSN_COST * 3);
13555 expand %{
13556 rFlagsReg cr;
13557 compI_reg_imm0(cr, src);
13558 cmovI_reg_imm1_gt(dst, src, cr);
13559 %}
13560 %}
13561
13562 // This pattern is automatically generated from aarch64_ad.m4
13563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13564 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13565 %{
13566 match(Set dst (MaxI imm src));
13567 ins_cost(INSN_COST * 3);
13568 expand %{
13569 rFlagsReg cr;
13570 compI_reg_imm0(cr, src);
13571 cmovI_reg_imm1_gt(dst, src, cr);
13572 %}
13573 %}
13574
13575 // This pattern is automatically generated from aarch64_ad.m4
13576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13577 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13578 %{
13579 match(Set dst (MaxI src imm));
13580 ins_cost(INSN_COST * 3);
13581 expand %{
13582 rFlagsReg cr;
13583 compI_reg_imm0(cr, src);
13584 cmovI_reg_immM1_ge(dst, src, cr);
13585 %}
13586 %}
13587
13588 // This pattern is automatically generated from aarch64_ad.m4
13589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13590 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13591 %{
13592 match(Set dst (MaxI imm src));
13593 ins_cost(INSN_COST * 3);
13594 expand %{
13595 rFlagsReg cr;
13596 compI_reg_imm0(cr, src);
13597 cmovI_reg_immM1_ge(dst, src, cr);
13598 %}
13599 %}
13600
13601 // This pattern is automatically generated from aarch64_ad.m4
13602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13603 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13604 %{
13605 match(Set dst (ReverseI src));
13606 ins_cost(INSN_COST);
13607 format %{ "rbitw $dst, $src" %}
13608 ins_encode %{
13609 __ rbitw($dst$$Register, $src$$Register);
13610 %}
13611 ins_pipe(ialu_reg);
13612 %}
13613
13614 // This pattern is automatically generated from aarch64_ad.m4
13615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13616 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13617 %{
13618 match(Set dst (ReverseL src));
13619 ins_cost(INSN_COST);
13620 format %{ "rbit $dst, $src" %}
13621 ins_encode %{
13622 __ rbit($dst$$Register, $src$$Register);
13623 %}
13624 ins_pipe(ialu_reg);
13625 %}
13626
13627
13628 // END This section of the file is automatically generated. Do not edit --------------
13629
13630
13631 // ============================================================================
13632 // Floating Point Arithmetic Instructions
13633
13634 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13635 match(Set dst (AddF src1 src2));
13636
13637 ins_cost(INSN_COST * 5);
13638 format %{ "fadds $dst, $src1, $src2" %}
13639
13640 ins_encode %{
13641 __ fadds(as_FloatRegister($dst$$reg),
13642 as_FloatRegister($src1$$reg),
13643 as_FloatRegister($src2$$reg));
13644 %}
13645
13646 ins_pipe(fp_dop_reg_reg_s);
13647 %}
13648
13649 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13650 match(Set dst (AddD src1 src2));
13651
13652 ins_cost(INSN_COST * 5);
13653 format %{ "faddd $dst, $src1, $src2" %}
13654
13655 ins_encode %{
13656 __ faddd(as_FloatRegister($dst$$reg),
13657 as_FloatRegister($src1$$reg),
13658 as_FloatRegister($src2$$reg));
13659 %}
13660
13661 ins_pipe(fp_dop_reg_reg_d);
13662 %}
13663
13664 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13665 match(Set dst (SubF src1 src2));
13666
13667 ins_cost(INSN_COST * 5);
13668 format %{ "fsubs $dst, $src1, $src2" %}
13669
13670 ins_encode %{
13671 __ fsubs(as_FloatRegister($dst$$reg),
13672 as_FloatRegister($src1$$reg),
13673 as_FloatRegister($src2$$reg));
13674 %}
13675
13676 ins_pipe(fp_dop_reg_reg_s);
13677 %}
13678
13679 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13680 match(Set dst (SubD src1 src2));
13681
13682 ins_cost(INSN_COST * 5);
13683 format %{ "fsubd $dst, $src1, $src2" %}
13684
13685 ins_encode %{
13686 __ fsubd(as_FloatRegister($dst$$reg),
13687 as_FloatRegister($src1$$reg),
13688 as_FloatRegister($src2$$reg));
13689 %}
13690
13691 ins_pipe(fp_dop_reg_reg_d);
13692 %}
13693
13694 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13695 match(Set dst (MulF src1 src2));
13696
13697 ins_cost(INSN_COST * 6);
13698 format %{ "fmuls $dst, $src1, $src2" %}
13699
13700 ins_encode %{
13701 __ fmuls(as_FloatRegister($dst$$reg),
13702 as_FloatRegister($src1$$reg),
13703 as_FloatRegister($src2$$reg));
13704 %}
13705
13706 ins_pipe(fp_dop_reg_reg_s);
13707 %}
13708
13709 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13710 match(Set dst (MulD src1 src2));
13711
13712 ins_cost(INSN_COST * 6);
13713 format %{ "fmuld $dst, $src1, $src2" %}
13714
13715 ins_encode %{
13716 __ fmuld(as_FloatRegister($dst$$reg),
13717 as_FloatRegister($src1$$reg),
13718 as_FloatRegister($src2$$reg));
13719 %}
13720
13721 ins_pipe(fp_dop_reg_reg_d);
13722 %}
13723
13724 // src1 * src2 + src3
13725 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13726 match(Set dst (FmaF src3 (Binary src1 src2)));
13727
13728 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13729
13730 ins_encode %{
13731 assert(UseFMA, "Needs FMA instructions support.");
13732 __ fmadds(as_FloatRegister($dst$$reg),
13733 as_FloatRegister($src1$$reg),
13734 as_FloatRegister($src2$$reg),
13735 as_FloatRegister($src3$$reg));
13736 %}
13737
13738 ins_pipe(pipe_class_default);
13739 %}
13740
13741 // src1 * src2 + src3
13742 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13743 match(Set dst (FmaD src3 (Binary src1 src2)));
13744
13745 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13746
13747 ins_encode %{
13748 assert(UseFMA, "Needs FMA instructions support.");
13749 __ fmaddd(as_FloatRegister($dst$$reg),
13750 as_FloatRegister($src1$$reg),
13751 as_FloatRegister($src2$$reg),
13752 as_FloatRegister($src3$$reg));
13753 %}
13754
13755 ins_pipe(pipe_class_default);
13756 %}
13757
13758 // src1 * (-src2) + src3
13759 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13760 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13761 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13762
13763 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13764
13765 ins_encode %{
13766 assert(UseFMA, "Needs FMA instructions support.");
13767 __ fmsubs(as_FloatRegister($dst$$reg),
13768 as_FloatRegister($src1$$reg),
13769 as_FloatRegister($src2$$reg),
13770 as_FloatRegister($src3$$reg));
13771 %}
13772
13773 ins_pipe(pipe_class_default);
13774 %}
13775
13776 // src1 * (-src2) + src3
13777 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13778 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13779 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13780
13781 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13782
13783 ins_encode %{
13784 assert(UseFMA, "Needs FMA instructions support.");
13785 __ fmsubd(as_FloatRegister($dst$$reg),
13786 as_FloatRegister($src1$$reg),
13787 as_FloatRegister($src2$$reg),
13788 as_FloatRegister($src3$$reg));
13789 %}
13790
13791 ins_pipe(pipe_class_default);
13792 %}
13793
13794 // src1 * (-src2) - src3
13795 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13796 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13797 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13798
13799 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13800
13801 ins_encode %{
13802 assert(UseFMA, "Needs FMA instructions support.");
13803 __ fnmadds(as_FloatRegister($dst$$reg),
13804 as_FloatRegister($src1$$reg),
13805 as_FloatRegister($src2$$reg),
13806 as_FloatRegister($src3$$reg));
13807 %}
13808
13809 ins_pipe(pipe_class_default);
13810 %}
13811
13812 // src1 * (-src2) - src3
13813 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13814 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13815 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13816
13817 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13818
13819 ins_encode %{
13820 assert(UseFMA, "Needs FMA instructions support.");
13821 __ fnmaddd(as_FloatRegister($dst$$reg),
13822 as_FloatRegister($src1$$reg),
13823 as_FloatRegister($src2$$reg),
13824 as_FloatRegister($src3$$reg));
13825 %}
13826
13827 ins_pipe(pipe_class_default);
13828 %}
13829
13830 // src1 * src2 - src3
13831 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13832 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13833
13834 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13835
13836 ins_encode %{
13837 assert(UseFMA, "Needs FMA instructions support.");
13838 __ fnmsubs(as_FloatRegister($dst$$reg),
13839 as_FloatRegister($src1$$reg),
13840 as_FloatRegister($src2$$reg),
13841 as_FloatRegister($src3$$reg));
13842 %}
13843
13844 ins_pipe(pipe_class_default);
13845 %}
13846
13847 // src1 * src2 - src3
13848 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13849 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13850
13851 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13852
13853 ins_encode %{
13854 assert(UseFMA, "Needs FMA instructions support.");
13855 // n.b. insn name should be fnmsubd
13856 __ fnmsub(as_FloatRegister($dst$$reg),
13857 as_FloatRegister($src1$$reg),
13858 as_FloatRegister($src2$$reg),
13859 as_FloatRegister($src3$$reg));
13860 %}
13861
13862 ins_pipe(pipe_class_default);
13863 %}
13864
13865
13866 // Math.max(FF)F
13867 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13868 match(Set dst (MaxF src1 src2));
13869
13870 format %{ "fmaxs $dst, $src1, $src2" %}
13871 ins_encode %{
13872 __ fmaxs(as_FloatRegister($dst$$reg),
13873 as_FloatRegister($src1$$reg),
13874 as_FloatRegister($src2$$reg));
13875 %}
13876
13877 ins_pipe(fp_dop_reg_reg_s);
13878 %}
13879
13880 // Math.min(FF)F
13881 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13882 match(Set dst (MinF src1 src2));
13883
13884 format %{ "fmins $dst, $src1, $src2" %}
13885 ins_encode %{
13886 __ fmins(as_FloatRegister($dst$$reg),
13887 as_FloatRegister($src1$$reg),
13888 as_FloatRegister($src2$$reg));
13889 %}
13890
13891 ins_pipe(fp_dop_reg_reg_s);
13892 %}
13893
13894 // Math.max(DD)D
13895 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13896 match(Set dst (MaxD src1 src2));
13897
13898 format %{ "fmaxd $dst, $src1, $src2" %}
13899 ins_encode %{
13900 __ fmaxd(as_FloatRegister($dst$$reg),
13901 as_FloatRegister($src1$$reg),
13902 as_FloatRegister($src2$$reg));
13903 %}
13904
13905 ins_pipe(fp_dop_reg_reg_d);
13906 %}
13907
13908 // Math.min(DD)D
13909 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13910 match(Set dst (MinD src1 src2));
13911
13912 format %{ "fmind $dst, $src1, $src2" %}
13913 ins_encode %{
13914 __ fmind(as_FloatRegister($dst$$reg),
13915 as_FloatRegister($src1$$reg),
13916 as_FloatRegister($src2$$reg));
13917 %}
13918
13919 ins_pipe(fp_dop_reg_reg_d);
13920 %}
13921
13922
13923 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13924 match(Set dst (DivF src1 src2));
13925
13926 ins_cost(INSN_COST * 18);
13927 format %{ "fdivs $dst, $src1, $src2" %}
13928
13929 ins_encode %{
13930 __ fdivs(as_FloatRegister($dst$$reg),
13931 as_FloatRegister($src1$$reg),
13932 as_FloatRegister($src2$$reg));
13933 %}
13934
13935 ins_pipe(fp_div_s);
13936 %}
13937
13938 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13939 match(Set dst (DivD src1 src2));
13940
13941 ins_cost(INSN_COST * 32);
13942 format %{ "fdivd $dst, $src1, $src2" %}
13943
13944 ins_encode %{
13945 __ fdivd(as_FloatRegister($dst$$reg),
13946 as_FloatRegister($src1$$reg),
13947 as_FloatRegister($src2$$reg));
13948 %}
13949
13950 ins_pipe(fp_div_d);
13951 %}
13952
13953 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13954 match(Set dst (NegF src));
13955
13956 ins_cost(INSN_COST * 3);
13957 format %{ "fneg $dst, $src" %}
13958
13959 ins_encode %{
13960 __ fnegs(as_FloatRegister($dst$$reg),
13961 as_FloatRegister($src$$reg));
13962 %}
13963
13964 ins_pipe(fp_uop_s);
13965 %}
13966
13967 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13968 match(Set dst (NegD src));
13969
13970 ins_cost(INSN_COST * 3);
13971 format %{ "fnegd $dst, $src" %}
13972
13973 ins_encode %{
13974 __ fnegd(as_FloatRegister($dst$$reg),
13975 as_FloatRegister($src$$reg));
13976 %}
13977
13978 ins_pipe(fp_uop_d);
13979 %}
13980
13981 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13982 %{
13983 match(Set dst (AbsI src));
13984
13985 effect(KILL cr);
13986 ins_cost(INSN_COST * 2);
13987 format %{ "cmpw $src, zr\n\t"
13988 "cnegw $dst, $src, Assembler::LT\t# int abs"
13989 %}
13990
13991 ins_encode %{
13992 __ cmpw(as_Register($src$$reg), zr);
13993 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13994 %}
13995 ins_pipe(pipe_class_default);
13996 %}
13997
13998 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13999 %{
14000 match(Set dst (AbsL src));
14001
14002 effect(KILL cr);
14003 ins_cost(INSN_COST * 2);
14004 format %{ "cmp $src, zr\n\t"
14005 "cneg $dst, $src, Assembler::LT\t# long abs"
14006 %}
14007
14008 ins_encode %{
14009 __ cmp(as_Register($src$$reg), zr);
14010 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14011 %}
14012 ins_pipe(pipe_class_default);
14013 %}
14014
14015 instruct absF_reg(vRegF dst, vRegF src) %{
14016 match(Set dst (AbsF src));
14017
14018 ins_cost(INSN_COST * 3);
14019 format %{ "fabss $dst, $src" %}
14020 ins_encode %{
14021 __ fabss(as_FloatRegister($dst$$reg),
14022 as_FloatRegister($src$$reg));
14023 %}
14024
14025 ins_pipe(fp_uop_s);
14026 %}
14027
14028 instruct absD_reg(vRegD dst, vRegD src) %{
14029 match(Set dst (AbsD src));
14030
14031 ins_cost(INSN_COST * 3);
14032 format %{ "fabsd $dst, $src" %}
14033 ins_encode %{
14034 __ fabsd(as_FloatRegister($dst$$reg),
14035 as_FloatRegister($src$$reg));
14036 %}
14037
14038 ins_pipe(fp_uop_d);
14039 %}
14040
14041 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14042 match(Set dst (AbsF (SubF src1 src2)));
14043
14044 ins_cost(INSN_COST * 3);
14045 format %{ "fabds $dst, $src1, $src2" %}
14046 ins_encode %{
14047 __ fabds(as_FloatRegister($dst$$reg),
14048 as_FloatRegister($src1$$reg),
14049 as_FloatRegister($src2$$reg));
14050 %}
14051
14052 ins_pipe(fp_uop_s);
14053 %}
14054
14055 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14056 match(Set dst (AbsD (SubD src1 src2)));
14057
14058 ins_cost(INSN_COST * 3);
14059 format %{ "fabdd $dst, $src1, $src2" %}
14060 ins_encode %{
14061 __ fabdd(as_FloatRegister($dst$$reg),
14062 as_FloatRegister($src1$$reg),
14063 as_FloatRegister($src2$$reg));
14064 %}
14065
14066 ins_pipe(fp_uop_d);
14067 %}
14068
14069 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14070 match(Set dst (SqrtD src));
14071
14072 ins_cost(INSN_COST * 50);
14073 format %{ "fsqrtd $dst, $src" %}
14074 ins_encode %{
14075 __ fsqrtd(as_FloatRegister($dst$$reg),
14076 as_FloatRegister($src$$reg));
14077 %}
14078
14079 ins_pipe(fp_div_s);
14080 %}
14081
14082 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14083 match(Set dst (SqrtF src));
14084
14085 ins_cost(INSN_COST * 50);
14086 format %{ "fsqrts $dst, $src" %}
14087 ins_encode %{
14088 __ fsqrts(as_FloatRegister($dst$$reg),
14089 as_FloatRegister($src$$reg));
14090 %}
14091
14092 ins_pipe(fp_div_d);
14093 %}
14094
14095 // Math.rint, floor, ceil
14096 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14097 match(Set dst (RoundDoubleMode src rmode));
14098 format %{ "frint $dst, $src, $rmode" %}
14099 ins_encode %{
14100 switch ($rmode$$constant) {
14101 case RoundDoubleModeNode::rmode_rint:
14102 __ frintnd(as_FloatRegister($dst$$reg),
14103 as_FloatRegister($src$$reg));
14104 break;
14105 case RoundDoubleModeNode::rmode_floor:
14106 __ frintmd(as_FloatRegister($dst$$reg),
14107 as_FloatRegister($src$$reg));
14108 break;
14109 case RoundDoubleModeNode::rmode_ceil:
14110 __ frintpd(as_FloatRegister($dst$$reg),
14111 as_FloatRegister($src$$reg));
14112 break;
14113 }
14114 %}
14115 ins_pipe(fp_uop_d);
14116 %}
14117
14118 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14119 match(Set dst (CopySignD src1 (Binary src2 zero)));
14120 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14121 format %{ "CopySignD $dst $src1 $src2" %}
14122 ins_encode %{
14123 FloatRegister dst = as_FloatRegister($dst$$reg),
14124 src1 = as_FloatRegister($src1$$reg),
14125 src2 = as_FloatRegister($src2$$reg),
14126 zero = as_FloatRegister($zero$$reg);
14127 __ fnegd(dst, zero);
14128 __ bsl(dst, __ T8B, src2, src1);
14129 %}
14130 ins_pipe(fp_uop_d);
14131 %}
14132
14133 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14134 match(Set dst (CopySignF src1 src2));
14135 effect(TEMP_DEF dst, USE src1, USE src2);
14136 format %{ "CopySignF $dst $src1 $src2" %}
14137 ins_encode %{
14138 FloatRegister dst = as_FloatRegister($dst$$reg),
14139 src1 = as_FloatRegister($src1$$reg),
14140 src2 = as_FloatRegister($src2$$reg);
14141 __ movi(dst, __ T2S, 0x80, 24);
14142 __ bsl(dst, __ T8B, src2, src1);
14143 %}
14144 ins_pipe(fp_uop_d);
14145 %}
14146
14147 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14148 match(Set dst (SignumD src (Binary zero one)));
14149 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14150 format %{ "signumD $dst, $src" %}
14151 ins_encode %{
14152 FloatRegister src = as_FloatRegister($src$$reg),
14153 dst = as_FloatRegister($dst$$reg),
14154 zero = as_FloatRegister($zero$$reg),
14155 one = as_FloatRegister($one$$reg);
14156 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14157 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14158 // Bit selection instruction gets bit from "one" for each enabled bit in
14159 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14160 // NaN the whole "src" will be copied because "dst" is zero. For all other
14161 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14162 // from "src", and all other bits are copied from 1.0.
14163 __ bsl(dst, __ T8B, one, src);
14164 %}
14165 ins_pipe(fp_uop_d);
14166 %}
14167
14168 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14169 match(Set dst (SignumF src (Binary zero one)));
14170 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14171 format %{ "signumF $dst, $src" %}
14172 ins_encode %{
14173 FloatRegister src = as_FloatRegister($src$$reg),
14174 dst = as_FloatRegister($dst$$reg),
14175 zero = as_FloatRegister($zero$$reg),
14176 one = as_FloatRegister($one$$reg);
14177 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14178 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14179 // Bit selection instruction gets bit from "one" for each enabled bit in
14180 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14181 // NaN the whole "src" will be copied because "dst" is zero. For all other
14182 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14183 // from "src", and all other bits are copied from 1.0.
14184 __ bsl(dst, __ T8B, one, src);
14185 %}
14186 ins_pipe(fp_uop_d);
14187 %}
14188
14189 instruct onspinwait() %{
14190 match(OnSpinWait);
14191 ins_cost(INSN_COST);
14192
14193 format %{ "onspinwait" %}
14194
14195 ins_encode %{
14196 __ spin_wait();
14197 %}
14198 ins_pipe(pipe_class_empty);
14199 %}
14200
14201 // ============================================================================
14202 // Logical Instructions
14203
14204 // Integer Logical Instructions
14205
14206 // And Instructions
14207
14208
14209 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14210 match(Set dst (AndI src1 src2));
14211
14212 format %{ "andw $dst, $src1, $src2\t# int" %}
14213
14214 ins_cost(INSN_COST);
14215 ins_encode %{
14216 __ andw(as_Register($dst$$reg),
14217 as_Register($src1$$reg),
14218 as_Register($src2$$reg));
14219 %}
14220
14221 ins_pipe(ialu_reg_reg);
14222 %}
14223
14224 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14225 match(Set dst (AndI src1 src2));
14226
14227 format %{ "andsw $dst, $src1, $src2\t# int" %}
14228
14229 ins_cost(INSN_COST);
14230 ins_encode %{
14231 __ andw(as_Register($dst$$reg),
14232 as_Register($src1$$reg),
14233 (uint64_t)($src2$$constant));
14234 %}
14235
14236 ins_pipe(ialu_reg_imm);
14237 %}
14238
14239 // Or Instructions
14240
14241 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14242 match(Set dst (OrI src1 src2));
14243
14244 format %{ "orrw $dst, $src1, $src2\t# int" %}
14245
14246 ins_cost(INSN_COST);
14247 ins_encode %{
14248 __ orrw(as_Register($dst$$reg),
14249 as_Register($src1$$reg),
14250 as_Register($src2$$reg));
14251 %}
14252
14253 ins_pipe(ialu_reg_reg);
14254 %}
14255
14256 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14257 match(Set dst (OrI src1 src2));
14258
14259 format %{ "orrw $dst, $src1, $src2\t# int" %}
14260
14261 ins_cost(INSN_COST);
14262 ins_encode %{
14263 __ orrw(as_Register($dst$$reg),
14264 as_Register($src1$$reg),
14265 (uint64_t)($src2$$constant));
14266 %}
14267
14268 ins_pipe(ialu_reg_imm);
14269 %}
14270
14271 // Xor Instructions
14272
14273 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14274 match(Set dst (XorI src1 src2));
14275
14276 format %{ "eorw $dst, $src1, $src2\t# int" %}
14277
14278 ins_cost(INSN_COST);
14279 ins_encode %{
14280 __ eorw(as_Register($dst$$reg),
14281 as_Register($src1$$reg),
14282 as_Register($src2$$reg));
14283 %}
14284
14285 ins_pipe(ialu_reg_reg);
14286 %}
14287
14288 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14289 match(Set dst (XorI src1 src2));
14290
14291 format %{ "eorw $dst, $src1, $src2\t# int" %}
14292
14293 ins_cost(INSN_COST);
14294 ins_encode %{
14295 __ eorw(as_Register($dst$$reg),
14296 as_Register($src1$$reg),
14297 (uint64_t)($src2$$constant));
14298 %}
14299
14300 ins_pipe(ialu_reg_imm);
14301 %}
14302
14303 // Long Logical Instructions
14304 // TODO
14305
14306 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14307 match(Set dst (AndL src1 src2));
14308
14309 format %{ "and $dst, $src1, $src2\t# int" %}
14310
14311 ins_cost(INSN_COST);
14312 ins_encode %{
14313 __ andr(as_Register($dst$$reg),
14314 as_Register($src1$$reg),
14315 as_Register($src2$$reg));
14316 %}
14317
14318 ins_pipe(ialu_reg_reg);
14319 %}
14320
14321 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14322 match(Set dst (AndL src1 src2));
14323
14324 format %{ "and $dst, $src1, $src2\t# int" %}
14325
14326 ins_cost(INSN_COST);
14327 ins_encode %{
14328 __ andr(as_Register($dst$$reg),
14329 as_Register($src1$$reg),
14330 (uint64_t)($src2$$constant));
14331 %}
14332
14333 ins_pipe(ialu_reg_imm);
14334 %}
14335
14336 // Or Instructions
14337
14338 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14339 match(Set dst (OrL src1 src2));
14340
14341 format %{ "orr $dst, $src1, $src2\t# int" %}
14342
14343 ins_cost(INSN_COST);
14344 ins_encode %{
14345 __ orr(as_Register($dst$$reg),
14346 as_Register($src1$$reg),
14347 as_Register($src2$$reg));
14348 %}
14349
14350 ins_pipe(ialu_reg_reg);
14351 %}
14352
14353 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14354 match(Set dst (OrL src1 src2));
14355
14356 format %{ "orr $dst, $src1, $src2\t# int" %}
14357
14358 ins_cost(INSN_COST);
14359 ins_encode %{
14360 __ orr(as_Register($dst$$reg),
14361 as_Register($src1$$reg),
14362 (uint64_t)($src2$$constant));
14363 %}
14364
14365 ins_pipe(ialu_reg_imm);
14366 %}
14367
14368 // Xor Instructions
14369
14370 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14371 match(Set dst (XorL src1 src2));
14372
14373 format %{ "eor $dst, $src1, $src2\t# int" %}
14374
14375 ins_cost(INSN_COST);
14376 ins_encode %{
14377 __ eor(as_Register($dst$$reg),
14378 as_Register($src1$$reg),
14379 as_Register($src2$$reg));
14380 %}
14381
14382 ins_pipe(ialu_reg_reg);
14383 %}
14384
14385 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14386 match(Set dst (XorL src1 src2));
14387
14388 ins_cost(INSN_COST);
14389 format %{ "eor $dst, $src1, $src2\t# int" %}
14390
14391 ins_encode %{
14392 __ eor(as_Register($dst$$reg),
14393 as_Register($src1$$reg),
14394 (uint64_t)($src2$$constant));
14395 %}
14396
14397 ins_pipe(ialu_reg_imm);
14398 %}
14399
14400 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14401 %{
14402 match(Set dst (ConvI2L src));
14403
14404 ins_cost(INSN_COST);
14405 format %{ "sxtw $dst, $src\t# i2l" %}
14406 ins_encode %{
14407 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14408 %}
14409 ins_pipe(ialu_reg_shift);
14410 %}
14411
14412 // this pattern occurs in bigmath arithmetic
14413 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14414 %{
14415 match(Set dst (AndL (ConvI2L src) mask));
14416
14417 ins_cost(INSN_COST);
14418 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14419 ins_encode %{
14420 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14421 %}
14422
14423 ins_pipe(ialu_reg_shift);
14424 %}
14425
14426 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14427 match(Set dst (ConvL2I src));
14428
14429 ins_cost(INSN_COST);
14430 format %{ "movw $dst, $src \t// l2i" %}
14431
14432 ins_encode %{
14433 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14434 %}
14435
14436 ins_pipe(ialu_reg);
14437 %}
14438
14439 instruct convD2F_reg(vRegF dst, vRegD src) %{
14440 match(Set dst (ConvD2F src));
14441
14442 ins_cost(INSN_COST * 5);
14443 format %{ "fcvtd $dst, $src \t// d2f" %}
14444
14445 ins_encode %{
14446 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14447 %}
14448
14449 ins_pipe(fp_d2f);
14450 %}
14451
14452 instruct convF2D_reg(vRegD dst, vRegF src) %{
14453 match(Set dst (ConvF2D src));
14454
14455 ins_cost(INSN_COST * 5);
14456 format %{ "fcvts $dst, $src \t// f2d" %}
14457
14458 ins_encode %{
14459 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14460 %}
14461
14462 ins_pipe(fp_f2d);
14463 %}
14464
14465 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14466 match(Set dst (ConvF2I src));
14467
14468 ins_cost(INSN_COST * 5);
14469 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14470
14471 ins_encode %{
14472 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14473 %}
14474
14475 ins_pipe(fp_f2i);
14476 %}
14477
14478 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14479 match(Set dst (ConvF2L src));
14480
14481 ins_cost(INSN_COST * 5);
14482 format %{ "fcvtzs $dst, $src \t// f2l" %}
14483
14484 ins_encode %{
14485 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14486 %}
14487
14488 ins_pipe(fp_f2l);
14489 %}
14490
14491 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14492 match(Set dst (ConvF2HF src));
14493 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14494 "smov $dst, $tmp\t# move result from $tmp to $dst"
14495 %}
14496 effect(TEMP tmp);
14497 ins_encode %{
14498 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14499 %}
14500 ins_pipe(pipe_slow);
14501 %}
14502
14503 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14504 match(Set dst (ConvHF2F src));
14505 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14506 "fcvt $dst, $tmp\t# convert half to single precision"
14507 %}
14508 effect(TEMP tmp);
14509 ins_encode %{
14510 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14511 %}
14512 ins_pipe(pipe_slow);
14513 %}
14514
14515 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14516 match(Set dst (ConvI2F src));
14517
14518 ins_cost(INSN_COST * 5);
14519 format %{ "scvtfws $dst, $src \t// i2f" %}
14520
14521 ins_encode %{
14522 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14523 %}
14524
14525 ins_pipe(fp_i2f);
14526 %}
14527
14528 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14529 match(Set dst (ConvL2F src));
14530
14531 ins_cost(INSN_COST * 5);
14532 format %{ "scvtfs $dst, $src \t// l2f" %}
14533
14534 ins_encode %{
14535 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14536 %}
14537
14538 ins_pipe(fp_l2f);
14539 %}
14540
14541 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14542 match(Set dst (ConvD2I src));
14543
14544 ins_cost(INSN_COST * 5);
14545 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14546
14547 ins_encode %{
14548 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14549 %}
14550
14551 ins_pipe(fp_d2i);
14552 %}
14553
14554 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14555 match(Set dst (ConvD2L src));
14556
14557 ins_cost(INSN_COST * 5);
14558 format %{ "fcvtzd $dst, $src \t// d2l" %}
14559
14560 ins_encode %{
14561 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14562 %}
14563
14564 ins_pipe(fp_d2l);
14565 %}
14566
14567 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14568 match(Set dst (ConvI2D src));
14569
14570 ins_cost(INSN_COST * 5);
14571 format %{ "scvtfwd $dst, $src \t// i2d" %}
14572
14573 ins_encode %{
14574 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14575 %}
14576
14577 ins_pipe(fp_i2d);
14578 %}
14579
14580 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14581 match(Set dst (ConvL2D src));
14582
14583 ins_cost(INSN_COST * 5);
14584 format %{ "scvtfd $dst, $src \t// l2d" %}
14585
14586 ins_encode %{
14587 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14588 %}
14589
14590 ins_pipe(fp_l2d);
14591 %}
14592
14593 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14594 %{
14595 match(Set dst (RoundD src));
14596 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14597 format %{ "java_round_double $dst,$src"%}
14598 ins_encode %{
14599 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14600 as_FloatRegister($ftmp$$reg));
14601 %}
14602 ins_pipe(pipe_slow);
14603 %}
14604
14605 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14606 %{
14607 match(Set dst (RoundF src));
14608 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14609 format %{ "java_round_float $dst,$src"%}
14610 ins_encode %{
14611 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14612 as_FloatRegister($ftmp$$reg));
14613 %}
14614 ins_pipe(pipe_slow);
14615 %}
14616
14617 // stack <-> reg and reg <-> reg shuffles with no conversion
14618
14619 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14620
14621 match(Set dst (MoveF2I src));
14622
14623 effect(DEF dst, USE src);
14624
14625 ins_cost(4 * INSN_COST);
14626
14627 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14628
14629 ins_encode %{
14630 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14631 %}
14632
14633 ins_pipe(iload_reg_reg);
14634
14635 %}
14636
14637 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14638
14639 match(Set dst (MoveI2F src));
14640
14641 effect(DEF dst, USE src);
14642
14643 ins_cost(4 * INSN_COST);
14644
14645 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14646
14647 ins_encode %{
14648 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14649 %}
14650
14651 ins_pipe(pipe_class_memory);
14652
14653 %}
14654
14655 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14656
14657 match(Set dst (MoveD2L src));
14658
14659 effect(DEF dst, USE src);
14660
14661 ins_cost(4 * INSN_COST);
14662
14663 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14664
14665 ins_encode %{
14666 __ ldr($dst$$Register, Address(sp, $src$$disp));
14667 %}
14668
14669 ins_pipe(iload_reg_reg);
14670
14671 %}
14672
14673 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14674
14675 match(Set dst (MoveL2D src));
14676
14677 effect(DEF dst, USE src);
14678
14679 ins_cost(4 * INSN_COST);
14680
14681 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14682
14683 ins_encode %{
14684 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14685 %}
14686
14687 ins_pipe(pipe_class_memory);
14688
14689 %}
14690
14691 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14692
14693 match(Set dst (MoveF2I src));
14694
14695 effect(DEF dst, USE src);
14696
14697 ins_cost(INSN_COST);
14698
14699 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14700
14701 ins_encode %{
14702 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14703 %}
14704
14705 ins_pipe(pipe_class_memory);
14706
14707 %}
14708
14709 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14710
14711 match(Set dst (MoveI2F src));
14712
14713 effect(DEF dst, USE src);
14714
14715 ins_cost(INSN_COST);
14716
14717 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14718
14719 ins_encode %{
14720 __ strw($src$$Register, Address(sp, $dst$$disp));
14721 %}
14722
14723 ins_pipe(istore_reg_reg);
14724
14725 %}
14726
14727 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14728
14729 match(Set dst (MoveD2L src));
14730
14731 effect(DEF dst, USE src);
14732
14733 ins_cost(INSN_COST);
14734
14735 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14736
14737 ins_encode %{
14738 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14739 %}
14740
14741 ins_pipe(pipe_class_memory);
14742
14743 %}
14744
14745 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14746
14747 match(Set dst (MoveL2D src));
14748
14749 effect(DEF dst, USE src);
14750
14751 ins_cost(INSN_COST);
14752
14753 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14754
14755 ins_encode %{
14756 __ str($src$$Register, Address(sp, $dst$$disp));
14757 %}
14758
14759 ins_pipe(istore_reg_reg);
14760
14761 %}
14762
14763 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14764
14765 match(Set dst (MoveF2I src));
14766
14767 effect(DEF dst, USE src);
14768
14769 ins_cost(INSN_COST);
14770
14771 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14772
14773 ins_encode %{
14774 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14775 %}
14776
14777 ins_pipe(fp_f2i);
14778
14779 %}
14780
14781 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14782
14783 match(Set dst (MoveI2F src));
14784
14785 effect(DEF dst, USE src);
14786
14787 ins_cost(INSN_COST);
14788
14789 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14790
14791 ins_encode %{
14792 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14793 %}
14794
14795 ins_pipe(fp_i2f);
14796
14797 %}
14798
14799 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14800
14801 match(Set dst (MoveD2L src));
14802
14803 effect(DEF dst, USE src);
14804
14805 ins_cost(INSN_COST);
14806
14807 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14808
14809 ins_encode %{
14810 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14811 %}
14812
14813 ins_pipe(fp_d2l);
14814
14815 %}
14816
14817 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14818
14819 match(Set dst (MoveL2D src));
14820
14821 effect(DEF dst, USE src);
14822
14823 ins_cost(INSN_COST);
14824
14825 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14826
14827 ins_encode %{
14828 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14829 %}
14830
14831 ins_pipe(fp_l2d);
14832
14833 %}
14834
14835 // ============================================================================
14836 // clearing of an array
14837
14838 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14839 %{
14840 match(Set dummy (ClearArray cnt base));
14841 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14842
14843 ins_cost(4 * INSN_COST);
14844 format %{ "ClearArray $cnt, $base" %}
14845
14846 ins_encode %{
14847 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14848 if (tpc == nullptr) {
14849 ciEnv::current()->record_failure("CodeCache is full");
14850 return;
14851 }
14852 %}
14853
14854 ins_pipe(pipe_class_memory);
14855 %}
14856
14857 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14858 %{
14859 predicate((uint64_t)n->in(2)->get_long()
14860 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14861 match(Set dummy (ClearArray cnt base));
14862 effect(TEMP temp, USE_KILL base, KILL cr);
14863
14864 ins_cost(4 * INSN_COST);
14865 format %{ "ClearArray $cnt, $base" %}
14866
14867 ins_encode %{
14868 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14869 if (tpc == nullptr) {
14870 ciEnv::current()->record_failure("CodeCache is full");
14871 return;
14872 }
14873 %}
14874
14875 ins_pipe(pipe_class_memory);
14876 %}
14877
14878 // ============================================================================
14879 // Overflow Math Instructions
14880
14881 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14882 %{
14883 match(Set cr (OverflowAddI op1 op2));
14884
14885 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14886 ins_cost(INSN_COST);
14887 ins_encode %{
14888 __ cmnw($op1$$Register, $op2$$Register);
14889 %}
14890
14891 ins_pipe(icmp_reg_reg);
14892 %}
14893
14894 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14895 %{
14896 match(Set cr (OverflowAddI op1 op2));
14897
14898 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14899 ins_cost(INSN_COST);
14900 ins_encode %{
14901 __ cmnw($op1$$Register, $op2$$constant);
14902 %}
14903
14904 ins_pipe(icmp_reg_imm);
14905 %}
14906
14907 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14908 %{
14909 match(Set cr (OverflowAddL op1 op2));
14910
14911 format %{ "cmn $op1, $op2\t# overflow check long" %}
14912 ins_cost(INSN_COST);
14913 ins_encode %{
14914 __ cmn($op1$$Register, $op2$$Register);
14915 %}
14916
14917 ins_pipe(icmp_reg_reg);
14918 %}
14919
14920 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14921 %{
14922 match(Set cr (OverflowAddL op1 op2));
14923
14924 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14925 ins_cost(INSN_COST);
14926 ins_encode %{
14927 __ adds(zr, $op1$$Register, $op2$$constant);
14928 %}
14929
14930 ins_pipe(icmp_reg_imm);
14931 %}
14932
14933 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14934 %{
14935 match(Set cr (OverflowSubI op1 op2));
14936
14937 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14938 ins_cost(INSN_COST);
14939 ins_encode %{
14940 __ cmpw($op1$$Register, $op2$$Register);
14941 %}
14942
14943 ins_pipe(icmp_reg_reg);
14944 %}
14945
14946 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14947 %{
14948 match(Set cr (OverflowSubI op1 op2));
14949
14950 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14951 ins_cost(INSN_COST);
14952 ins_encode %{
14953 __ cmpw($op1$$Register, $op2$$constant);
14954 %}
14955
14956 ins_pipe(icmp_reg_imm);
14957 %}
14958
14959 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14960 %{
14961 match(Set cr (OverflowSubL op1 op2));
14962
14963 format %{ "cmp $op1, $op2\t# overflow check long" %}
14964 ins_cost(INSN_COST);
14965 ins_encode %{
14966 __ cmp($op1$$Register, $op2$$Register);
14967 %}
14968
14969 ins_pipe(icmp_reg_reg);
14970 %}
14971
14972 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14973 %{
14974 match(Set cr (OverflowSubL op1 op2));
14975
14976 format %{ "cmp $op1, $op2\t# overflow check long" %}
14977 ins_cost(INSN_COST);
14978 ins_encode %{
14979 __ subs(zr, $op1$$Register, $op2$$constant);
14980 %}
14981
14982 ins_pipe(icmp_reg_imm);
14983 %}
14984
14985 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14986 %{
14987 match(Set cr (OverflowSubI zero op1));
14988
14989 format %{ "cmpw zr, $op1\t# overflow check int" %}
14990 ins_cost(INSN_COST);
14991 ins_encode %{
14992 __ cmpw(zr, $op1$$Register);
14993 %}
14994
14995 ins_pipe(icmp_reg_imm);
14996 %}
14997
14998 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14999 %{
15000 match(Set cr (OverflowSubL zero op1));
15001
15002 format %{ "cmp zr, $op1\t# overflow check long" %}
15003 ins_cost(INSN_COST);
15004 ins_encode %{
15005 __ cmp(zr, $op1$$Register);
15006 %}
15007
15008 ins_pipe(icmp_reg_imm);
15009 %}
15010
15011 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15012 %{
15013 match(Set cr (OverflowMulI op1 op2));
15014
15015 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15016 "cmp rscratch1, rscratch1, sxtw\n\t"
15017 "movw rscratch1, #0x80000000\n\t"
15018 "cselw rscratch1, rscratch1, zr, NE\n\t"
15019 "cmpw rscratch1, #1" %}
15020 ins_cost(5 * INSN_COST);
15021 ins_encode %{
15022 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15023 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15024 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15025 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15026 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15027 %}
15028
15029 ins_pipe(pipe_slow);
15030 %}
15031
15032 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15033 %{
15034 match(If cmp (OverflowMulI op1 op2));
15035 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15036 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15037 effect(USE labl, KILL cr);
15038
15039 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15040 "cmp rscratch1, rscratch1, sxtw\n\t"
15041 "b$cmp $labl" %}
15042 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15043 ins_encode %{
15044 Label* L = $labl$$label;
15045 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15046 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15047 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15048 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15049 %}
15050
15051 ins_pipe(pipe_serial);
15052 %}
15053
15054 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15055 %{
15056 match(Set cr (OverflowMulL op1 op2));
15057
15058 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15059 "smulh rscratch2, $op1, $op2\n\t"
15060 "cmp rscratch2, rscratch1, ASR #63\n\t"
15061 "movw rscratch1, #0x80000000\n\t"
15062 "cselw rscratch1, rscratch1, zr, NE\n\t"
15063 "cmpw rscratch1, #1" %}
15064 ins_cost(6 * INSN_COST);
15065 ins_encode %{
15066 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15067 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15068 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15069 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15070 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15071 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15072 %}
15073
15074 ins_pipe(pipe_slow);
15075 %}
15076
15077 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15078 %{
15079 match(If cmp (OverflowMulL op1 op2));
15080 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15081 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15082 effect(USE labl, KILL cr);
15083
15084 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15085 "smulh rscratch2, $op1, $op2\n\t"
15086 "cmp rscratch2, rscratch1, ASR #63\n\t"
15087 "b$cmp $labl" %}
15088 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15089 ins_encode %{
15090 Label* L = $labl$$label;
15091 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15092 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15093 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15094 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15095 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15096 %}
15097
15098 ins_pipe(pipe_serial);
15099 %}
15100
15101 // ============================================================================
15102 // Compare Instructions
15103
15104 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15105 %{
15106 match(Set cr (CmpI op1 op2));
15107
15108 effect(DEF cr, USE op1, USE op2);
15109
15110 ins_cost(INSN_COST);
15111 format %{ "cmpw $op1, $op2" %}
15112
15113 ins_encode(aarch64_enc_cmpw(op1, op2));
15114
15115 ins_pipe(icmp_reg_reg);
15116 %}
15117
15118 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15119 %{
15120 match(Set cr (CmpI op1 zero));
15121
15122 effect(DEF cr, USE op1);
15123
15124 ins_cost(INSN_COST);
15125 format %{ "cmpw $op1, 0" %}
15126
15127 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15128
15129 ins_pipe(icmp_reg_imm);
15130 %}
15131
15132 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15133 %{
15134 match(Set cr (CmpI op1 op2));
15135
15136 effect(DEF cr, USE op1);
15137
15138 ins_cost(INSN_COST);
15139 format %{ "cmpw $op1, $op2" %}
15140
15141 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15142
15143 ins_pipe(icmp_reg_imm);
15144 %}
15145
15146 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15147 %{
15148 match(Set cr (CmpI op1 op2));
15149
15150 effect(DEF cr, USE op1);
15151
15152 ins_cost(INSN_COST * 2);
15153 format %{ "cmpw $op1, $op2" %}
15154
15155 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15156
15157 ins_pipe(icmp_reg_imm);
15158 %}
15159
15160 // Unsigned compare Instructions; really, same as signed compare
15161 // except it should only be used to feed an If or a CMovI which takes a
15162 // cmpOpU.
15163
15164 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15165 %{
15166 match(Set cr (CmpU op1 op2));
15167
15168 effect(DEF cr, USE op1, USE op2);
15169
15170 ins_cost(INSN_COST);
15171 format %{ "cmpw $op1, $op2\t# unsigned" %}
15172
15173 ins_encode(aarch64_enc_cmpw(op1, op2));
15174
15175 ins_pipe(icmp_reg_reg);
15176 %}
15177
15178 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15179 %{
15180 match(Set cr (CmpU op1 zero));
15181
15182 effect(DEF cr, USE op1);
15183
15184 ins_cost(INSN_COST);
15185 format %{ "cmpw $op1, #0\t# unsigned" %}
15186
15187 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15188
15189 ins_pipe(icmp_reg_imm);
15190 %}
15191
15192 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15193 %{
15194 match(Set cr (CmpU op1 op2));
15195
15196 effect(DEF cr, USE op1);
15197
15198 ins_cost(INSN_COST);
15199 format %{ "cmpw $op1, $op2\t# unsigned" %}
15200
15201 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15202
15203 ins_pipe(icmp_reg_imm);
15204 %}
15205
15206 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15207 %{
15208 match(Set cr (CmpU op1 op2));
15209
15210 effect(DEF cr, USE op1);
15211
15212 ins_cost(INSN_COST * 2);
15213 format %{ "cmpw $op1, $op2\t# unsigned" %}
15214
15215 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15216
15217 ins_pipe(icmp_reg_imm);
15218 %}
15219
15220 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15221 %{
15222 match(Set cr (CmpL op1 op2));
15223
15224 effect(DEF cr, USE op1, USE op2);
15225
15226 ins_cost(INSN_COST);
15227 format %{ "cmp $op1, $op2" %}
15228
15229 ins_encode(aarch64_enc_cmp(op1, op2));
15230
15231 ins_pipe(icmp_reg_reg);
15232 %}
15233
15234 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15235 %{
15236 match(Set cr (CmpL op1 zero));
15237
15238 effect(DEF cr, USE op1);
15239
15240 ins_cost(INSN_COST);
15241 format %{ "tst $op1" %}
15242
15243 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15244
15245 ins_pipe(icmp_reg_imm);
15246 %}
15247
15248 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15249 %{
15250 match(Set cr (CmpL op1 op2));
15251
15252 effect(DEF cr, USE op1);
15253
15254 ins_cost(INSN_COST);
15255 format %{ "cmp $op1, $op2" %}
15256
15257 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15258
15259 ins_pipe(icmp_reg_imm);
15260 %}
15261
15262 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15263 %{
15264 match(Set cr (CmpL op1 op2));
15265
15266 effect(DEF cr, USE op1);
15267
15268 ins_cost(INSN_COST * 2);
15269 format %{ "cmp $op1, $op2" %}
15270
15271 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15272
15273 ins_pipe(icmp_reg_imm);
15274 %}
15275
15276 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15277 %{
15278 match(Set cr (CmpUL op1 op2));
15279
15280 effect(DEF cr, USE op1, USE op2);
15281
15282 ins_cost(INSN_COST);
15283 format %{ "cmp $op1, $op2" %}
15284
15285 ins_encode(aarch64_enc_cmp(op1, op2));
15286
15287 ins_pipe(icmp_reg_reg);
15288 %}
15289
15290 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15291 %{
15292 match(Set cr (CmpUL op1 zero));
15293
15294 effect(DEF cr, USE op1);
15295
15296 ins_cost(INSN_COST);
15297 format %{ "tst $op1" %}
15298
15299 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15300
15301 ins_pipe(icmp_reg_imm);
15302 %}
15303
15304 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15305 %{
15306 match(Set cr (CmpUL op1 op2));
15307
15308 effect(DEF cr, USE op1);
15309
15310 ins_cost(INSN_COST);
15311 format %{ "cmp $op1, $op2" %}
15312
15313 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15314
15315 ins_pipe(icmp_reg_imm);
15316 %}
15317
15318 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15319 %{
15320 match(Set cr (CmpUL op1 op2));
15321
15322 effect(DEF cr, USE op1);
15323
15324 ins_cost(INSN_COST * 2);
15325 format %{ "cmp $op1, $op2" %}
15326
15327 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15328
15329 ins_pipe(icmp_reg_imm);
15330 %}
15331
15332 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15333 %{
15334 match(Set cr (CmpP op1 op2));
15335
15336 effect(DEF cr, USE op1, USE op2);
15337
15338 ins_cost(INSN_COST);
15339 format %{ "cmp $op1, $op2\t // ptr" %}
15340
15341 ins_encode(aarch64_enc_cmpp(op1, op2));
15342
15343 ins_pipe(icmp_reg_reg);
15344 %}
15345
15346 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15347 %{
15348 match(Set cr (CmpN op1 op2));
15349
15350 effect(DEF cr, USE op1, USE op2);
15351
15352 ins_cost(INSN_COST);
15353 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15354
15355 ins_encode(aarch64_enc_cmpn(op1, op2));
15356
15357 ins_pipe(icmp_reg_reg);
15358 %}
15359
15360 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15361 %{
15362 match(Set cr (CmpP op1 zero));
15363
15364 effect(DEF cr, USE op1, USE zero);
15365
15366 ins_cost(INSN_COST);
15367 format %{ "cmp $op1, 0\t // ptr" %}
15368
15369 ins_encode(aarch64_enc_testp(op1));
15370
15371 ins_pipe(icmp_reg_imm);
15372 %}
15373
15374 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15375 %{
15376 match(Set cr (CmpN op1 zero));
15377
15378 effect(DEF cr, USE op1, USE zero);
15379
15380 ins_cost(INSN_COST);
15381 format %{ "cmp $op1, 0\t // compressed ptr" %}
15382
15383 ins_encode(aarch64_enc_testn(op1));
15384
15385 ins_pipe(icmp_reg_imm);
15386 %}
15387
15388 // FP comparisons
15389 //
15390 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15391 // using normal cmpOp. See declaration of rFlagsReg for details.
15392
15393 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15394 %{
15395 match(Set cr (CmpF src1 src2));
15396
15397 ins_cost(3 * INSN_COST);
15398 format %{ "fcmps $src1, $src2" %}
15399
15400 ins_encode %{
15401 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15402 %}
15403
15404 ins_pipe(pipe_class_compare);
15405 %}
15406
15407 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15408 %{
15409 match(Set cr (CmpF src1 src2));
15410
15411 ins_cost(3 * INSN_COST);
15412 format %{ "fcmps $src1, 0.0" %}
15413
15414 ins_encode %{
15415 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15416 %}
15417
15418 ins_pipe(pipe_class_compare);
15419 %}
15420 // FROM HERE
15421
15422 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15423 %{
15424 match(Set cr (CmpD src1 src2));
15425
15426 ins_cost(3 * INSN_COST);
15427 format %{ "fcmpd $src1, $src2" %}
15428
15429 ins_encode %{
15430 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15431 %}
15432
15433 ins_pipe(pipe_class_compare);
15434 %}
15435
15436 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15437 %{
15438 match(Set cr (CmpD src1 src2));
15439
15440 ins_cost(3 * INSN_COST);
15441 format %{ "fcmpd $src1, 0.0" %}
15442
15443 ins_encode %{
15444 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15445 %}
15446
15447 ins_pipe(pipe_class_compare);
15448 %}
15449
15450 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15451 %{
15452 match(Set dst (CmpF3 src1 src2));
15453 effect(KILL cr);
15454
15455 ins_cost(5 * INSN_COST);
15456 format %{ "fcmps $src1, $src2\n\t"
15457 "csinvw($dst, zr, zr, eq\n\t"
15458 "csnegw($dst, $dst, $dst, lt)"
15459 %}
15460
15461 ins_encode %{
15462 Label done;
15463 FloatRegister s1 = as_FloatRegister($src1$$reg);
15464 FloatRegister s2 = as_FloatRegister($src2$$reg);
15465 Register d = as_Register($dst$$reg);
15466 __ fcmps(s1, s2);
15467 // installs 0 if EQ else -1
15468 __ csinvw(d, zr, zr, Assembler::EQ);
15469 // keeps -1 if less or unordered else installs 1
15470 __ csnegw(d, d, d, Assembler::LT);
15471 __ bind(done);
15472 %}
15473
15474 ins_pipe(pipe_class_default);
15475
15476 %}
15477
15478 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15479 %{
15480 match(Set dst (CmpD3 src1 src2));
15481 effect(KILL cr);
15482
15483 ins_cost(5 * INSN_COST);
15484 format %{ "fcmpd $src1, $src2\n\t"
15485 "csinvw($dst, zr, zr, eq\n\t"
15486 "csnegw($dst, $dst, $dst, lt)"
15487 %}
15488
15489 ins_encode %{
15490 Label done;
15491 FloatRegister s1 = as_FloatRegister($src1$$reg);
15492 FloatRegister s2 = as_FloatRegister($src2$$reg);
15493 Register d = as_Register($dst$$reg);
15494 __ fcmpd(s1, s2);
15495 // installs 0 if EQ else -1
15496 __ csinvw(d, zr, zr, Assembler::EQ);
15497 // keeps -1 if less or unordered else installs 1
15498 __ csnegw(d, d, d, Assembler::LT);
15499 __ bind(done);
15500 %}
15501 ins_pipe(pipe_class_default);
15502
15503 %}
15504
15505 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15506 %{
15507 match(Set dst (CmpF3 src1 zero));
15508 effect(KILL cr);
15509
15510 ins_cost(5 * INSN_COST);
15511 format %{ "fcmps $src1, 0.0\n\t"
15512 "csinvw($dst, zr, zr, eq\n\t"
15513 "csnegw($dst, $dst, $dst, lt)"
15514 %}
15515
15516 ins_encode %{
15517 Label done;
15518 FloatRegister s1 = as_FloatRegister($src1$$reg);
15519 Register d = as_Register($dst$$reg);
15520 __ fcmps(s1, 0.0);
15521 // installs 0 if EQ else -1
15522 __ csinvw(d, zr, zr, Assembler::EQ);
15523 // keeps -1 if less or unordered else installs 1
15524 __ csnegw(d, d, d, Assembler::LT);
15525 __ bind(done);
15526 %}
15527
15528 ins_pipe(pipe_class_default);
15529
15530 %}
15531
15532 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15533 %{
15534 match(Set dst (CmpD3 src1 zero));
15535 effect(KILL cr);
15536
15537 ins_cost(5 * INSN_COST);
15538 format %{ "fcmpd $src1, 0.0\n\t"
15539 "csinvw($dst, zr, zr, eq\n\t"
15540 "csnegw($dst, $dst, $dst, lt)"
15541 %}
15542
15543 ins_encode %{
15544 Label done;
15545 FloatRegister s1 = as_FloatRegister($src1$$reg);
15546 Register d = as_Register($dst$$reg);
15547 __ fcmpd(s1, 0.0);
15548 // installs 0 if EQ else -1
15549 __ csinvw(d, zr, zr, Assembler::EQ);
15550 // keeps -1 if less or unordered else installs 1
15551 __ csnegw(d, d, d, Assembler::LT);
15552 __ bind(done);
15553 %}
15554 ins_pipe(pipe_class_default);
15555
15556 %}
15557
15558 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15559 %{
15560 match(Set dst (CmpLTMask p q));
15561 effect(KILL cr);
15562
15563 ins_cost(3 * INSN_COST);
15564
15565 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15566 "csetw $dst, lt\n\t"
15567 "subw $dst, zr, $dst"
15568 %}
15569
15570 ins_encode %{
15571 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15572 __ csetw(as_Register($dst$$reg), Assembler::LT);
15573 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15574 %}
15575
15576 ins_pipe(ialu_reg_reg);
15577 %}
15578
15579 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15580 %{
15581 match(Set dst (CmpLTMask src zero));
15582 effect(KILL cr);
15583
15584 ins_cost(INSN_COST);
15585
15586 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15587
15588 ins_encode %{
15589 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15590 %}
15591
15592 ins_pipe(ialu_reg_shift);
15593 %}
15594
15595 // ============================================================================
15596 // Max and Min
15597
15598 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15599
15600 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15601 %{
15602 effect(DEF cr, USE src);
15603 ins_cost(INSN_COST);
15604 format %{ "cmpw $src, 0" %}
15605
15606 ins_encode %{
15607 __ cmpw($src$$Register, 0);
15608 %}
15609 ins_pipe(icmp_reg_imm);
15610 %}
15611
15612 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15613 %{
15614 match(Set dst (MinI src1 src2));
15615 ins_cost(INSN_COST * 3);
15616
15617 expand %{
15618 rFlagsReg cr;
15619 compI_reg_reg(cr, src1, src2);
15620 cmovI_reg_reg_lt(dst, src1, src2, cr);
15621 %}
15622 %}
15623
15624 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15625 %{
15626 match(Set dst (MaxI src1 src2));
15627 ins_cost(INSN_COST * 3);
15628
15629 expand %{
15630 rFlagsReg cr;
15631 compI_reg_reg(cr, src1, src2);
15632 cmovI_reg_reg_gt(dst, src1, src2, cr);
15633 %}
15634 %}
15635
15636
15637 // ============================================================================
15638 // Branch Instructions
15639
15640 // Direct Branch.
15641 instruct branch(label lbl)
15642 %{
15643 match(Goto);
15644
15645 effect(USE lbl);
15646
15647 ins_cost(BRANCH_COST);
15648 format %{ "b $lbl" %}
15649
15650 ins_encode(aarch64_enc_b(lbl));
15651
15652 ins_pipe(pipe_branch);
15653 %}
15654
15655 // Conditional Near Branch
15656 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15657 %{
15658 // Same match rule as `branchConFar'.
15659 match(If cmp cr);
15660
15661 effect(USE lbl);
15662
15663 ins_cost(BRANCH_COST);
15664 // If set to 1 this indicates that the current instruction is a
15665 // short variant of a long branch. This avoids using this
15666 // instruction in first-pass matching. It will then only be used in
15667 // the `Shorten_branches' pass.
15668 // ins_short_branch(1);
15669 format %{ "b$cmp $lbl" %}
15670
15671 ins_encode(aarch64_enc_br_con(cmp, lbl));
15672
15673 ins_pipe(pipe_branch_cond);
15674 %}
15675
15676 // Conditional Near Branch Unsigned
15677 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15678 %{
15679 // Same match rule as `branchConFar'.
15680 match(If cmp cr);
15681
15682 effect(USE lbl);
15683
15684 ins_cost(BRANCH_COST);
15685 // If set to 1 this indicates that the current instruction is a
15686 // short variant of a long branch. This avoids using this
15687 // instruction in first-pass matching. It will then only be used in
15688 // the `Shorten_branches' pass.
15689 // ins_short_branch(1);
15690 format %{ "b$cmp $lbl\t# unsigned" %}
15691
15692 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15693
15694 ins_pipe(pipe_branch_cond);
15695 %}
15696
15697 // Make use of CBZ and CBNZ. These instructions, as well as being
15698 // shorter than (cmp; branch), have the additional benefit of not
15699 // killing the flags.
15700
15701 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15702 match(If cmp (CmpI op1 op2));
15703 effect(USE labl);
15704
15705 ins_cost(BRANCH_COST);
15706 format %{ "cbw$cmp $op1, $labl" %}
15707 ins_encode %{
15708 Label* L = $labl$$label;
15709 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15710 if (cond == Assembler::EQ)
15711 __ cbzw($op1$$Register, *L);
15712 else
15713 __ cbnzw($op1$$Register, *L);
15714 %}
15715 ins_pipe(pipe_cmp_branch);
15716 %}
15717
15718 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15719 match(If cmp (CmpL op1 op2));
15720 effect(USE labl);
15721
15722 ins_cost(BRANCH_COST);
15723 format %{ "cb$cmp $op1, $labl" %}
15724 ins_encode %{
15725 Label* L = $labl$$label;
15726 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15727 if (cond == Assembler::EQ)
15728 __ cbz($op1$$Register, *L);
15729 else
15730 __ cbnz($op1$$Register, *L);
15731 %}
15732 ins_pipe(pipe_cmp_branch);
15733 %}
15734
15735 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15736 match(If cmp (CmpP op1 op2));
15737 effect(USE labl);
15738
15739 ins_cost(BRANCH_COST);
15740 format %{ "cb$cmp $op1, $labl" %}
15741 ins_encode %{
15742 Label* L = $labl$$label;
15743 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15744 if (cond == Assembler::EQ)
15745 __ cbz($op1$$Register, *L);
15746 else
15747 __ cbnz($op1$$Register, *L);
15748 %}
15749 ins_pipe(pipe_cmp_branch);
15750 %}
15751
15752 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15753 match(If cmp (CmpN op1 op2));
15754 effect(USE labl);
15755
15756 ins_cost(BRANCH_COST);
15757 format %{ "cbw$cmp $op1, $labl" %}
15758 ins_encode %{
15759 Label* L = $labl$$label;
15760 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15761 if (cond == Assembler::EQ)
15762 __ cbzw($op1$$Register, *L);
15763 else
15764 __ cbnzw($op1$$Register, *L);
15765 %}
15766 ins_pipe(pipe_cmp_branch);
15767 %}
15768
15769 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15770 match(If cmp (CmpP (DecodeN oop) zero));
15771 effect(USE labl);
15772
15773 ins_cost(BRANCH_COST);
15774 format %{ "cb$cmp $oop, $labl" %}
15775 ins_encode %{
15776 Label* L = $labl$$label;
15777 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15778 if (cond == Assembler::EQ)
15779 __ cbzw($oop$$Register, *L);
15780 else
15781 __ cbnzw($oop$$Register, *L);
15782 %}
15783 ins_pipe(pipe_cmp_branch);
15784 %}
15785
15786 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15787 match(If cmp (CmpU op1 op2));
15788 effect(USE labl);
15789
15790 ins_cost(BRANCH_COST);
15791 format %{ "cbw$cmp $op1, $labl" %}
15792 ins_encode %{
15793 Label* L = $labl$$label;
15794 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15795 if (cond == Assembler::EQ || cond == Assembler::LS) {
15796 __ cbzw($op1$$Register, *L);
15797 } else {
15798 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15799 __ cbnzw($op1$$Register, *L);
15800 }
15801 %}
15802 ins_pipe(pipe_cmp_branch);
15803 %}
15804
15805 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15806 match(If cmp (CmpUL op1 op2));
15807 effect(USE labl);
15808
15809 ins_cost(BRANCH_COST);
15810 format %{ "cb$cmp $op1, $labl" %}
15811 ins_encode %{
15812 Label* L = $labl$$label;
15813 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15814 if (cond == Assembler::EQ || cond == Assembler::LS) {
15815 __ cbz($op1$$Register, *L);
15816 } else {
15817 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15818 __ cbnz($op1$$Register, *L);
15819 }
15820 %}
15821 ins_pipe(pipe_cmp_branch);
15822 %}
15823
15824 // Test bit and Branch
15825
15826 // Patterns for short (< 32KiB) variants
15827 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15828 match(If cmp (CmpL op1 op2));
15829 effect(USE labl);
15830
15831 ins_cost(BRANCH_COST);
15832 format %{ "cb$cmp $op1, $labl # long" %}
15833 ins_encode %{
15834 Label* L = $labl$$label;
15835 Assembler::Condition cond =
15836 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15837 __ tbr(cond, $op1$$Register, 63, *L);
15838 %}
15839 ins_pipe(pipe_cmp_branch);
15840 ins_short_branch(1);
15841 %}
15842
15843 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15844 match(If cmp (CmpI op1 op2));
15845 effect(USE labl);
15846
15847 ins_cost(BRANCH_COST);
15848 format %{ "cb$cmp $op1, $labl # int" %}
15849 ins_encode %{
15850 Label* L = $labl$$label;
15851 Assembler::Condition cond =
15852 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15853 __ tbr(cond, $op1$$Register, 31, *L);
15854 %}
15855 ins_pipe(pipe_cmp_branch);
15856 ins_short_branch(1);
15857 %}
15858
15859 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15860 match(If cmp (CmpL (AndL op1 op2) op3));
15861 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15862 effect(USE labl);
15863
15864 ins_cost(BRANCH_COST);
15865 format %{ "tb$cmp $op1, $op2, $labl" %}
15866 ins_encode %{
15867 Label* L = $labl$$label;
15868 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15869 int bit = exact_log2_long($op2$$constant);
15870 __ tbr(cond, $op1$$Register, bit, *L);
15871 %}
15872 ins_pipe(pipe_cmp_branch);
15873 ins_short_branch(1);
15874 %}
15875
15876 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15877 match(If cmp (CmpI (AndI op1 op2) op3));
15878 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15879 effect(USE labl);
15880
15881 ins_cost(BRANCH_COST);
15882 format %{ "tb$cmp $op1, $op2, $labl" %}
15883 ins_encode %{
15884 Label* L = $labl$$label;
15885 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15886 int bit = exact_log2((juint)$op2$$constant);
15887 __ tbr(cond, $op1$$Register, bit, *L);
15888 %}
15889 ins_pipe(pipe_cmp_branch);
15890 ins_short_branch(1);
15891 %}
15892
15893 // And far variants
15894 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15895 match(If cmp (CmpL op1 op2));
15896 effect(USE labl);
15897
15898 ins_cost(BRANCH_COST);
15899 format %{ "cb$cmp $op1, $labl # long" %}
15900 ins_encode %{
15901 Label* L = $labl$$label;
15902 Assembler::Condition cond =
15903 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15904 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15905 %}
15906 ins_pipe(pipe_cmp_branch);
15907 %}
15908
15909 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15910 match(If cmp (CmpI op1 op2));
15911 effect(USE labl);
15912
15913 ins_cost(BRANCH_COST);
15914 format %{ "cb$cmp $op1, $labl # int" %}
15915 ins_encode %{
15916 Label* L = $labl$$label;
15917 Assembler::Condition cond =
15918 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15919 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15920 %}
15921 ins_pipe(pipe_cmp_branch);
15922 %}
15923
15924 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15925 match(If cmp (CmpL (AndL op1 op2) op3));
15926 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15927 effect(USE labl);
15928
15929 ins_cost(BRANCH_COST);
15930 format %{ "tb$cmp $op1, $op2, $labl" %}
15931 ins_encode %{
15932 Label* L = $labl$$label;
15933 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15934 int bit = exact_log2_long($op2$$constant);
15935 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15936 %}
15937 ins_pipe(pipe_cmp_branch);
15938 %}
15939
15940 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15941 match(If cmp (CmpI (AndI op1 op2) op3));
15942 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15943 effect(USE labl);
15944
15945 ins_cost(BRANCH_COST);
15946 format %{ "tb$cmp $op1, $op2, $labl" %}
15947 ins_encode %{
15948 Label* L = $labl$$label;
15949 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15950 int bit = exact_log2((juint)$op2$$constant);
15951 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15952 %}
15953 ins_pipe(pipe_cmp_branch);
15954 %}
15955
15956 // Test bits
15957
15958 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15959 match(Set cr (CmpL (AndL op1 op2) op3));
15960 predicate(Assembler::operand_valid_for_logical_immediate
15961 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15962
15963 ins_cost(INSN_COST);
15964 format %{ "tst $op1, $op2 # long" %}
15965 ins_encode %{
15966 __ tst($op1$$Register, $op2$$constant);
15967 %}
15968 ins_pipe(ialu_reg_reg);
15969 %}
15970
15971 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15972 match(Set cr (CmpI (AndI op1 op2) op3));
15973 predicate(Assembler::operand_valid_for_logical_immediate
15974 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15975
15976 ins_cost(INSN_COST);
15977 format %{ "tst $op1, $op2 # int" %}
15978 ins_encode %{
15979 __ tstw($op1$$Register, $op2$$constant);
15980 %}
15981 ins_pipe(ialu_reg_reg);
15982 %}
15983
15984 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15985 match(Set cr (CmpL (AndL op1 op2) op3));
15986
15987 ins_cost(INSN_COST);
15988 format %{ "tst $op1, $op2 # long" %}
15989 ins_encode %{
15990 __ tst($op1$$Register, $op2$$Register);
15991 %}
15992 ins_pipe(ialu_reg_reg);
15993 %}
15994
15995 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15996 match(Set cr (CmpI (AndI op1 op2) op3));
15997
15998 ins_cost(INSN_COST);
15999 format %{ "tstw $op1, $op2 # int" %}
16000 ins_encode %{
16001 __ tstw($op1$$Register, $op2$$Register);
16002 %}
16003 ins_pipe(ialu_reg_reg);
16004 %}
16005
16006
16007 // Conditional Far Branch
16008 // Conditional Far Branch Unsigned
16009 // TODO: fixme
16010
16011 // counted loop end branch near
16012 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16013 %{
16014 match(CountedLoopEnd cmp cr);
16015
16016 effect(USE lbl);
16017
16018 ins_cost(BRANCH_COST);
16019 // short variant.
16020 // ins_short_branch(1);
16021 format %{ "b$cmp $lbl \t// counted loop end" %}
16022
16023 ins_encode(aarch64_enc_br_con(cmp, lbl));
16024
16025 ins_pipe(pipe_branch);
16026 %}
16027
16028 // counted loop end branch far
16029 // TODO: fixme
16030
16031 // ============================================================================
16032 // inlined locking and unlocking
16033
16034 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16035 %{
16036 predicate(LockingMode != LM_LIGHTWEIGHT);
16037 match(Set cr (FastLock object box));
16038 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16039
16040 ins_cost(5 * INSN_COST);
16041 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16042
16043 ins_encode %{
16044 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16045 %}
16046
16047 ins_pipe(pipe_serial);
16048 %}
16049
16050 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16051 %{
16052 predicate(LockingMode != LM_LIGHTWEIGHT);
16053 match(Set cr (FastUnlock object box));
16054 effect(TEMP tmp, TEMP tmp2);
16055
16056 ins_cost(5 * INSN_COST);
16057 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16058
16059 ins_encode %{
16060 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16061 %}
16062
16063 ins_pipe(pipe_serial);
16064 %}
16065
16066 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16067 %{
16068 predicate(LockingMode == LM_LIGHTWEIGHT);
16069 match(Set cr (FastLock object box));
16070 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16071
16072 ins_cost(5 * INSN_COST);
16073 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16074
16075 ins_encode %{
16076 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16077 %}
16078
16079 ins_pipe(pipe_serial);
16080 %}
16081
16082 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16083 %{
16084 predicate(LockingMode == LM_LIGHTWEIGHT);
16085 match(Set cr (FastUnlock object box));
16086 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16087
16088 ins_cost(5 * INSN_COST);
16089 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16090
16091 ins_encode %{
16092 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16093 %}
16094
16095 ins_pipe(pipe_serial);
16096 %}
16097
16098 // ============================================================================
16099 // Safepoint Instructions
16100
16101 // TODO
16102 // provide a near and far version of this code
16103
16104 instruct safePoint(rFlagsReg cr, iRegP poll)
16105 %{
16106 match(SafePoint poll);
16107 effect(KILL cr);
16108
16109 format %{
16110 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16111 %}
16112 ins_encode %{
16113 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16114 %}
16115 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16116 %}
16117
16118
16119 // ============================================================================
16120 // Procedure Call/Return Instructions
16121
16122 // Call Java Static Instruction
16123
16124 instruct CallStaticJavaDirect(method meth)
16125 %{
16126 match(CallStaticJava);
16127
16128 effect(USE meth);
16129
16130 ins_cost(CALL_COST);
16131
16132 format %{ "call,static $meth \t// ==> " %}
16133
16134 ins_encode(aarch64_enc_java_static_call(meth),
16135 aarch64_enc_call_epilog);
16136
16137 ins_pipe(pipe_class_call);
16138 %}
16139
16140 // TO HERE
16141
16142 // Call Java Dynamic Instruction
16143 instruct CallDynamicJavaDirect(method meth)
16144 %{
16145 match(CallDynamicJava);
16146
16147 effect(USE meth);
16148
16149 ins_cost(CALL_COST);
16150
16151 format %{ "CALL,dynamic $meth \t// ==> " %}
16152
16153 ins_encode(aarch64_enc_java_dynamic_call(meth),
16154 aarch64_enc_call_epilog);
16155
16156 ins_pipe(pipe_class_call);
16157 %}
16158
16159 // Call Runtime Instruction
16160
16161 instruct CallRuntimeDirect(method meth)
16162 %{
16163 match(CallRuntime);
16164
16165 effect(USE meth);
16166
16167 ins_cost(CALL_COST);
16168
16169 format %{ "CALL, runtime $meth" %}
16170
16171 ins_encode( aarch64_enc_java_to_runtime(meth) );
16172
16173 ins_pipe(pipe_class_call);
16174 %}
16175
16176 // Call Runtime Instruction
16177
16178 instruct CallLeafDirect(method meth)
16179 %{
16180 match(CallLeaf);
16181
16182 effect(USE meth);
16183
16184 ins_cost(CALL_COST);
16185
16186 format %{ "CALL, runtime leaf $meth" %}
16187
16188 ins_encode( aarch64_enc_java_to_runtime(meth) );
16189
16190 ins_pipe(pipe_class_call);
16191 %}
16192
16193 // Call Runtime Instruction without safepoint and with vector arguments
16194 instruct CallLeafDirectVector(method meth)
16195 %{
16196 match(CallLeafVector);
16197
16198 effect(USE meth);
16199
16200 ins_cost(CALL_COST);
16201
16202 format %{ "CALL, runtime leaf vector $meth" %}
16203
16204 ins_encode(aarch64_enc_java_to_runtime(meth));
16205
16206 ins_pipe(pipe_class_call);
16207 %}
16208
16209 // Call Runtime Instruction
16210
16211 instruct CallLeafNoFPDirect(method meth)
16212 %{
16213 match(CallLeafNoFP);
16214
16215 effect(USE meth);
16216
16217 ins_cost(CALL_COST);
16218
16219 format %{ "CALL, runtime leaf nofp $meth" %}
16220
16221 ins_encode( aarch64_enc_java_to_runtime(meth) );
16222
16223 ins_pipe(pipe_class_call);
16224 %}
16225
16226 // Tail Call; Jump from runtime stub to Java code.
16227 // Also known as an 'interprocedural jump'.
16228 // Target of jump will eventually return to caller.
16229 // TailJump below removes the return address.
16230 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16231 // emitted just above the TailCall which has reset rfp to the caller state.
16232 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16233 %{
16234 match(TailCall jump_target method_ptr);
16235
16236 ins_cost(CALL_COST);
16237
16238 format %{ "br $jump_target\t# $method_ptr holds method" %}
16239
16240 ins_encode(aarch64_enc_tail_call(jump_target));
16241
16242 ins_pipe(pipe_class_call);
16243 %}
16244
16245 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16246 %{
16247 match(TailJump jump_target ex_oop);
16248
16249 ins_cost(CALL_COST);
16250
16251 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16252
16253 ins_encode(aarch64_enc_tail_jmp(jump_target));
16254
16255 ins_pipe(pipe_class_call);
16256 %}
16257
16258 // Forward exception.
16259 instruct ForwardExceptionjmp()
16260 %{
16261 match(ForwardException);
16262 ins_cost(CALL_COST);
16263
16264 format %{ "b forward_exception_stub" %}
16265 ins_encode %{
16266 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16267 %}
16268 ins_pipe(pipe_class_call);
16269 %}
16270
16271 // Create exception oop: created by stack-crawling runtime code.
16272 // Created exception is now available to this handler, and is setup
16273 // just prior to jumping to this handler. No code emitted.
16274 // TODO check
16275 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16276 instruct CreateException(iRegP_R0 ex_oop)
16277 %{
16278 match(Set ex_oop (CreateEx));
16279
16280 format %{ " -- \t// exception oop; no code emitted" %}
16281
16282 size(0);
16283
16284 ins_encode( /*empty*/ );
16285
16286 ins_pipe(pipe_class_empty);
16287 %}
16288
16289 // Rethrow exception: The exception oop will come in the first
16290 // argument position. Then JUMP (not call) to the rethrow stub code.
16291 instruct RethrowException() %{
16292 match(Rethrow);
16293 ins_cost(CALL_COST);
16294
16295 format %{ "b rethrow_stub" %}
16296
16297 ins_encode( aarch64_enc_rethrow() );
16298
16299 ins_pipe(pipe_class_call);
16300 %}
16301
16302
16303 // Return Instruction
16304 // epilog node loads ret address into lr as part of frame pop
16305 instruct Ret()
16306 %{
16307 match(Return);
16308
16309 format %{ "ret\t// return register" %}
16310
16311 ins_encode( aarch64_enc_ret() );
16312
16313 ins_pipe(pipe_branch);
16314 %}
16315
16316 // Die now.
16317 instruct ShouldNotReachHere() %{
16318 match(Halt);
16319
16320 ins_cost(CALL_COST);
16321 format %{ "ShouldNotReachHere" %}
16322
16323 ins_encode %{
16324 if (is_reachable()) {
16325 __ stop(_halt_reason);
16326 }
16327 %}
16328
16329 ins_pipe(pipe_class_default);
16330 %}
16331
16332 // ============================================================================
16333 // Partial Subtype Check
16334 //
16335 // superklass array for an instance of the superklass. Set a hidden
16336 // internal cache on a hit (cache is checked with exposed code in
16337 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16338 // encoding ALSO sets flags.
16339
16340 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16341 %{
16342 match(Set result (PartialSubtypeCheck sub super));
16343 predicate(!UseSecondarySupersTable);
16344 effect(KILL cr, KILL temp);
16345
16346 ins_cost(20 * INSN_COST); // slightly larger than the next version
16347 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16348
16349 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16350
16351 opcode(0x1); // Force zero of result reg on hit
16352
16353 ins_pipe(pipe_class_memory);
16354 %}
16355
16356 // Two versions of partialSubtypeCheck, both used when we need to
16357 // search for a super class in the secondary supers array. The first
16358 // is used when we don't know _a priori_ the class being searched
16359 // for. The second, far more common, is used when we do know: this is
16360 // used for instanceof, checkcast, and any case where C2 can determine
16361 // it by constant propagation.
16362
16363 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16364 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16365 rFlagsReg cr)
16366 %{
16367 match(Set result (PartialSubtypeCheck sub super));
16368 predicate(UseSecondarySupersTable);
16369 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16370
16371 ins_cost(10 * INSN_COST); // slightly larger than the next version
16372 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16373
16374 ins_encode %{
16375 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16376 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16377 $vtemp$$FloatRegister,
16378 $result$$Register, /*L_success*/nullptr);
16379 %}
16380
16381 ins_pipe(pipe_class_memory);
16382 %}
16383
16384 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16385 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16386 rFlagsReg cr)
16387 %{
16388 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16389 predicate(UseSecondarySupersTable);
16390 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16391
16392 ins_cost(5 * INSN_COST); // smaller than the next version
16393 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16394
16395 ins_encode %{
16396 bool success = false;
16397 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16398 if (InlineSecondarySupersTest) {
16399 success =
16400 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16401 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16402 $vtemp$$FloatRegister,
16403 $result$$Register,
16404 super_klass_slot);
16405 } else {
16406 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16407 success = (call != nullptr);
16408 }
16409 if (!success) {
16410 ciEnv::current()->record_failure("CodeCache is full");
16411 return;
16412 }
16413 %}
16414
16415 ins_pipe(pipe_class_memory);
16416 %}
16417
16418 // Intrisics for String.compareTo()
16419
16420 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16421 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16422 %{
16423 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16424 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16425 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16426
16427 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16428 ins_encode %{
16429 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16430 __ string_compare($str1$$Register, $str2$$Register,
16431 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16432 $tmp1$$Register, $tmp2$$Register,
16433 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16434 %}
16435 ins_pipe(pipe_class_memory);
16436 %}
16437
16438 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16439 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16440 %{
16441 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16442 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16443 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16444
16445 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16446 ins_encode %{
16447 __ string_compare($str1$$Register, $str2$$Register,
16448 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16449 $tmp1$$Register, $tmp2$$Register,
16450 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16451 %}
16452 ins_pipe(pipe_class_memory);
16453 %}
16454
16455 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16456 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16457 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16458 %{
16459 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16460 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16461 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16462 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16463
16464 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16465 ins_encode %{
16466 __ string_compare($str1$$Register, $str2$$Register,
16467 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16468 $tmp1$$Register, $tmp2$$Register,
16469 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16470 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16471 %}
16472 ins_pipe(pipe_class_memory);
16473 %}
16474
16475 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16476 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16477 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16478 %{
16479 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16480 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16481 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16482 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16483
16484 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16485 ins_encode %{
16486 __ string_compare($str1$$Register, $str2$$Register,
16487 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16488 $tmp1$$Register, $tmp2$$Register,
16489 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16490 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16491 %}
16492 ins_pipe(pipe_class_memory);
16493 %}
16494
16495 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16496 // these string_compare variants as NEON register type for convenience so that the prototype of
16497 // string_compare can be shared with all variants.
16498
16499 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16500 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16501 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16502 pRegGov_P1 pgtmp2, rFlagsReg cr)
16503 %{
16504 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16505 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16506 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16507 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16508
16509 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16510 ins_encode %{
16511 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16512 __ string_compare($str1$$Register, $str2$$Register,
16513 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16514 $tmp1$$Register, $tmp2$$Register,
16515 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16516 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16517 StrIntrinsicNode::LL);
16518 %}
16519 ins_pipe(pipe_class_memory);
16520 %}
16521
16522 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16523 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16524 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16525 pRegGov_P1 pgtmp2, rFlagsReg cr)
16526 %{
16527 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16528 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16529 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16530 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16531
16532 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16533 ins_encode %{
16534 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16535 __ string_compare($str1$$Register, $str2$$Register,
16536 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16537 $tmp1$$Register, $tmp2$$Register,
16538 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16539 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16540 StrIntrinsicNode::LU);
16541 %}
16542 ins_pipe(pipe_class_memory);
16543 %}
16544
16545 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16546 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16547 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16548 pRegGov_P1 pgtmp2, rFlagsReg cr)
16549 %{
16550 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16551 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16552 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16553 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16554
16555 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16556 ins_encode %{
16557 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16558 __ string_compare($str1$$Register, $str2$$Register,
16559 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16560 $tmp1$$Register, $tmp2$$Register,
16561 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16562 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16563 StrIntrinsicNode::UL);
16564 %}
16565 ins_pipe(pipe_class_memory);
16566 %}
16567
16568 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16569 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16570 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16571 pRegGov_P1 pgtmp2, rFlagsReg cr)
16572 %{
16573 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16574 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16575 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16576 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16577
16578 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16579 ins_encode %{
16580 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16581 __ string_compare($str1$$Register, $str2$$Register,
16582 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16583 $tmp1$$Register, $tmp2$$Register,
16584 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16585 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16586 StrIntrinsicNode::UU);
16587 %}
16588 ins_pipe(pipe_class_memory);
16589 %}
16590
16591 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16592 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16593 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16594 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16595 %{
16596 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16597 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16598 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16599 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16600 TEMP vtmp0, TEMP vtmp1, KILL cr);
16601 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16602 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16603
16604 ins_encode %{
16605 __ string_indexof($str1$$Register, $str2$$Register,
16606 $cnt1$$Register, $cnt2$$Register,
16607 $tmp1$$Register, $tmp2$$Register,
16608 $tmp3$$Register, $tmp4$$Register,
16609 $tmp5$$Register, $tmp6$$Register,
16610 -1, $result$$Register, StrIntrinsicNode::UU);
16611 %}
16612 ins_pipe(pipe_class_memory);
16613 %}
16614
16615 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16616 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16617 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16618 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16619 %{
16620 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16621 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16622 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16623 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16624 TEMP vtmp0, TEMP vtmp1, KILL cr);
16625 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16626 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16627
16628 ins_encode %{
16629 __ string_indexof($str1$$Register, $str2$$Register,
16630 $cnt1$$Register, $cnt2$$Register,
16631 $tmp1$$Register, $tmp2$$Register,
16632 $tmp3$$Register, $tmp4$$Register,
16633 $tmp5$$Register, $tmp6$$Register,
16634 -1, $result$$Register, StrIntrinsicNode::LL);
16635 %}
16636 ins_pipe(pipe_class_memory);
16637 %}
16638
16639 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16640 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16641 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16642 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16643 %{
16644 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16645 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16646 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16647 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16648 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16649 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16650 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16651
16652 ins_encode %{
16653 __ string_indexof($str1$$Register, $str2$$Register,
16654 $cnt1$$Register, $cnt2$$Register,
16655 $tmp1$$Register, $tmp2$$Register,
16656 $tmp3$$Register, $tmp4$$Register,
16657 $tmp5$$Register, $tmp6$$Register,
16658 -1, $result$$Register, StrIntrinsicNode::UL);
16659 %}
16660 ins_pipe(pipe_class_memory);
16661 %}
16662
16663 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16664 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16665 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16666 %{
16667 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16668 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16669 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16670 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16671 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16672 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16673
16674 ins_encode %{
16675 int icnt2 = (int)$int_cnt2$$constant;
16676 __ string_indexof($str1$$Register, $str2$$Register,
16677 $cnt1$$Register, zr,
16678 $tmp1$$Register, $tmp2$$Register,
16679 $tmp3$$Register, $tmp4$$Register, zr, zr,
16680 icnt2, $result$$Register, StrIntrinsicNode::UU);
16681 %}
16682 ins_pipe(pipe_class_memory);
16683 %}
16684
16685 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16686 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16687 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16688 %{
16689 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16690 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16691 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16692 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16693 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16694 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16695
16696 ins_encode %{
16697 int icnt2 = (int)$int_cnt2$$constant;
16698 __ string_indexof($str1$$Register, $str2$$Register,
16699 $cnt1$$Register, zr,
16700 $tmp1$$Register, $tmp2$$Register,
16701 $tmp3$$Register, $tmp4$$Register, zr, zr,
16702 icnt2, $result$$Register, StrIntrinsicNode::LL);
16703 %}
16704 ins_pipe(pipe_class_memory);
16705 %}
16706
16707 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16708 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16709 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16710 %{
16711 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16712 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16713 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16714 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16715 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16716 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16717
16718 ins_encode %{
16719 int icnt2 = (int)$int_cnt2$$constant;
16720 __ string_indexof($str1$$Register, $str2$$Register,
16721 $cnt1$$Register, zr,
16722 $tmp1$$Register, $tmp2$$Register,
16723 $tmp3$$Register, $tmp4$$Register, zr, zr,
16724 icnt2, $result$$Register, StrIntrinsicNode::UL);
16725 %}
16726 ins_pipe(pipe_class_memory);
16727 %}
16728
16729 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16730 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16731 iRegINoSp tmp3, rFlagsReg cr)
16732 %{
16733 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16734 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16735 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16736 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16737
16738 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16739
16740 ins_encode %{
16741 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16742 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16743 $tmp3$$Register);
16744 %}
16745 ins_pipe(pipe_class_memory);
16746 %}
16747
16748 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16749 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16750 iRegINoSp tmp3, rFlagsReg cr)
16751 %{
16752 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16753 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16754 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16755 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16756
16757 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16758
16759 ins_encode %{
16760 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16761 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16762 $tmp3$$Register);
16763 %}
16764 ins_pipe(pipe_class_memory);
16765 %}
16766
16767 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16768 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16769 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16770 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16771 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16772 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16773 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16774 ins_encode %{
16775 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16776 $result$$Register, $ztmp1$$FloatRegister,
16777 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16778 $ptmp$$PRegister, true /* isL */);
16779 %}
16780 ins_pipe(pipe_class_memory);
16781 %}
16782
16783 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16784 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16785 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16786 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16787 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16788 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16789 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16790 ins_encode %{
16791 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16792 $result$$Register, $ztmp1$$FloatRegister,
16793 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16794 $ptmp$$PRegister, false /* isL */);
16795 %}
16796 ins_pipe(pipe_class_memory);
16797 %}
16798
16799 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16800 iRegI_R0 result, rFlagsReg cr)
16801 %{
16802 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16803 match(Set result (StrEquals (Binary str1 str2) cnt));
16804 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16805
16806 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16807 ins_encode %{
16808 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16809 __ string_equals($str1$$Register, $str2$$Register,
16810 $result$$Register, $cnt$$Register);
16811 %}
16812 ins_pipe(pipe_class_memory);
16813 %}
16814
16815 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16816 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16817 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16818 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16819 iRegP_R10 tmp, rFlagsReg cr)
16820 %{
16821 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16822 match(Set result (AryEq ary1 ary2));
16823 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16824 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16825 TEMP vtmp6, TEMP vtmp7, KILL cr);
16826
16827 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16828 ins_encode %{
16829 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16830 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16831 $result$$Register, $tmp$$Register, 1);
16832 if (tpc == nullptr) {
16833 ciEnv::current()->record_failure("CodeCache is full");
16834 return;
16835 }
16836 %}
16837 ins_pipe(pipe_class_memory);
16838 %}
16839
16840 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16841 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16842 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16843 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16844 iRegP_R10 tmp, rFlagsReg cr)
16845 %{
16846 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16847 match(Set result (AryEq ary1 ary2));
16848 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16849 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16850 TEMP vtmp6, TEMP vtmp7, KILL cr);
16851
16852 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16853 ins_encode %{
16854 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16855 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16856 $result$$Register, $tmp$$Register, 2);
16857 if (tpc == nullptr) {
16858 ciEnv::current()->record_failure("CodeCache is full");
16859 return;
16860 }
16861 %}
16862 ins_pipe(pipe_class_memory);
16863 %}
16864
16865 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16866 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16867 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16868 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16869 %{
16870 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16871 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16872 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16873
16874 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16875 ins_encode %{
16876 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16877 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16878 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16879 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16880 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16881 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16882 (BasicType)$basic_type$$constant);
16883 if (tpc == nullptr) {
16884 ciEnv::current()->record_failure("CodeCache is full");
16885 return;
16886 }
16887 %}
16888 ins_pipe(pipe_class_memory);
16889 %}
16890
16891 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16892 %{
16893 match(Set result (CountPositives ary1 len));
16894 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16895 format %{ "count positives byte[] $ary1,$len -> $result" %}
16896 ins_encode %{
16897 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16898 if (tpc == nullptr) {
16899 ciEnv::current()->record_failure("CodeCache is full");
16900 return;
16901 }
16902 %}
16903 ins_pipe( pipe_slow );
16904 %}
16905
16906 // fast char[] to byte[] compression
16907 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16908 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16909 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16910 iRegI_R0 result, rFlagsReg cr)
16911 %{
16912 match(Set result (StrCompressedCopy src (Binary dst len)));
16913 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16914 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16915
16916 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16917 ins_encode %{
16918 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16919 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16920 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16921 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16922 %}
16923 ins_pipe(pipe_slow);
16924 %}
16925
16926 // fast byte[] to char[] inflation
16927 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16928 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16929 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16930 %{
16931 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16932 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16933 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16934 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16935
16936 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16937 ins_encode %{
16938 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16939 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16940 $vtmp2$$FloatRegister, $tmp$$Register);
16941 if (tpc == nullptr) {
16942 ciEnv::current()->record_failure("CodeCache is full");
16943 return;
16944 }
16945 %}
16946 ins_pipe(pipe_class_memory);
16947 %}
16948
16949 // encode char[] to byte[] in ISO_8859_1
16950 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16951 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16952 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16953 iRegI_R0 result, rFlagsReg cr)
16954 %{
16955 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16956 match(Set result (EncodeISOArray src (Binary dst len)));
16957 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16958 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16959
16960 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16961 ins_encode %{
16962 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16963 $result$$Register, false,
16964 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16965 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16966 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16967 %}
16968 ins_pipe(pipe_class_memory);
16969 %}
16970
16971 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16972 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16973 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16974 iRegI_R0 result, rFlagsReg cr)
16975 %{
16976 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16977 match(Set result (EncodeISOArray src (Binary dst len)));
16978 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16979 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16980
16981 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16982 ins_encode %{
16983 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16984 $result$$Register, true,
16985 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16986 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16987 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16988 %}
16989 ins_pipe(pipe_class_memory);
16990 %}
16991
16992 //----------------------------- CompressBits/ExpandBits ------------------------
16993
16994 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16995 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16996 match(Set dst (CompressBits src mask));
16997 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16998 format %{ "mov $tsrc, $src\n\t"
16999 "mov $tmask, $mask\n\t"
17000 "bext $tdst, $tsrc, $tmask\n\t"
17001 "mov $dst, $tdst"
17002 %}
17003 ins_encode %{
17004 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17005 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17006 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17007 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17008 %}
17009 ins_pipe(pipe_slow);
17010 %}
17011
17012 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17013 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17014 match(Set dst (CompressBits (LoadI mem) mask));
17015 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17016 format %{ "ldrs $tsrc, $mem\n\t"
17017 "ldrs $tmask, $mask\n\t"
17018 "bext $tdst, $tsrc, $tmask\n\t"
17019 "mov $dst, $tdst"
17020 %}
17021 ins_encode %{
17022 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17024 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17025 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17026 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17027 %}
17028 ins_pipe(pipe_slow);
17029 %}
17030
17031 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17032 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17033 match(Set dst (CompressBits src mask));
17034 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17035 format %{ "mov $tsrc, $src\n\t"
17036 "mov $tmask, $mask\n\t"
17037 "bext $tdst, $tsrc, $tmask\n\t"
17038 "mov $dst, $tdst"
17039 %}
17040 ins_encode %{
17041 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17042 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17043 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17044 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17045 %}
17046 ins_pipe(pipe_slow);
17047 %}
17048
17049 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17050 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17051 match(Set dst (CompressBits (LoadL mem) mask));
17052 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17053 format %{ "ldrd $tsrc, $mem\n\t"
17054 "ldrd $tmask, $mask\n\t"
17055 "bext $tdst, $tsrc, $tmask\n\t"
17056 "mov $dst, $tdst"
17057 %}
17058 ins_encode %{
17059 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17060 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17061 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17062 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17063 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17064 %}
17065 ins_pipe(pipe_slow);
17066 %}
17067
17068 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17069 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17070 match(Set dst (ExpandBits src mask));
17071 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17072 format %{ "mov $tsrc, $src\n\t"
17073 "mov $tmask, $mask\n\t"
17074 "bdep $tdst, $tsrc, $tmask\n\t"
17075 "mov $dst, $tdst"
17076 %}
17077 ins_encode %{
17078 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17079 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17080 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17081 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17082 %}
17083 ins_pipe(pipe_slow);
17084 %}
17085
17086 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17087 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17088 match(Set dst (ExpandBits (LoadI mem) mask));
17089 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17090 format %{ "ldrs $tsrc, $mem\n\t"
17091 "ldrs $tmask, $mask\n\t"
17092 "bdep $tdst, $tsrc, $tmask\n\t"
17093 "mov $dst, $tdst"
17094 %}
17095 ins_encode %{
17096 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17097 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17098 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17099 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17100 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17101 %}
17102 ins_pipe(pipe_slow);
17103 %}
17104
17105 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17106 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17107 match(Set dst (ExpandBits src mask));
17108 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17109 format %{ "mov $tsrc, $src\n\t"
17110 "mov $tmask, $mask\n\t"
17111 "bdep $tdst, $tsrc, $tmask\n\t"
17112 "mov $dst, $tdst"
17113 %}
17114 ins_encode %{
17115 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17116 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17117 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17118 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17119 %}
17120 ins_pipe(pipe_slow);
17121 %}
17122
17123
17124 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17125 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17126 match(Set dst (ExpandBits (LoadL mem) mask));
17127 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17128 format %{ "ldrd $tsrc, $mem\n\t"
17129 "ldrd $tmask, $mask\n\t"
17130 "bdep $tdst, $tsrc, $tmask\n\t"
17131 "mov $dst, $tdst"
17132 %}
17133 ins_encode %{
17134 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17135 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17136 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17137 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17138 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17139 %}
17140 ins_pipe(pipe_slow);
17141 %}
17142
17143 // ============================================================================
17144 // This name is KNOWN by the ADLC and cannot be changed.
17145 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17146 // for this guy.
17147 instruct tlsLoadP(thread_RegP dst)
17148 %{
17149 match(Set dst (ThreadLocal));
17150
17151 ins_cost(0);
17152
17153 format %{ " -- \t// $dst=Thread::current(), empty" %}
17154
17155 size(0);
17156
17157 ins_encode( /*empty*/ );
17158
17159 ins_pipe(pipe_class_empty);
17160 %}
17161
17162 //----------PEEPHOLE RULES-----------------------------------------------------
17163 // These must follow all instruction definitions as they use the names
17164 // defined in the instructions definitions.
17165 //
17166 // peepmatch ( root_instr_name [preceding_instruction]* );
17167 //
17168 // peepconstraint %{
17169 // (instruction_number.operand_name relational_op instruction_number.operand_name
17170 // [, ...] );
17171 // // instruction numbers are zero-based using left to right order in peepmatch
17172 //
17173 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17174 // // provide an instruction_number.operand_name for each operand that appears
17175 // // in the replacement instruction's match rule
17176 //
17177 // ---------VM FLAGS---------------------------------------------------------
17178 //
17179 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17180 //
17181 // Each peephole rule is given an identifying number starting with zero and
17182 // increasing by one in the order seen by the parser. An individual peephole
17183 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17184 // on the command-line.
17185 //
17186 // ---------CURRENT LIMITATIONS----------------------------------------------
17187 //
17188 // Only match adjacent instructions in same basic block
17189 // Only equality constraints
17190 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17191 // Only one replacement instruction
17192 //
17193 // ---------EXAMPLE----------------------------------------------------------
17194 //
17195 // // pertinent parts of existing instructions in architecture description
17196 // instruct movI(iRegINoSp dst, iRegI src)
17197 // %{
17198 // match(Set dst (CopyI src));
17199 // %}
17200 //
17201 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17202 // %{
17203 // match(Set dst (AddI dst src));
17204 // effect(KILL cr);
17205 // %}
17206 //
17207 // // Change (inc mov) to lea
17208 // peephole %{
17209 // // increment preceded by register-register move
17210 // peepmatch ( incI_iReg movI );
17211 // // require that the destination register of the increment
17212 // // match the destination register of the move
17213 // peepconstraint ( 0.dst == 1.dst );
17214 // // construct a replacement instruction that sets
17215 // // the destination to ( move's source register + one )
17216 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17217 // %}
17218 //
17219
17220 // Implementation no longer uses movX instructions since
17221 // machine-independent system no longer uses CopyX nodes.
17222 //
17223 // peephole
17224 // %{
17225 // peepmatch (incI_iReg movI);
17226 // peepconstraint (0.dst == 1.dst);
17227 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17228 // %}
17229
17230 // peephole
17231 // %{
17232 // peepmatch (decI_iReg movI);
17233 // peepconstraint (0.dst == 1.dst);
17234 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17235 // %}
17236
17237 // peephole
17238 // %{
17239 // peepmatch (addI_iReg_imm movI);
17240 // peepconstraint (0.dst == 1.dst);
17241 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17242 // %}
17243
17244 // peephole
17245 // %{
17246 // peepmatch (incL_iReg movL);
17247 // peepconstraint (0.dst == 1.dst);
17248 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17249 // %}
17250
17251 // peephole
17252 // %{
17253 // peepmatch (decL_iReg movL);
17254 // peepconstraint (0.dst == 1.dst);
17255 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17256 // %}
17257
17258 // peephole
17259 // %{
17260 // peepmatch (addL_iReg_imm movL);
17261 // peepconstraint (0.dst == 1.dst);
17262 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17263 // %}
17264
17265 // peephole
17266 // %{
17267 // peepmatch (addP_iReg_imm movP);
17268 // peepconstraint (0.dst == 1.dst);
17269 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17270 // %}
17271
17272 // // Change load of spilled value to only a spill
17273 // instruct storeI(memory mem, iRegI src)
17274 // %{
17275 // match(Set mem (StoreI mem src));
17276 // %}
17277 //
17278 // instruct loadI(iRegINoSp dst, memory mem)
17279 // %{
17280 // match(Set dst (LoadI mem));
17281 // %}
17282 //
17283
17284 //----------SMARTSPILL RULES---------------------------------------------------
17285 // These must follow all instruction definitions as they use the names
17286 // defined in the instructions definitions.
17287
17288 // Local Variables:
17289 // mode: c++
17290 // End: