1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 __ verified_entry(C, 0);
1773
1774 if (C->stub_function() == nullptr) {
1775 __ entry_barrier();
1776 }
1777
1778 if (!Compile::current()->output()->in_scratch_emit_size()) {
1779 __ bind(*_verified_entry);
1780 }
1781
1782 if (VerifyStackAtCalls) {
1783 Unimplemented();
1784 }
1785
1786 C->output()->set_frame_complete(__ offset());
1787
1788 if (C->has_mach_constant_base_node()) {
1789 // NOTE: We set the table base offset here because users might be
1790 // emitted before MachConstantBaseNode.
1791 ConstantTable& constant_table = C->output()->constant_table();
1792 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1793 }
1794 }
1795
1796 int MachPrologNode::reloc() const
1797 {
1798 return 0;
1799 }
1800
1801 //=============================================================================
1802
1803 #ifndef PRODUCT
1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1805 Compile* C = ra_->C;
1806 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1807
1808 st->print("# pop frame %d\n\t",framesize);
1809
1810 if (framesize == 0) {
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1813 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, #%d\n\t", framesize);
1815 } else {
1816 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1817 st->print("add sp, sp, rscratch1\n\t");
1818 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1819 }
1820 if (VM_Version::use_rop_protection()) {
1821 st->print("autiaz\n\t");
1822 st->print("ldr zr, [lr]\n\t");
1823 }
1824
1825 if (do_polling() && C->is_method_compilation()) {
1826 st->print("# test polling word\n\t");
1827 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1828 st->print("cmp sp, rscratch1\n\t");
1829 st->print("bhi #slow_path");
1830 }
1831 }
1832 #endif
1833
1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1835 Compile* C = ra_->C;
1836 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1837
1838 __ remove_frame(framesize, C->needs_stack_repair());
1839
1840 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1841 __ reserved_stack_check();
1842 }
1843
1844 if (do_polling() && C->is_method_compilation()) {
1845 Label dummy_label;
1846 Label* code_stub = &dummy_label;
1847 if (!C->output()->in_scratch_emit_size()) {
1848 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1849 C->output()->add_stub(stub);
1850 code_stub = &stub->entry();
1851 }
1852 __ relocate(relocInfo::poll_return_type);
1853 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1854 }
1855 }
1856
1857 int MachEpilogNode::reloc() const {
1858 // Return number of relocatable values contained in this instruction.
1859 return 1; // 1 for polling page.
1860 }
1861
1862 const Pipeline * MachEpilogNode::pipeline() const {
1863 return MachNode::pipeline_class();
1864 }
1865
1866 //=============================================================================
1867
1868 static enum RC rc_class(OptoReg::Name reg) {
1869
1870 if (reg == OptoReg::Bad) {
1871 return rc_bad;
1872 }
1873
1874 // we have 32 int registers * 2 halves
1875 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1876
1877 if (reg < slots_of_int_registers) {
1878 return rc_int;
1879 }
1880
1881 // we have 32 float register * 8 halves
1882 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1883 if (reg < slots_of_int_registers + slots_of_float_registers) {
1884 return rc_float;
1885 }
1886
1887 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1888 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1889 return rc_predicate;
1890 }
1891
1892 // Between predicate regs & stack is the flags.
1893 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1894
1895 return rc_stack;
1896 }
1897
1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1899 Compile* C = ra_->C;
1900
1901 // Get registers to move.
1902 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1903 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1904 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1905 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1906
1907 enum RC src_hi_rc = rc_class(src_hi);
1908 enum RC src_lo_rc = rc_class(src_lo);
1909 enum RC dst_hi_rc = rc_class(dst_hi);
1910 enum RC dst_lo_rc = rc_class(dst_lo);
1911
1912 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1913
1914 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1915 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1916 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1917 "expected aligned-adjacent pairs");
1918 }
1919
1920 if (src_lo == dst_lo && src_hi == dst_hi) {
1921 return 0; // Self copy, no move.
1922 }
1923
1924 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1925 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1926 int src_offset = ra_->reg2offset(src_lo);
1927 int dst_offset = ra_->reg2offset(dst_lo);
1928
1929 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1930 uint ireg = ideal_reg();
1931 if (ireg == Op_VecA && masm) {
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (masm) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (masm) {
1979 switch (src_lo_rc) {
1980 case rc_int:
1981 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1982 if (is64) {
1983 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 } else {
1986 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1987 as_Register(Matcher::_regEncode[src_lo]));
1988 }
1989 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1990 if (is64) {
1991 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 } else {
1994 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 }
1997 } else { // gpr --> stack spill
1998 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1999 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2000 }
2001 break;
2002 case rc_float:
2003 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2004 if (is64) {
2005 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 } else {
2008 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 }
2011 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2012 if (is64) {
2013 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 }
2019 } else { // fpr --> stack spill
2020 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2021 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 is64 ? __ D : __ S, dst_offset);
2023 }
2024 break;
2025 case rc_stack:
2026 if (dst_lo_rc == rc_int) { // stack --> gpr load
2027 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2028 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 is64 ? __ D : __ S, src_offset);
2031 } else if (dst_lo_rc == rc_predicate) {
2032 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2033 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2034 } else { // stack --> stack copy
2035 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2036 if (ideal_reg() == Op_RegVectMask) {
2037 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2038 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2039 } else {
2040 __ unspill(rscratch1, is64, src_offset);
2041 __ spill(rscratch1, is64, dst_offset);
2042 }
2043 }
2044 break;
2045 case rc_predicate:
2046 if (dst_lo_rc == rc_predicate) {
2047 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2048 } else if (dst_lo_rc == rc_stack) {
2049 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2050 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2051 } else {
2052 assert(false, "bad src and dst rc_class combination.");
2053 ShouldNotReachHere();
2054 }
2055 break;
2056 default:
2057 assert(false, "bad rc_class for spill");
2058 ShouldNotReachHere();
2059 }
2060 }
2061
2062 if (st) {
2063 st->print("spill ");
2064 if (src_lo_rc == rc_stack) {
2065 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2066 } else {
2067 st->print("%s -> ", Matcher::regName[src_lo]);
2068 }
2069 if (dst_lo_rc == rc_stack) {
2070 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2071 } else {
2072 st->print("%s", Matcher::regName[dst_lo]);
2073 }
2074 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2075 int vsize = 0;
2076 switch (ideal_reg()) {
2077 case Op_VecD:
2078 vsize = 64;
2079 break;
2080 case Op_VecX:
2081 vsize = 128;
2082 break;
2083 case Op_VecA:
2084 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2085 break;
2086 default:
2087 assert(false, "bad register type for spill");
2088 ShouldNotReachHere();
2089 }
2090 st->print("\t# vector spill size = %d", vsize);
2091 } else if (ideal_reg() == Op_RegVectMask) {
2092 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2093 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2094 st->print("\t# predicate spill size = %d", vsize);
2095 } else {
2096 st->print("\t# spill size = %d", is64 ? 64 : 32);
2097 }
2098 }
2099
2100 return 0;
2101
2102 }
2103
2104 #ifndef PRODUCT
2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2106 if (!ra_)
2107 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2108 else
2109 implementation(nullptr, ra_, false, st);
2110 }
2111 #endif
2112
2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2114 implementation(masm, ra_, false, nullptr);
2115 }
2116
2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2118 return MachNode::size(ra_);
2119 }
2120
2121 //=============================================================================
2122
2123 #ifndef PRODUCT
2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2125 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2126 int reg = ra_->get_reg_first(this);
2127 st->print("add %s, rsp, #%d]\t# box lock",
2128 Matcher::regName[reg], offset);
2129 }
2130 #endif
2131
2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2133 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2134 int reg = ra_->get_encode(this);
2135
2136 // This add will handle any 24-bit signed offset. 24 bits allows an
2137 // 8 megabyte stack frame.
2138 __ add(as_Register(reg), sp, offset);
2139 }
2140
2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2142 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2143 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2144
2145 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2146 return NativeInstruction::instruction_size;
2147 } else {
2148 return 2 * NativeInstruction::instruction_size;
2149 }
2150 }
2151
2152 ///=============================================================================
2153 #ifndef PRODUCT
2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2155 {
2156 st->print_cr("# MachVEPNode");
2157 if (!_verified) {
2158 st->print_cr("\t load_class");
2159 } else {
2160 st->print_cr("\t unpack_inline_arg");
2161 }
2162 }
2163 #endif
2164
2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2166 {
2167 if (!_verified) {
2168 __ ic_check(1);
2169 } else {
2170 // insert a nop at the start of the prolog so we can patch in a
2171 // branch if we need to invalidate the method later
2172 __ nop();
2173
2174 // TODO 8284443 Avoid creation of temporary frame
2175 if (ra_->C->stub_function() == nullptr) {
2176 __ verified_entry(ra_->C, 0);
2177 __ entry_barrier();
2178 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2179 __ remove_frame(framesize, false);
2180 }
2181 // Unpack inline type args passed as oop and then jump to
2182 // the verified entry point (skipping the unverified entry).
2183 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2184 // Emit code for verified entry and save increment for stack repair on return
2185 __ verified_entry(ra_->C, sp_inc);
2186 if (Compile::current()->output()->in_scratch_emit_size()) {
2187 Label dummy_verified_entry;
2188 __ b(dummy_verified_entry);
2189 } else {
2190 __ b(*_verified_entry);
2191 }
2192 }
2193 }
2194
2195 //=============================================================================
2196 #ifndef PRODUCT
2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2198 {
2199 st->print_cr("# MachUEPNode");
2200 if (UseCompressedClassPointers) {
2201 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2202 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2203 st->print_cr("\tcmpw rscratch1, r10");
2204 } else {
2205 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2206 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2207 st->print_cr("\tcmp rscratch1, r10");
2208 }
2209 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2210 }
2211 #endif
2212
2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2214 {
2215 __ ic_check(InteriorEntryAlignment);
2216 }
2217
2218 // REQUIRED EMIT CODE
2219
2220 //=============================================================================
2221
2222 // Emit exception handler code.
2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2224 {
2225 // mov rscratch1 #exception_blob_entry_point
2226 // br rscratch1
2227 // Note that the code buffer's insts_mark is always relative to insts.
2228 // That's why we must use the macroassembler to generate a handler.
2229 address base = __ start_a_stub(size_exception_handler());
2230 if (base == nullptr) {
2231 ciEnv::current()->record_failure("CodeCache is full");
2232 return 0; // CodeBuffer::expand failed
2233 }
2234 int offset = __ offset();
2235 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2236 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2237 __ end_a_stub();
2238 return offset;
2239 }
2240
2241 // Emit deopt handler code.
2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2243 {
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 address base = __ start_a_stub(size_deopt_handler());
2247 if (base == nullptr) {
2248 ciEnv::current()->record_failure("CodeCache is full");
2249 return 0; // CodeBuffer::expand failed
2250 }
2251 int offset = __ offset();
2252
2253 __ adr(lr, __ pc());
2254 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2255
2256 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2257 __ end_a_stub();
2258 return offset;
2259 }
2260
2261 // REQUIRED MATCHER CODE
2262
2263 //=============================================================================
2264
2265 bool Matcher::match_rule_supported(int opcode) {
2266 if (!has_match_rule(opcode))
2267 return false;
2268
2269 switch (opcode) {
2270 case Op_OnSpinWait:
2271 return VM_Version::supports_on_spin_wait();
2272 case Op_CacheWB:
2273 case Op_CacheWBPreSync:
2274 case Op_CacheWBPostSync:
2275 if (!VM_Version::supports_data_cache_line_flush()) {
2276 return false;
2277 }
2278 break;
2279 case Op_ExpandBits:
2280 case Op_CompressBits:
2281 if (!VM_Version::supports_svebitperm()) {
2282 return false;
2283 }
2284 break;
2285 case Op_FmaF:
2286 case Op_FmaD:
2287 case Op_FmaVF:
2288 case Op_FmaVD:
2289 if (!UseFMA) {
2290 return false;
2291 }
2292 break;
2293 }
2294
2295 return true; // Per default match rules are supported.
2296 }
2297
2298 const RegMask* Matcher::predicate_reg_mask(void) {
2299 return &_PR_REG_mask;
2300 }
2301
2302 bool Matcher::supports_vector_calling_convention(void) {
2303 return EnableVectorSupport && UseVectorStubs;
2304 }
2305
2306 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2307 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2308 int lo = V0_num;
2309 int hi = V0_H_num;
2310 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2311 hi = V0_K_num;
2312 }
2313 return OptoRegPair(hi, lo);
2314 }
2315
2316 // Is this branch offset short enough that a short branch can be used?
2317 //
2318 // NOTE: If the platform does not provide any short branch variants, then
2319 // this method should return false for offset 0.
2320 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2321 // The passed offset is relative to address of the branch.
2322
2323 return (-32768 <= offset && offset < 32768);
2324 }
2325
2326 // Vector width in bytes.
2327 int Matcher::vector_width_in_bytes(BasicType bt) {
2328 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2329 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2330 // Minimum 2 values in vector
2331 if (size < 2*type2aelembytes(bt)) size = 0;
2332 // But never < 4
2333 if (size < 4) size = 0;
2334 return size;
2335 }
2336
2337 // Limits on vector size (number of elements) loaded into vector.
2338 int Matcher::max_vector_size(const BasicType bt) {
2339 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2340 }
2341
2342 int Matcher::min_vector_size(const BasicType bt) {
2343 int max_size = max_vector_size(bt);
2344 // Limit the min vector size to 8 bytes.
2345 int size = 8 / type2aelembytes(bt);
2346 if (bt == T_BYTE) {
2347 // To support vector api shuffle/rearrange.
2348 size = 4;
2349 } else if (bt == T_BOOLEAN) {
2350 // To support vector api load/store mask.
2351 size = 2;
2352 }
2353 if (size < 2) size = 2;
2354 return MIN2(size, max_size);
2355 }
2356
2357 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2358 return Matcher::max_vector_size(bt);
2359 }
2360
2361 // Actual max scalable vector register length.
2362 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2363 return Matcher::max_vector_size(bt);
2364 }
2365
2366 // Vector ideal reg.
2367 uint Matcher::vector_ideal_reg(int len) {
2368 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2369 return Op_VecA;
2370 }
2371 switch(len) {
2372 // For 16-bit/32-bit mask vector, reuse VecD.
2373 case 2:
2374 case 4:
2375 case 8: return Op_VecD;
2376 case 16: return Op_VecX;
2377 }
2378 ShouldNotReachHere();
2379 return 0;
2380 }
2381
2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2383 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2384 switch (ideal_reg) {
2385 case Op_VecA: return new vecAOper();
2386 case Op_VecD: return new vecDOper();
2387 case Op_VecX: return new vecXOper();
2388 }
2389 ShouldNotReachHere();
2390 return nullptr;
2391 }
2392
2393 bool Matcher::is_reg2reg_move(MachNode* m) {
2394 return false;
2395 }
2396
2397 bool Matcher::is_generic_vector(MachOper* opnd) {
2398 return opnd->opcode() == VREG;
2399 }
2400
2401 // Return whether or not this register is ever used as an argument.
2402 // This function is used on startup to build the trampoline stubs in
2403 // generateOptoStub. Registers not mentioned will be killed by the VM
2404 // call in the trampoline, and arguments in those registers not be
2405 // available to the callee.
2406 bool Matcher::can_be_java_arg(int reg)
2407 {
2408 return
2409 reg == R0_num || reg == R0_H_num ||
2410 reg == R1_num || reg == R1_H_num ||
2411 reg == R2_num || reg == R2_H_num ||
2412 reg == R3_num || reg == R3_H_num ||
2413 reg == R4_num || reg == R4_H_num ||
2414 reg == R5_num || reg == R5_H_num ||
2415 reg == R6_num || reg == R6_H_num ||
2416 reg == R7_num || reg == R7_H_num ||
2417 reg == V0_num || reg == V0_H_num ||
2418 reg == V1_num || reg == V1_H_num ||
2419 reg == V2_num || reg == V2_H_num ||
2420 reg == V3_num || reg == V3_H_num ||
2421 reg == V4_num || reg == V4_H_num ||
2422 reg == V5_num || reg == V5_H_num ||
2423 reg == V6_num || reg == V6_H_num ||
2424 reg == V7_num || reg == V7_H_num;
2425 }
2426
2427 bool Matcher::is_spillable_arg(int reg)
2428 {
2429 return can_be_java_arg(reg);
2430 }
2431
2432 uint Matcher::int_pressure_limit()
2433 {
2434 // JDK-8183543: When taking the number of available registers as int
2435 // register pressure threshold, the jtreg test:
2436 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2437 // failed due to C2 compilation failure with
2438 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2439 //
2440 // A derived pointer is live at CallNode and then is flagged by RA
2441 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2442 // derived pointers and lastly fail to spill after reaching maximum
2443 // number of iterations. Lowering the default pressure threshold to
2444 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2445 // a high register pressure area of the code so that split_DEF can
2446 // generate DefinitionSpillCopy for the derived pointer.
2447 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2448 if (!PreserveFramePointer) {
2449 // When PreserveFramePointer is off, frame pointer is allocatable,
2450 // but different from other SOC registers, it is excluded from
2451 // fatproj's mask because its save type is No-Save. Decrease 1 to
2452 // ensure high pressure at fatproj when PreserveFramePointer is off.
2453 // See check_pressure_at_fatproj().
2454 default_int_pressure_threshold--;
2455 }
2456 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2457 }
2458
2459 uint Matcher::float_pressure_limit()
2460 {
2461 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2462 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2463 }
2464
2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2466 return false;
2467 }
2468
2469 RegMask Matcher::divI_proj_mask() {
2470 ShouldNotReachHere();
2471 return RegMask();
2472 }
2473
2474 // Register for MODI projection of divmodI.
2475 RegMask Matcher::modI_proj_mask() {
2476 ShouldNotReachHere();
2477 return RegMask();
2478 }
2479
2480 // Register for DIVL projection of divmodL.
2481 RegMask Matcher::divL_proj_mask() {
2482 ShouldNotReachHere();
2483 return RegMask();
2484 }
2485
2486 // Register for MODL projection of divmodL.
2487 RegMask Matcher::modL_proj_mask() {
2488 ShouldNotReachHere();
2489 return RegMask();
2490 }
2491
2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2493 return FP_REG_mask();
2494 }
2495
2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2497 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2498 Node* u = addp->fast_out(i);
2499 if (u->is_LoadStore()) {
2500 // On AArch64, LoadStoreNodes (i.e. compare and swap
2501 // instructions) only take register indirect as an operand, so
2502 // any attempt to use an AddPNode as an input to a LoadStoreNode
2503 // must fail.
2504 return false;
2505 }
2506 if (u->is_Mem()) {
2507 int opsize = u->as_Mem()->memory_size();
2508 assert(opsize > 0, "unexpected memory operand size");
2509 if (u->as_Mem()->memory_size() != (1<<shift)) {
2510 return false;
2511 }
2512 }
2513 }
2514 return true;
2515 }
2516
2517 // Convert BootTest condition to Assembler condition.
2518 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2519 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2520 Assembler::Condition result;
2521 switch(cond) {
2522 case BoolTest::eq:
2523 result = Assembler::EQ; break;
2524 case BoolTest::ne:
2525 result = Assembler::NE; break;
2526 case BoolTest::le:
2527 result = Assembler::LE; break;
2528 case BoolTest::ge:
2529 result = Assembler::GE; break;
2530 case BoolTest::lt:
2531 result = Assembler::LT; break;
2532 case BoolTest::gt:
2533 result = Assembler::GT; break;
2534 case BoolTest::ule:
2535 result = Assembler::LS; break;
2536 case BoolTest::uge:
2537 result = Assembler::HS; break;
2538 case BoolTest::ult:
2539 result = Assembler::LO; break;
2540 case BoolTest::ugt:
2541 result = Assembler::HI; break;
2542 case BoolTest::overflow:
2543 result = Assembler::VS; break;
2544 case BoolTest::no_overflow:
2545 result = Assembler::VC; break;
2546 default:
2547 ShouldNotReachHere();
2548 return Assembler::Condition(-1);
2549 }
2550
2551 // Check conversion
2552 if (cond & BoolTest::unsigned_compare) {
2553 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2554 } else {
2555 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2556 }
2557
2558 return result;
2559 }
2560
2561 // Binary src (Replicate con)
2562 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2563 if (n == nullptr || m == nullptr) {
2564 return false;
2565 }
2566
2567 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2568 return false;
2569 }
2570
2571 Node* imm_node = m->in(1);
2572 if (!imm_node->is_Con()) {
2573 return false;
2574 }
2575
2576 const Type* t = imm_node->bottom_type();
2577 if (!(t->isa_int() || t->isa_long())) {
2578 return false;
2579 }
2580
2581 switch (n->Opcode()) {
2582 case Op_AndV:
2583 case Op_OrV:
2584 case Op_XorV: {
2585 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2586 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2587 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2588 }
2589 case Op_AddVB:
2590 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2591 case Op_AddVS:
2592 case Op_AddVI:
2593 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2594 case Op_AddVL:
2595 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2596 default:
2597 return false;
2598 }
2599 }
2600
2601 // (XorV src (Replicate m1))
2602 // (XorVMask src (MaskAll m1))
2603 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2604 if (n != nullptr && m != nullptr) {
2605 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2606 VectorNode::is_all_ones_vector(m);
2607 }
2608 return false;
2609 }
2610
2611 // Should the matcher clone input 'm' of node 'n'?
2612 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2613 if (is_vshift_con_pattern(n, m) ||
2614 is_vector_bitwise_not_pattern(n, m) ||
2615 is_valid_sve_arith_imm_pattern(n, m) ||
2616 is_encode_and_store_pattern(n, m)) {
2617 mstack.push(m, Visit);
2618 return true;
2619 }
2620 return false;
2621 }
2622
2623 // Should the Matcher clone shifts on addressing modes, expecting them
2624 // to be subsumed into complex addressing expressions or compute them
2625 // into registers?
2626 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2627
2628 // Loads and stores with indirect memory input (e.g., volatile loads and
2629 // stores) do not subsume the input into complex addressing expressions. If
2630 // the addressing expression is input to at least one such load or store, do
2631 // not clone the addressing expression. Query needs_acquiring_load and
2632 // needs_releasing_store as a proxy for indirect memory input, as it is not
2633 // possible to directly query for indirect memory input at this stage.
2634 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2635 Node* n = m->fast_out(i);
2636 if (n->is_Load() && needs_acquiring_load(n)) {
2637 return false;
2638 }
2639 if (n->is_Store() && needs_releasing_store(n)) {
2640 return false;
2641 }
2642 }
2643
2644 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2645 return true;
2646 }
2647
2648 Node *off = m->in(AddPNode::Offset);
2649 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2650 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2651 // Are there other uses besides address expressions?
2652 !is_visited(off)) {
2653 address_visited.set(off->_idx); // Flag as address_visited
2654 mstack.push(off->in(2), Visit);
2655 Node *conv = off->in(1);
2656 if (conv->Opcode() == Op_ConvI2L &&
2657 // Are there other uses besides address expressions?
2658 !is_visited(conv)) {
2659 address_visited.set(conv->_idx); // Flag as address_visited
2660 mstack.push(conv->in(1), Pre_Visit);
2661 } else {
2662 mstack.push(conv, Pre_Visit);
2663 }
2664 address_visited.test_set(m->_idx); // Flag as address_visited
2665 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2666 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2667 return true;
2668 } else if (off->Opcode() == Op_ConvI2L &&
2669 // Are there other uses besides address expressions?
2670 !is_visited(off)) {
2671 address_visited.test_set(m->_idx); // Flag as address_visited
2672 address_visited.set(off->_idx); // Flag as address_visited
2673 mstack.push(off->in(1), Pre_Visit);
2674 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2675 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2676 return true;
2677 }
2678 return false;
2679 }
2680
2681 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2682 { \
2683 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2684 guarantee(DISP == 0, "mode not permitted for volatile"); \
2685 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2686 __ INSN(REG, as_Register(BASE)); \
2687 }
2688
2689
2690 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2691 {
2692 Address::extend scale;
2693
2694 // Hooboy, this is fugly. We need a way to communicate to the
2695 // encoder that the index needs to be sign extended, so we have to
2696 // enumerate all the cases.
2697 switch (opcode) {
2698 case INDINDEXSCALEDI2L:
2699 case INDINDEXSCALEDI2LN:
2700 case INDINDEXI2L:
2701 case INDINDEXI2LN:
2702 scale = Address::sxtw(size);
2703 break;
2704 default:
2705 scale = Address::lsl(size);
2706 }
2707
2708 if (index == -1) {
2709 return Address(base, disp);
2710 } else {
2711 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2712 return Address(base, as_Register(index), scale);
2713 }
2714 }
2715
2716
2717 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2718 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2719 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2720 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2721 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2722
2723 // Used for all non-volatile memory accesses. The use of
2724 // $mem->opcode() to discover whether this pattern uses sign-extended
2725 // offsets is something of a kludge.
2726 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2727 Register reg, int opcode,
2728 Register base, int index, int scale, int disp,
2729 int size_in_memory)
2730 {
2731 Address addr = mem2address(opcode, base, index, scale, disp);
2732 if (addr.getMode() == Address::base_plus_offset) {
2733 /* Fix up any out-of-range offsets. */
2734 assert_different_registers(rscratch1, base);
2735 assert_different_registers(rscratch1, reg);
2736 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2737 }
2738 (masm->*insn)(reg, addr);
2739 }
2740
2741 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2742 FloatRegister reg, int opcode,
2743 Register base, int index, int size, int disp,
2744 int size_in_memory)
2745 {
2746 Address::extend scale;
2747
2748 switch (opcode) {
2749 case INDINDEXSCALEDI2L:
2750 case INDINDEXSCALEDI2LN:
2751 scale = Address::sxtw(size);
2752 break;
2753 default:
2754 scale = Address::lsl(size);
2755 }
2756
2757 if (index == -1) {
2758 // Fix up any out-of-range offsets.
2759 assert_different_registers(rscratch1, base);
2760 Address addr = Address(base, disp);
2761 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2762 (masm->*insn)(reg, addr);
2763 } else {
2764 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2765 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2766 }
2767 }
2768
2769 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2770 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2771 int opcode, Register base, int index, int size, int disp)
2772 {
2773 if (index == -1) {
2774 (masm->*insn)(reg, T, Address(base, disp));
2775 } else {
2776 assert(disp == 0, "unsupported address mode");
2777 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2778 }
2779 }
2780
2781 %}
2782
2783
2784
2785 //----------ENCODING BLOCK-----------------------------------------------------
2786 // This block specifies the encoding classes used by the compiler to
2787 // output byte streams. Encoding classes are parameterized macros
2788 // used by Machine Instruction Nodes in order to generate the bit
2789 // encoding of the instruction. Operands specify their base encoding
2790 // interface with the interface keyword. There are currently
2791 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2792 // COND_INTER. REG_INTER causes an operand to generate a function
2793 // which returns its register number when queried. CONST_INTER causes
2794 // an operand to generate a function which returns the value of the
2795 // constant when queried. MEMORY_INTER causes an operand to generate
2796 // four functions which return the Base Register, the Index Register,
2797 // the Scale Value, and the Offset Value of the operand when queried.
2798 // COND_INTER causes an operand to generate six functions which return
2799 // the encoding code (ie - encoding bits for the instruction)
2800 // associated with each basic boolean condition for a conditional
2801 // instruction.
2802 //
2803 // Instructions specify two basic values for encoding. Again, a
2804 // function is available to check if the constant displacement is an
2805 // oop. They use the ins_encode keyword to specify their encoding
2806 // classes (which must be a sequence of enc_class names, and their
2807 // parameters, specified in the encoding block), and they use the
2808 // opcode keyword to specify, in order, their primary, secondary, and
2809 // tertiary opcode. Only the opcode sections which a particular
2810 // instruction needs for encoding need to be specified.
2811 encode %{
2812 // Build emit functions for each basic byte or larger field in the
2813 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2814 // from C++ code in the enc_class source block. Emit functions will
2815 // live in the main source block for now. In future, we can
2816 // generalize this by adding a syntax that specifies the sizes of
2817 // fields in an order, so that the adlc can build the emit functions
2818 // automagically
2819
2820 // catch all for unimplemented encodings
2821 enc_class enc_unimplemented %{
2822 __ unimplemented("C2 catch all");
2823 %}
2824
2825 // BEGIN Non-volatile memory access
2826
2827 // This encoding class is generated automatically from ad_encode.m4.
2828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2829 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2830 Register dst_reg = as_Register($dst$$reg);
2831 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2832 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2833 %}
2834
2835 // This encoding class is generated automatically from ad_encode.m4.
2836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2837 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2838 Register dst_reg = as_Register($dst$$reg);
2839 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2841 %}
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2926 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2934 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2942 Register src_reg = as_Register($src$$reg);
2943 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strb0(memory1 mem) %{
2950 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2957 Register src_reg = as_Register($src$$reg);
2958 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh0(memory2 mem) %{
2965 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2966 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2967 %}
2968
2969 // This encoding class is generated automatically from ad_encode.m4.
2970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2971 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2972 Register src_reg = as_Register($src$$reg);
2973 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strw0(memory4 mem) %{
2980 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2981 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2982 %}
2983
2984 // This encoding class is generated automatically from ad_encode.m4.
2985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2986 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2987 Register src_reg = as_Register($src$$reg);
2988 // we sometimes get asked to store the stack pointer into the
2989 // current thread -- we cannot do that directly on AArch64
2990 if (src_reg == r31_sp) {
2991 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2992 __ mov(rscratch2, sp);
2993 src_reg = rscratch2;
2994 }
2995 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_str0(memory8 mem) %{
3002 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3004 %}
3005
3006 // This encoding class is generated automatically from ad_encode.m4.
3007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3008 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3009 FloatRegister src_reg = as_FloatRegister($src$$reg);
3010 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3017 FloatRegister src_reg = as_FloatRegister($src$$reg);
3018 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3025 __ membar(Assembler::StoreStore);
3026 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3028 %}
3029
3030 // END Non-volatile memory access
3031
3032 // Vector loads and stores
3033 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3034 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3035 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3036 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3037 %}
3038
3039 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3040 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3041 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3042 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3043 %}
3044
3045 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3046 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3047 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3048 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3049 %}
3050
3051 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3052 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3053 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3054 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3055 %}
3056
3057 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3058 FloatRegister src_reg = as_FloatRegister($src$$reg);
3059 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3060 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3061 %}
3062
3063 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3064 FloatRegister src_reg = as_FloatRegister($src$$reg);
3065 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3066 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3067 %}
3068
3069 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3070 FloatRegister src_reg = as_FloatRegister($src$$reg);
3071 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3072 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3073 %}
3074
3075 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3076 FloatRegister src_reg = as_FloatRegister($src$$reg);
3077 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3078 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3079 %}
3080
3081 // volatile loads and stores
3082
3083 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3084 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3085 rscratch1, stlrb);
3086 %}
3087
3088 enc_class aarch64_enc_stlrb0(memory mem) %{
3089 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3090 rscratch1, stlrb);
3091 %}
3092
3093 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3094 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3095 rscratch1, stlrh);
3096 %}
3097
3098 enc_class aarch64_enc_stlrh0(memory mem) %{
3099 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3100 rscratch1, stlrh);
3101 %}
3102
3103 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3104 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3105 rscratch1, stlrw);
3106 %}
3107
3108 enc_class aarch64_enc_stlrw0(memory mem) %{
3109 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, stlrw);
3111 %}
3112
3113 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3114 Register dst_reg = as_Register($dst$$reg);
3115 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, ldarb);
3117 __ sxtbw(dst_reg, dst_reg);
3118 %}
3119
3120 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3121 Register dst_reg = as_Register($dst$$reg);
3122 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, ldarb);
3124 __ sxtb(dst_reg, dst_reg);
3125 %}
3126
3127 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3128 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 %}
3131
3132 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3133 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3134 rscratch1, ldarb);
3135 %}
3136
3137 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3138 Register dst_reg = as_Register($dst$$reg);
3139 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, ldarh);
3141 __ sxthw(dst_reg, dst_reg);
3142 %}
3143
3144 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3145 Register dst_reg = as_Register($dst$$reg);
3146 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarh);
3148 __ sxth(dst_reg, dst_reg);
3149 %}
3150
3151 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3152 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 %}
3155
3156 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3157 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, ldarh);
3159 %}
3160
3161 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3162 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, ldarw);
3164 %}
3165
3166 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3167 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, ldarw);
3169 %}
3170
3171 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldar);
3174 %}
3175
3176 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3177 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarw);
3179 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3180 %}
3181
3182 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3183 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldar);
3185 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3186 %}
3187
3188 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3189 Register src_reg = as_Register($src$$reg);
3190 // we sometimes get asked to store the stack pointer into the
3191 // current thread -- we cannot do that directly on AArch64
3192 if (src_reg == r31_sp) {
3193 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3194 __ mov(rscratch2, sp);
3195 src_reg = rscratch2;
3196 }
3197 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, stlr);
3199 %}
3200
3201 enc_class aarch64_enc_stlr0(memory mem) %{
3202 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, stlr);
3204 %}
3205
3206 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3207 {
3208 FloatRegister src_reg = as_FloatRegister($src$$reg);
3209 __ fmovs(rscratch2, src_reg);
3210 }
3211 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlrw);
3213 %}
3214
3215 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3216 {
3217 FloatRegister src_reg = as_FloatRegister($src$$reg);
3218 __ fmovd(rscratch2, src_reg);
3219 }
3220 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, stlr);
3222 %}
3223
3224 // synchronized read/update encodings
3225
3226 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3227 Register dst_reg = as_Register($dst$$reg);
3228 Register base = as_Register($mem$$base);
3229 int index = $mem$$index;
3230 int scale = $mem$$scale;
3231 int disp = $mem$$disp;
3232 if (index == -1) {
3233 if (disp != 0) {
3234 __ lea(rscratch1, Address(base, disp));
3235 __ ldaxr(dst_reg, rscratch1);
3236 } else {
3237 // TODO
3238 // should we ever get anything other than this case?
3239 __ ldaxr(dst_reg, base);
3240 }
3241 } else {
3242 Register index_reg = as_Register(index);
3243 if (disp == 0) {
3244 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3245 __ ldaxr(dst_reg, rscratch1);
3246 } else {
3247 __ lea(rscratch1, Address(base, disp));
3248 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3249 __ ldaxr(dst_reg, rscratch1);
3250 }
3251 }
3252 %}
3253
3254 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3255 Register src_reg = as_Register($src$$reg);
3256 Register base = as_Register($mem$$base);
3257 int index = $mem$$index;
3258 int scale = $mem$$scale;
3259 int disp = $mem$$disp;
3260 if (index == -1) {
3261 if (disp != 0) {
3262 __ lea(rscratch2, Address(base, disp));
3263 __ stlxr(rscratch1, src_reg, rscratch2);
3264 } else {
3265 // TODO
3266 // should we ever get anything other than this case?
3267 __ stlxr(rscratch1, src_reg, base);
3268 }
3269 } else {
3270 Register index_reg = as_Register(index);
3271 if (disp == 0) {
3272 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3273 __ stlxr(rscratch1, src_reg, rscratch2);
3274 } else {
3275 __ lea(rscratch2, Address(base, disp));
3276 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3277 __ stlxr(rscratch1, src_reg, rscratch2);
3278 }
3279 }
3280 __ cmpw(rscratch1, zr);
3281 %}
3282
3283 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3284 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3285 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3286 Assembler::xword, /*acquire*/ false, /*release*/ true,
3287 /*weak*/ false, noreg);
3288 %}
3289
3290 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3291 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3292 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3293 Assembler::word, /*acquire*/ false, /*release*/ true,
3294 /*weak*/ false, noreg);
3295 %}
3296
3297 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3298 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3299 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3300 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3301 /*weak*/ false, noreg);
3302 %}
3303
3304 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3305 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3306 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3307 Assembler::byte, /*acquire*/ false, /*release*/ true,
3308 /*weak*/ false, noreg);
3309 %}
3310
3311
3312 // The only difference between aarch64_enc_cmpxchg and
3313 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3314 // CompareAndSwap sequence to serve as a barrier on acquiring a
3315 // lock.
3316 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::xword, /*acquire*/ true, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::word, /*acquire*/ true, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3338 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3339 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3340 Assembler::byte, /*acquire*/ true, /*release*/ true,
3341 /*weak*/ false, noreg);
3342 %}
3343
3344 // auxiliary used for CompareAndSwapX to set result register
3345 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3346 Register res_reg = as_Register($res$$reg);
3347 __ cset(res_reg, Assembler::EQ);
3348 %}
3349
3350 // prefetch encodings
3351
3352 enc_class aarch64_enc_prefetchw(memory mem) %{
3353 Register base = as_Register($mem$$base);
3354 int index = $mem$$index;
3355 int scale = $mem$$scale;
3356 int disp = $mem$$disp;
3357 if (index == -1) {
3358 // Fix up any out-of-range offsets.
3359 assert_different_registers(rscratch1, base);
3360 Address addr = Address(base, disp);
3361 addr = __ legitimize_address(addr, 8, rscratch1);
3362 __ prfm(addr, PSTL1KEEP);
3363 } else {
3364 Register index_reg = as_Register(index);
3365 if (disp == 0) {
3366 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3367 } else {
3368 __ lea(rscratch1, Address(base, disp));
3369 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3370 }
3371 }
3372 %}
3373
3374 // mov encodings
3375
3376 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3377 uint32_t con = (uint32_t)$src$$constant;
3378 Register dst_reg = as_Register($dst$$reg);
3379 if (con == 0) {
3380 __ movw(dst_reg, zr);
3381 } else {
3382 __ movw(dst_reg, con);
3383 }
3384 %}
3385
3386 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3387 Register dst_reg = as_Register($dst$$reg);
3388 uint64_t con = (uint64_t)$src$$constant;
3389 if (con == 0) {
3390 __ mov(dst_reg, zr);
3391 } else {
3392 __ mov(dst_reg, con);
3393 }
3394 %}
3395
3396 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3397 Register dst_reg = as_Register($dst$$reg);
3398 address con = (address)$src$$constant;
3399 if (con == nullptr || con == (address)1) {
3400 ShouldNotReachHere();
3401 } else {
3402 relocInfo::relocType rtype = $src->constant_reloc();
3403 if (rtype == relocInfo::oop_type) {
3404 __ movoop(dst_reg, (jobject)con);
3405 } else if (rtype == relocInfo::metadata_type) {
3406 __ mov_metadata(dst_reg, (Metadata*)con);
3407 } else {
3408 assert(rtype == relocInfo::none, "unexpected reloc type");
3409 if (! __ is_valid_AArch64_address(con) ||
3410 con < (address)(uintptr_t)os::vm_page_size()) {
3411 __ mov(dst_reg, con);
3412 } else {
3413 uint64_t offset;
3414 __ adrp(dst_reg, con, offset);
3415 __ add(dst_reg, dst_reg, offset);
3416 }
3417 }
3418 }
3419 %}
3420
3421 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3422 Register dst_reg = as_Register($dst$$reg);
3423 __ mov(dst_reg, zr);
3424 %}
3425
3426 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, (uint64_t)1);
3429 %}
3430
3431 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3432 __ load_byte_map_base($dst$$Register);
3433 %}
3434
3435 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 address con = (address)$src$$constant;
3438 if (con == nullptr) {
3439 ShouldNotReachHere();
3440 } else {
3441 relocInfo::relocType rtype = $src->constant_reloc();
3442 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3443 __ set_narrow_oop(dst_reg, (jobject)con);
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 address con = (address)$src$$constant;
3455 if (con == nullptr) {
3456 ShouldNotReachHere();
3457 } else {
3458 relocInfo::relocType rtype = $src->constant_reloc();
3459 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3460 __ set_narrow_klass(dst_reg, (Klass *)con);
3461 }
3462 %}
3463
3464 // arithmetic encodings
3465
3466 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ subw(dst_reg, src_reg, -con);
3474 } else {
3475 __ addw(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src_reg = as_Register($src1$$reg);
3482 int32_t con = (int32_t)$src2$$constant;
3483 // add has primary == 0, subtract has primary == 1
3484 if ($primary) { con = -con; }
3485 if (con < 0) {
3486 __ sub(dst_reg, src_reg, -con);
3487 } else {
3488 __ add(dst_reg, src_reg, con);
3489 }
3490 %}
3491
3492 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3497 %}
3498
3499 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src1_reg = as_Register($src1$$reg);
3502 Register src2_reg = as_Register($src2$$reg);
3503 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3504 %}
3505
3506 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 Register src1_reg = as_Register($src1$$reg);
3509 Register src2_reg = as_Register($src2$$reg);
3510 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3511 %}
3512
3513 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3514 Register dst_reg = as_Register($dst$$reg);
3515 Register src1_reg = as_Register($src1$$reg);
3516 Register src2_reg = as_Register($src2$$reg);
3517 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3518 %}
3519
3520 // compare instruction encodings
3521
3522 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3523 Register reg1 = as_Register($src1$$reg);
3524 Register reg2 = as_Register($src2$$reg);
3525 __ cmpw(reg1, reg2);
3526 %}
3527
3528 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3529 Register reg = as_Register($src1$$reg);
3530 int32_t val = $src2$$constant;
3531 if (val >= 0) {
3532 __ subsw(zr, reg, val);
3533 } else {
3534 __ addsw(zr, reg, -val);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3539 Register reg1 = as_Register($src1$$reg);
3540 uint32_t val = (uint32_t)$src2$$constant;
3541 __ movw(rscratch1, val);
3542 __ cmpw(reg1, rscratch1);
3543 %}
3544
3545 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 Register reg2 = as_Register($src2$$reg);
3548 __ cmp(reg1, reg2);
3549 %}
3550
3551 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3552 Register reg = as_Register($src1$$reg);
3553 int64_t val = $src2$$constant;
3554 if (val >= 0) {
3555 __ subs(zr, reg, val);
3556 } else if (val != -val) {
3557 __ adds(zr, reg, -val);
3558 } else {
3559 // aargh, Long.MIN_VALUE is a special case
3560 __ orr(rscratch1, zr, (uint64_t)val);
3561 __ subs(zr, reg, rscratch1);
3562 }
3563 %}
3564
3565 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 uint64_t val = (uint64_t)$src2$$constant;
3568 __ mov(rscratch1, val);
3569 __ cmp(reg1, rscratch1);
3570 %}
3571
3572 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3573 Register reg1 = as_Register($src1$$reg);
3574 Register reg2 = as_Register($src2$$reg);
3575 __ cmp(reg1, reg2);
3576 %}
3577
3578 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3579 Register reg1 = as_Register($src1$$reg);
3580 Register reg2 = as_Register($src2$$reg);
3581 __ cmpw(reg1, reg2);
3582 %}
3583
3584 enc_class aarch64_enc_testp(iRegP src) %{
3585 Register reg = as_Register($src$$reg);
3586 __ cmp(reg, zr);
3587 %}
3588
3589 enc_class aarch64_enc_testn(iRegN src) %{
3590 Register reg = as_Register($src$$reg);
3591 __ cmpw(reg, zr);
3592 %}
3593
3594 enc_class aarch64_enc_b(label lbl) %{
3595 Label *L = $lbl$$label;
3596 __ b(*L);
3597 %}
3598
3599 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3600 Label *L = $lbl$$label;
3601 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3602 %}
3603
3604 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3605 Label *L = $lbl$$label;
3606 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3607 %}
3608
3609 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3610 %{
3611 Register sub_reg = as_Register($sub$$reg);
3612 Register super_reg = as_Register($super$$reg);
3613 Register temp_reg = as_Register($temp$$reg);
3614 Register result_reg = as_Register($result$$reg);
3615
3616 Label miss;
3617 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3618 nullptr, &miss,
3619 /*set_cond_codes:*/ true);
3620 if ($primary) {
3621 __ mov(result_reg, zr);
3622 }
3623 __ bind(miss);
3624 %}
3625
3626 enc_class aarch64_enc_java_static_call(method meth) %{
3627 address addr = (address)$meth$$method;
3628 address call;
3629 if (!_method) {
3630 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3631 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3632 if (call == nullptr) {
3633 ciEnv::current()->record_failure("CodeCache is full");
3634 return;
3635 }
3636 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3637 // The NOP here is purely to ensure that eliding a call to
3638 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3639 __ nop();
3640 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3641 } else {
3642 int method_index = resolved_method_index(masm);
3643 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3644 : static_call_Relocation::spec(method_index);
3645 call = __ trampoline_call(Address(addr, rspec));
3646 if (call == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3651 // Calls of the same statically bound method can share
3652 // a stub to the interpreter.
3653 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3654 } else {
3655 // Emit stub for static call
3656 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3657 if (stub == nullptr) {
3658 ciEnv::current()->record_failure("CodeCache is full");
3659 return;
3660 }
3661 }
3662 }
3663
3664 __ post_call_nop();
3665
3666 // Only non uncommon_trap calls need to reinitialize ptrue.
3667 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3673 int method_index = resolved_method_index(masm);
3674 address call = __ ic_call((address)$meth$$method, method_index);
3675 if (call == nullptr) {
3676 ciEnv::current()->record_failure("CodeCache is full");
3677 return;
3678 }
3679 __ post_call_nop();
3680 if (Compile::current()->max_vector_size() > 0) {
3681 __ reinitialize_ptrue();
3682 }
3683 %}
3684
3685 enc_class aarch64_enc_call_epilog() %{
3686 if (VerifyStackAtCalls) {
3687 // Check that stack depth is unchanged: find majik cookie on stack
3688 __ call_Unimplemented();
3689 }
3690 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3691 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3692 // Search for the corresponding projection, get the register and emit code that initialized it.
3693 uint con = (tf()->range_cc()->cnt() - 1);
3694 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3695 ProjNode* proj = fast_out(i)->as_Proj();
3696 if (proj->_con == con) {
3697 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3698 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3699 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3700 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3701 __ cmp(r0, zr);
3702 __ cset(toReg, Assembler::NE);
3703 if (reg->is_stack()) {
3704 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3705 __ str(toReg, Address(sp, st_off));
3706 }
3707 break;
3708 }
3709 }
3710 if (return_value_is_used()) {
3711 // An inline type is returned as fields in multiple registers.
3712 // R0 either contains an oop if the inline type is buffered or a pointer
3713 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3714 // if the lowest bit is set to allow C2 to use the oop after null checking.
3715 // r0 &= (r0 & 1) - 1
3716 __ andr(rscratch1, r0, 0x1);
3717 __ sub(rscratch1, rscratch1, 0x1);
3718 __ andr(r0, r0, rscratch1);
3719 }
3720 }
3721 %}
3722
3723 enc_class aarch64_enc_java_to_runtime(method meth) %{
3724 // some calls to generated routines (arraycopy code) are scheduled
3725 // by C2 as runtime calls. if so we can call them using a br (they
3726 // will be in a reachable segment) otherwise we have to use a blr
3727 // which loads the absolute address into a register.
3728 address entry = (address)$meth$$method;
3729 CodeBlob *cb = CodeCache::find_blob(entry);
3730 if (cb) {
3731 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3732 if (call == nullptr) {
3733 ciEnv::current()->record_failure("CodeCache is full");
3734 return;
3735 }
3736 __ post_call_nop();
3737 } else {
3738 Label retaddr;
3739 // Make the anchor frame walkable
3740 __ adr(rscratch2, retaddr);
3741 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3742 __ lea(rscratch1, RuntimeAddress(entry));
3743 __ blr(rscratch1);
3744 __ bind(retaddr);
3745 __ post_call_nop();
3746 }
3747 if (Compile::current()->max_vector_size() > 0) {
3748 __ reinitialize_ptrue();
3749 }
3750 %}
3751
3752 enc_class aarch64_enc_rethrow() %{
3753 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3754 %}
3755
3756 enc_class aarch64_enc_ret() %{
3757 #ifdef ASSERT
3758 if (Compile::current()->max_vector_size() > 0) {
3759 __ verify_ptrue();
3760 }
3761 #endif
3762 __ ret(lr);
3763 %}
3764
3765 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3766 Register target_reg = as_Register($jump_target$$reg);
3767 __ br(target_reg);
3768 %}
3769
3770 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3771 Register target_reg = as_Register($jump_target$$reg);
3772 // exception oop should be in r0
3773 // ret addr has been popped into lr
3774 // callee expects it in r3
3775 __ mov(r3, lr);
3776 __ br(target_reg);
3777 %}
3778
3779 %}
3780
3781 //----------FRAME--------------------------------------------------------------
3782 // Definition of frame structure and management information.
3783 //
3784 // S T A C K L A Y O U T Allocators stack-slot number
3785 // | (to get allocators register number
3786 // G Owned by | | v add OptoReg::stack0())
3787 // r CALLER | |
3788 // o | +--------+ pad to even-align allocators stack-slot
3789 // w V | pad0 | numbers; owned by CALLER
3790 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3791 // h ^ | in | 5
3792 // | | args | 4 Holes in incoming args owned by SELF
3793 // | | | | 3
3794 // | | +--------+
3795 // V | | old out| Empty on Intel, window on Sparc
3796 // | old |preserve| Must be even aligned.
3797 // | SP-+--------+----> Matcher::_old_SP, even aligned
3798 // | | in | 3 area for Intel ret address
3799 // Owned by |preserve| Empty on Sparc.
3800 // SELF +--------+
3801 // | | pad2 | 2 pad to align old SP
3802 // | +--------+ 1
3803 // | | locks | 0
3804 // | +--------+----> OptoReg::stack0(), even aligned
3805 // | | pad1 | 11 pad to align new SP
3806 // | +--------+
3807 // | | | 10
3808 // | | spills | 9 spills
3809 // V | | 8 (pad0 slot for callee)
3810 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3811 // ^ | out | 7
3812 // | | args | 6 Holes in outgoing args owned by CALLEE
3813 // Owned by +--------+
3814 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3815 // | new |preserve| Must be even-aligned.
3816 // | SP-+--------+----> Matcher::_new_SP, even aligned
3817 // | | |
3818 //
3819 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3820 // known from SELF's arguments and the Java calling convention.
3821 // Region 6-7 is determined per call site.
3822 // Note 2: If the calling convention leaves holes in the incoming argument
3823 // area, those holes are owned by SELF. Holes in the outgoing area
3824 // are owned by the CALLEE. Holes should not be necessary in the
3825 // incoming area, as the Java calling convention is completely under
3826 // the control of the AD file. Doubles can be sorted and packed to
3827 // avoid holes. Holes in the outgoing arguments may be necessary for
3828 // varargs C calling conventions.
3829 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3830 // even aligned with pad0 as needed.
3831 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3832 // (the latter is true on Intel but is it false on AArch64?)
3833 // region 6-11 is even aligned; it may be padded out more so that
3834 // the region from SP to FP meets the minimum stack alignment.
3835 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3836 // alignment. Region 11, pad1, may be dynamically extended so that
3837 // SP meets the minimum alignment.
3838
3839 frame %{
3840 // These three registers define part of the calling convention
3841 // between compiled code and the interpreter.
3842
3843 // Inline Cache Register or Method for I2C.
3844 inline_cache_reg(R12);
3845
3846 // Number of stack slots consumed by locking an object
3847 sync_stack_slots(2);
3848
3849 // Compiled code's Frame Pointer
3850 frame_pointer(R31);
3851
3852 // Interpreter stores its frame pointer in a register which is
3853 // stored to the stack by I2CAdaptors.
3854 // I2CAdaptors convert from interpreted java to compiled java.
3855 interpreter_frame_pointer(R29);
3856
3857 // Stack alignment requirement
3858 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3859
3860 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3861 // for calls to C. Supports the var-args backing area for register parms.
3862 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3863
3864 // The after-PROLOG location of the return address. Location of
3865 // return address specifies a type (REG or STACK) and a number
3866 // representing the register number (i.e. - use a register name) or
3867 // stack slot.
3868 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3869 // Otherwise, it is above the locks and verification slot and alignment word
3870 // TODO this may well be correct but need to check why that - 2 is there
3871 // ppc port uses 0 but we definitely need to allow for fixed_slots
3872 // which folds in the space used for monitors
3873 return_addr(STACK - 2 +
3874 align_up((Compile::current()->in_preserve_stack_slots() +
3875 Compile::current()->fixed_slots()),
3876 stack_alignment_in_slots()));
3877
3878 // Location of compiled Java return values. Same as C for now.
3879 return_value
3880 %{
3881 // TODO do we allow ideal_reg == Op_RegN???
3882 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3883 "only return normal values");
3884
3885 static const int lo[Op_RegL + 1] = { // enum name
3886 0, // Op_Node
3887 0, // Op_Set
3888 R0_num, // Op_RegN
3889 R0_num, // Op_RegI
3890 R0_num, // Op_RegP
3891 V0_num, // Op_RegF
3892 V0_num, // Op_RegD
3893 R0_num // Op_RegL
3894 };
3895
3896 static const int hi[Op_RegL + 1] = { // enum name
3897 0, // Op_Node
3898 0, // Op_Set
3899 OptoReg::Bad, // Op_RegN
3900 OptoReg::Bad, // Op_RegI
3901 R0_H_num, // Op_RegP
3902 OptoReg::Bad, // Op_RegF
3903 V0_H_num, // Op_RegD
3904 R0_H_num // Op_RegL
3905 };
3906
3907 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3908 %}
3909 %}
3910
3911 //----------ATTRIBUTES---------------------------------------------------------
3912 //----------Operand Attributes-------------------------------------------------
3913 op_attrib op_cost(1); // Required cost attribute
3914
3915 //----------Instruction Attributes---------------------------------------------
3916 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3917 ins_attrib ins_size(32); // Required size attribute (in bits)
3918 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3919 // a non-matching short branch variant
3920 // of some long branch?
3921 ins_attrib ins_alignment(4); // Required alignment attribute (must
3922 // be a power of 2) specifies the
3923 // alignment that some part of the
3924 // instruction (not necessarily the
3925 // start) requires. If > 1, a
3926 // compute_padding() function must be
3927 // provided for the instruction
3928
3929 //----------OPERANDS-----------------------------------------------------------
3930 // Operand definitions must precede instruction definitions for correct parsing
3931 // in the ADLC because operands constitute user defined types which are used in
3932 // instruction definitions.
3933
3934 //----------Simple Operands----------------------------------------------------
3935
3936 // Integer operands 32 bit
3937 // 32 bit immediate
3938 operand immI()
3939 %{
3940 match(ConI);
3941
3942 op_cost(0);
3943 format %{ %}
3944 interface(CONST_INTER);
3945 %}
3946
3947 // 32 bit zero
3948 operand immI0()
3949 %{
3950 predicate(n->get_int() == 0);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 // 32 bit unit increment
3959 operand immI_1()
3960 %{
3961 predicate(n->get_int() == 1);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 // 32 bit unit decrement
3970 operand immI_M1()
3971 %{
3972 predicate(n->get_int() == -1);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 // Shift values for add/sub extension shift
3981 operand immIExt()
3982 %{
3983 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 operand immI_gt_1()
3992 %{
3993 predicate(n->get_int() > 1);
3994 match(ConI);
3995
3996 op_cost(0);
3997 format %{ %}
3998 interface(CONST_INTER);
3999 %}
4000
4001 operand immI_le_4()
4002 %{
4003 predicate(n->get_int() <= 4);
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_16()
4012 %{
4013 predicate(n->get_int() == 16);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_24()
4022 %{
4023 predicate(n->get_int() == 24);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_32()
4032 %{
4033 predicate(n->get_int() == 32);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_48()
4042 %{
4043 predicate(n->get_int() == 48);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immI_56()
4052 %{
4053 predicate(n->get_int() == 56);
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immI_255()
4062 %{
4063 predicate(n->get_int() == 255);
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 operand immI_65535()
4072 %{
4073 predicate(n->get_int() == 65535);
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immI_positive()
4082 %{
4083 predicate(n->get_int() > 0);
4084 match(ConI);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 // BoolTest condition for signed compare
4092 operand immI_cmp_cond()
4093 %{
4094 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4095 match(ConI);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 // BoolTest condition for unsigned compare
4103 operand immI_cmpU_cond()
4104 %{
4105 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4106 match(ConI);
4107
4108 op_cost(0);
4109 format %{ %}
4110 interface(CONST_INTER);
4111 %}
4112
4113 operand immL_255()
4114 %{
4115 predicate(n->get_long() == 255L);
4116 match(ConL);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immL_65535()
4124 %{
4125 predicate(n->get_long() == 65535L);
4126 match(ConL);
4127
4128 op_cost(0);
4129 format %{ %}
4130 interface(CONST_INTER);
4131 %}
4132
4133 operand immL_4294967295()
4134 %{
4135 predicate(n->get_long() == 4294967295L);
4136 match(ConL);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 operand immL_bitmask()
4144 %{
4145 predicate((n->get_long() != 0)
4146 && ((n->get_long() & 0xc000000000000000l) == 0)
4147 && is_power_of_2(n->get_long() + 1));
4148 match(ConL);
4149
4150 op_cost(0);
4151 format %{ %}
4152 interface(CONST_INTER);
4153 %}
4154
4155 operand immI_bitmask()
4156 %{
4157 predicate((n->get_int() != 0)
4158 && ((n->get_int() & 0xc0000000) == 0)
4159 && is_power_of_2(n->get_int() + 1));
4160 match(ConI);
4161
4162 op_cost(0);
4163 format %{ %}
4164 interface(CONST_INTER);
4165 %}
4166
4167 operand immL_positive_bitmaskI()
4168 %{
4169 predicate((n->get_long() != 0)
4170 && ((julong)n->get_long() < 0x80000000ULL)
4171 && is_power_of_2(n->get_long() + 1));
4172 match(ConL);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 // Scale values for scaled offset addressing modes (up to long but not quad)
4180 operand immIScale()
4181 %{
4182 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 // 5 bit signed integer
4191 operand immI5()
4192 %{
4193 predicate(Assembler::is_simm(n->get_int(), 5));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 // 7 bit unsigned integer
4202 operand immIU7()
4203 %{
4204 predicate(Assembler::is_uimm(n->get_int(), 7));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 // Offset for scaled or unscaled immediate loads and stores
4213 operand immIOffset()
4214 %{
4215 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4216 match(ConI);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 operand immIOffset1()
4224 %{
4225 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4226 match(ConI);
4227
4228 op_cost(0);
4229 format %{ %}
4230 interface(CONST_INTER);
4231 %}
4232
4233 operand immIOffset2()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immIOffset4()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4246 match(ConI);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immIOffset8()
4254 %{
4255 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4256 match(ConI);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immIOffset16()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4266 match(ConI);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immLOffset()
4274 %{
4275 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4276 match(ConL);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immLoffset1()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4286 match(ConL);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 operand immLoffset2()
4294 %{
4295 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 operand immLoffset4()
4304 %{
4305 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 operand immLoffset8()
4314 %{
4315 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 operand immLoffset16()
4324 %{
4325 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4326 match(ConL);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 // 5 bit signed long integer
4334 operand immL5()
4335 %{
4336 predicate(Assembler::is_simm(n->get_long(), 5));
4337 match(ConL);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 // 7 bit unsigned long integer
4345 operand immLU7()
4346 %{
4347 predicate(Assembler::is_uimm(n->get_long(), 7));
4348 match(ConL);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value.
4356 operand immI8()
4357 %{
4358 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4359 match(ConI);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 // 8 bit signed value (simm8), or #simm8 LSL 8.
4367 operand immI8_shift8()
4368 %{
4369 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4370 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 8 bit signed value (simm8), or #simm8 LSL 8.
4379 operand immL8_shift8()
4380 %{
4381 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4382 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4383 match(ConL);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 // 8 bit integer valid for vector add sub immediate
4391 operand immBAddSubV()
4392 %{
4393 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 // 32 bit integer valid for add sub immediate
4402 operand immIAddSub()
4403 %{
4404 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4405 match(ConI);
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // 32 bit integer valid for vector add sub immediate
4412 operand immIAddSubV()
4413 %{
4414 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4415 match(ConI);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 // 32 bit unsigned integer valid for logical immediate
4423
4424 operand immBLog()
4425 %{
4426 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4427 match(ConI);
4428
4429 op_cost(0);
4430 format %{ %}
4431 interface(CONST_INTER);
4432 %}
4433
4434 operand immSLog()
4435 %{
4436 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4437 match(ConI);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 operand immILog()
4445 %{
4446 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // Integer operands 64 bit
4455 // 64 bit immediate
4456 operand immL()
4457 %{
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit zero
4466 operand immL0()
4467 %{
4468 predicate(n->get_long() == 0);
4469 match(ConL);
4470
4471 op_cost(0);
4472 format %{ %}
4473 interface(CONST_INTER);
4474 %}
4475
4476 // 64 bit unit decrement
4477 operand immL_M1()
4478 %{
4479 predicate(n->get_long() == -1);
4480 match(ConL);
4481
4482 op_cost(0);
4483 format %{ %}
4484 interface(CONST_INTER);
4485 %}
4486
4487 // 64 bit integer valid for add sub immediate
4488 operand immLAddSub()
4489 %{
4490 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4491 match(ConL);
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 // 64 bit integer valid for addv subv immediate
4498 operand immLAddSubV()
4499 %{
4500 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4501 match(ConL);
4502
4503 op_cost(0);
4504 format %{ %}
4505 interface(CONST_INTER);
4506 %}
4507
4508 // 64 bit integer valid for logical immediate
4509 operand immLLog()
4510 %{
4511 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4512 match(ConL);
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // Long Immediate: low 32-bit mask
4519 operand immL_32bits()
4520 %{
4521 predicate(n->get_long() == 0xFFFFFFFFL);
4522 match(ConL);
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // Pointer operands
4529 // Pointer Immediate
4530 operand immP()
4531 %{
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // nullptr Pointer Immediate
4540 operand immP0()
4541 %{
4542 predicate(n->get_ptr() == 0);
4543 match(ConP);
4544
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // Pointer Immediate One
4551 // this is used in object initialization (initial object header)
4552 operand immP_1()
4553 %{
4554 predicate(n->get_ptr() == 1);
4555 match(ConP);
4556
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Card Table Byte Map Base
4563 operand immByteMapBase()
4564 %{
4565 // Get base of card map
4566 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4567 SHENANDOAHGC_ONLY(!BarrierSet::barrier_set()->is_a(BarrierSet::ShenandoahBarrierSet) &&)
4568 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4569 match(ConP);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // Float and Double operands
4577 // Double Immediate
4578 operand immD()
4579 %{
4580 match(ConD);
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // Double Immediate: +0.0d
4587 operand immD0()
4588 %{
4589 predicate(jlong_cast(n->getd()) == 0);
4590 match(ConD);
4591
4592 op_cost(0);
4593 format %{ %}
4594 interface(CONST_INTER);
4595 %}
4596
4597 // constant 'double +0.0'.
4598 operand immDPacked()
4599 %{
4600 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4601 match(ConD);
4602 op_cost(0);
4603 format %{ %}
4604 interface(CONST_INTER);
4605 %}
4606
4607 // Float Immediate
4608 operand immF()
4609 %{
4610 match(ConF);
4611 op_cost(0);
4612 format %{ %}
4613 interface(CONST_INTER);
4614 %}
4615
4616 // Float Immediate: +0.0f.
4617 operand immF0()
4618 %{
4619 predicate(jint_cast(n->getf()) == 0);
4620 match(ConF);
4621
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 //
4628 operand immFPacked()
4629 %{
4630 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4631 match(ConF);
4632 op_cost(0);
4633 format %{ %}
4634 interface(CONST_INTER);
4635 %}
4636
4637 // Narrow pointer operands
4638 // Narrow Pointer Immediate
4639 operand immN()
4640 %{
4641 match(ConN);
4642
4643 op_cost(0);
4644 format %{ %}
4645 interface(CONST_INTER);
4646 %}
4647
4648 // Narrow nullptr Pointer Immediate
4649 operand immN0()
4650 %{
4651 predicate(n->get_narrowcon() == 0);
4652 match(ConN);
4653
4654 op_cost(0);
4655 format %{ %}
4656 interface(CONST_INTER);
4657 %}
4658
4659 operand immNKlass()
4660 %{
4661 match(ConNKlass);
4662
4663 op_cost(0);
4664 format %{ %}
4665 interface(CONST_INTER);
4666 %}
4667
4668 // Integer 32 bit Register Operands
4669 // Integer 32 bitRegister (excludes SP)
4670 operand iRegI()
4671 %{
4672 constraint(ALLOC_IN_RC(any_reg32));
4673 match(RegI);
4674 match(iRegINoSp);
4675 op_cost(0);
4676 format %{ %}
4677 interface(REG_INTER);
4678 %}
4679
4680 // Integer 32 bit Register not Special
4681 operand iRegINoSp()
4682 %{
4683 constraint(ALLOC_IN_RC(no_special_reg32));
4684 match(RegI);
4685 op_cost(0);
4686 format %{ %}
4687 interface(REG_INTER);
4688 %}
4689
4690 // Integer 64 bit Register Operands
4691 // Integer 64 bit Register (includes SP)
4692 operand iRegL()
4693 %{
4694 constraint(ALLOC_IN_RC(any_reg));
4695 match(RegL);
4696 match(iRegLNoSp);
4697 op_cost(0);
4698 format %{ %}
4699 interface(REG_INTER);
4700 %}
4701
4702 // Integer 64 bit Register not Special
4703 operand iRegLNoSp()
4704 %{
4705 constraint(ALLOC_IN_RC(no_special_reg));
4706 match(RegL);
4707 match(iRegL_R0);
4708 format %{ %}
4709 interface(REG_INTER);
4710 %}
4711
4712 // Pointer Register Operands
4713 // Pointer Register
4714 operand iRegP()
4715 %{
4716 constraint(ALLOC_IN_RC(ptr_reg));
4717 match(RegP);
4718 match(iRegPNoSp);
4719 match(iRegP_R0);
4720 //match(iRegP_R2);
4721 //match(iRegP_R4);
4722 match(iRegP_R5);
4723 match(thread_RegP);
4724 op_cost(0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // Pointer 64 bit Register not Special
4730 operand iRegPNoSp()
4731 %{
4732 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4733 match(RegP);
4734 // match(iRegP);
4735 // match(iRegP_R0);
4736 // match(iRegP_R2);
4737 // match(iRegP_R4);
4738 // match(iRegP_R5);
4739 // match(thread_RegP);
4740 op_cost(0);
4741 format %{ %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // This operand is not allowed to use rfp even if
4746 // rfp is not used to hold the frame pointer.
4747 operand iRegPNoSpNoRfp()
4748 %{
4749 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4750 match(RegP);
4751 match(iRegPNoSp);
4752 op_cost(0);
4753 format %{ %}
4754 interface(REG_INTER);
4755 %}
4756
4757 // Pointer 64 bit Register R0 only
4758 operand iRegP_R0()
4759 %{
4760 constraint(ALLOC_IN_RC(r0_reg));
4761 match(RegP);
4762 // match(iRegP);
4763 match(iRegPNoSp);
4764 op_cost(0);
4765 format %{ %}
4766 interface(REG_INTER);
4767 %}
4768
4769 // Pointer 64 bit Register R1 only
4770 operand iRegP_R1()
4771 %{
4772 constraint(ALLOC_IN_RC(r1_reg));
4773 match(RegP);
4774 // match(iRegP);
4775 match(iRegPNoSp);
4776 op_cost(0);
4777 format %{ %}
4778 interface(REG_INTER);
4779 %}
4780
4781 // Pointer 64 bit Register R2 only
4782 operand iRegP_R2()
4783 %{
4784 constraint(ALLOC_IN_RC(r2_reg));
4785 match(RegP);
4786 // match(iRegP);
4787 match(iRegPNoSp);
4788 op_cost(0);
4789 format %{ %}
4790 interface(REG_INTER);
4791 %}
4792
4793 // Pointer 64 bit Register R3 only
4794 operand iRegP_R3()
4795 %{
4796 constraint(ALLOC_IN_RC(r3_reg));
4797 match(RegP);
4798 // match(iRegP);
4799 match(iRegPNoSp);
4800 op_cost(0);
4801 format %{ %}
4802 interface(REG_INTER);
4803 %}
4804
4805 // Pointer 64 bit Register R4 only
4806 operand iRegP_R4()
4807 %{
4808 constraint(ALLOC_IN_RC(r4_reg));
4809 match(RegP);
4810 // match(iRegP);
4811 match(iRegPNoSp);
4812 op_cost(0);
4813 format %{ %}
4814 interface(REG_INTER);
4815 %}
4816
4817 // Pointer 64 bit Register R5 only
4818 operand iRegP_R5()
4819 %{
4820 constraint(ALLOC_IN_RC(r5_reg));
4821 match(RegP);
4822 // match(iRegP);
4823 match(iRegPNoSp);
4824 op_cost(0);
4825 format %{ %}
4826 interface(REG_INTER);
4827 %}
4828
4829 // Pointer 64 bit Register R10 only
4830 operand iRegP_R10()
4831 %{
4832 constraint(ALLOC_IN_RC(r10_reg));
4833 match(RegP);
4834 // match(iRegP);
4835 match(iRegPNoSp);
4836 op_cost(0);
4837 format %{ %}
4838 interface(REG_INTER);
4839 %}
4840
4841 // Long 64 bit Register R0 only
4842 operand iRegL_R0()
4843 %{
4844 constraint(ALLOC_IN_RC(r0_reg));
4845 match(RegL);
4846 match(iRegLNoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852 // Long 64 bit Register R11 only
4853 operand iRegL_R11()
4854 %{
4855 constraint(ALLOC_IN_RC(r11_reg));
4856 match(RegL);
4857 match(iRegLNoSp);
4858 op_cost(0);
4859 format %{ %}
4860 interface(REG_INTER);
4861 %}
4862
4863 // Register R0 only
4864 operand iRegI_R0()
4865 %{
4866 constraint(ALLOC_IN_RC(int_r0_reg));
4867 match(RegI);
4868 match(iRegINoSp);
4869 op_cost(0);
4870 format %{ %}
4871 interface(REG_INTER);
4872 %}
4873
4874 // Register R2 only
4875 operand iRegI_R2()
4876 %{
4877 constraint(ALLOC_IN_RC(int_r2_reg));
4878 match(RegI);
4879 match(iRegINoSp);
4880 op_cost(0);
4881 format %{ %}
4882 interface(REG_INTER);
4883 %}
4884
4885 // Register R3 only
4886 operand iRegI_R3()
4887 %{
4888 constraint(ALLOC_IN_RC(int_r3_reg));
4889 match(RegI);
4890 match(iRegINoSp);
4891 op_cost(0);
4892 format %{ %}
4893 interface(REG_INTER);
4894 %}
4895
4896
4897 // Register R4 only
4898 operand iRegI_R4()
4899 %{
4900 constraint(ALLOC_IN_RC(int_r4_reg));
4901 match(RegI);
4902 match(iRegINoSp);
4903 op_cost(0);
4904 format %{ %}
4905 interface(REG_INTER);
4906 %}
4907
4908
4909 // Pointer Register Operands
4910 // Narrow Pointer Register
4911 operand iRegN()
4912 %{
4913 constraint(ALLOC_IN_RC(any_reg32));
4914 match(RegN);
4915 match(iRegNNoSp);
4916 op_cost(0);
4917 format %{ %}
4918 interface(REG_INTER);
4919 %}
4920
4921 // Integer 64 bit Register not Special
4922 operand iRegNNoSp()
4923 %{
4924 constraint(ALLOC_IN_RC(no_special_reg32));
4925 match(RegN);
4926 op_cost(0);
4927 format %{ %}
4928 interface(REG_INTER);
4929 %}
4930
4931 // Float Register
4932 // Float register operands
4933 operand vRegF()
4934 %{
4935 constraint(ALLOC_IN_RC(float_reg));
4936 match(RegF);
4937
4938 op_cost(0);
4939 format %{ %}
4940 interface(REG_INTER);
4941 %}
4942
4943 // Double Register
4944 // Double register operands
4945 operand vRegD()
4946 %{
4947 constraint(ALLOC_IN_RC(double_reg));
4948 match(RegD);
4949
4950 op_cost(0);
4951 format %{ %}
4952 interface(REG_INTER);
4953 %}
4954
4955 // Generic vector class. This will be used for
4956 // all vector operands, including NEON and SVE.
4957 operand vReg()
4958 %{
4959 constraint(ALLOC_IN_RC(dynamic));
4960 match(VecA);
4961 match(VecD);
4962 match(VecX);
4963
4964 op_cost(0);
4965 format %{ %}
4966 interface(REG_INTER);
4967 %}
4968
4969 operand vecA()
4970 %{
4971 constraint(ALLOC_IN_RC(vectora_reg));
4972 match(VecA);
4973
4974 op_cost(0);
4975 format %{ %}
4976 interface(REG_INTER);
4977 %}
4978
4979 operand vecD()
4980 %{
4981 constraint(ALLOC_IN_RC(vectord_reg));
4982 match(VecD);
4983
4984 op_cost(0);
4985 format %{ %}
4986 interface(REG_INTER);
4987 %}
4988
4989 operand vecX()
4990 %{
4991 constraint(ALLOC_IN_RC(vectorx_reg));
4992 match(VecX);
4993
4994 op_cost(0);
4995 format %{ %}
4996 interface(REG_INTER);
4997 %}
4998
4999 operand vRegD_V0()
5000 %{
5001 constraint(ALLOC_IN_RC(v0_reg));
5002 match(RegD);
5003 op_cost(0);
5004 format %{ %}
5005 interface(REG_INTER);
5006 %}
5007
5008 operand vRegD_V1()
5009 %{
5010 constraint(ALLOC_IN_RC(v1_reg));
5011 match(RegD);
5012 op_cost(0);
5013 format %{ %}
5014 interface(REG_INTER);
5015 %}
5016
5017 operand vRegD_V2()
5018 %{
5019 constraint(ALLOC_IN_RC(v2_reg));
5020 match(RegD);
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 operand vRegD_V3()
5027 %{
5028 constraint(ALLOC_IN_RC(v3_reg));
5029 match(RegD);
5030 op_cost(0);
5031 format %{ %}
5032 interface(REG_INTER);
5033 %}
5034
5035 operand vRegD_V4()
5036 %{
5037 constraint(ALLOC_IN_RC(v4_reg));
5038 match(RegD);
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 operand vRegD_V5()
5045 %{
5046 constraint(ALLOC_IN_RC(v5_reg));
5047 match(RegD);
5048 op_cost(0);
5049 format %{ %}
5050 interface(REG_INTER);
5051 %}
5052
5053 operand vRegD_V6()
5054 %{
5055 constraint(ALLOC_IN_RC(v6_reg));
5056 match(RegD);
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vRegD_V7()
5063 %{
5064 constraint(ALLOC_IN_RC(v7_reg));
5065 match(RegD);
5066 op_cost(0);
5067 format %{ %}
5068 interface(REG_INTER);
5069 %}
5070
5071 operand vRegD_V12()
5072 %{
5073 constraint(ALLOC_IN_RC(v12_reg));
5074 match(RegD);
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vRegD_V13()
5081 %{
5082 constraint(ALLOC_IN_RC(v13_reg));
5083 match(RegD);
5084 op_cost(0);
5085 format %{ %}
5086 interface(REG_INTER);
5087 %}
5088
5089 operand pReg()
5090 %{
5091 constraint(ALLOC_IN_RC(pr_reg));
5092 match(RegVectMask);
5093 match(pRegGov);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand pRegGov()
5100 %{
5101 constraint(ALLOC_IN_RC(gov_pr));
5102 match(RegVectMask);
5103 match(pReg);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand pRegGov_P0()
5110 %{
5111 constraint(ALLOC_IN_RC(p0_reg));
5112 match(RegVectMask);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand pRegGov_P1()
5119 %{
5120 constraint(ALLOC_IN_RC(p1_reg));
5121 match(RegVectMask);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 // Flags register, used as output of signed compare instructions
5128
5129 // note that on AArch64 we also use this register as the output for
5130 // for floating point compare instructions (CmpF CmpD). this ensures
5131 // that ordered inequality tests use GT, GE, LT or LE none of which
5132 // pass through cases where the result is unordered i.e. one or both
5133 // inputs to the compare is a NaN. this means that the ideal code can
5134 // replace e.g. a GT with an LE and not end up capturing the NaN case
5135 // (where the comparison should always fail). EQ and NE tests are
5136 // always generated in ideal code so that unordered folds into the NE
5137 // case, matching the behaviour of AArch64 NE.
5138 //
5139 // This differs from x86 where the outputs of FP compares use a
5140 // special FP flags registers and where compares based on this
5141 // register are distinguished into ordered inequalities (cmpOpUCF) and
5142 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5143 // to explicitly handle the unordered case in branches. x86 also has
5144 // to include extra CMoveX rules to accept a cmpOpUCF input.
5145
5146 operand rFlagsReg()
5147 %{
5148 constraint(ALLOC_IN_RC(int_flags));
5149 match(RegFlags);
5150
5151 op_cost(0);
5152 format %{ "RFLAGS" %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Flags register, used as output of unsigned compare instructions
5157 operand rFlagsRegU()
5158 %{
5159 constraint(ALLOC_IN_RC(int_flags));
5160 match(RegFlags);
5161
5162 op_cost(0);
5163 format %{ "RFLAGSU" %}
5164 interface(REG_INTER);
5165 %}
5166
5167 // Special Registers
5168
5169 // Method Register
5170 operand inline_cache_RegP(iRegP reg)
5171 %{
5172 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5173 match(reg);
5174 match(iRegPNoSp);
5175 op_cost(0);
5176 format %{ %}
5177 interface(REG_INTER);
5178 %}
5179
5180 // Thread Register
5181 operand thread_RegP(iRegP reg)
5182 %{
5183 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5184 match(reg);
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 //----------Memory Operands----------------------------------------------------
5191
5192 operand indirect(iRegP reg)
5193 %{
5194 constraint(ALLOC_IN_RC(ptr_reg));
5195 match(reg);
5196 op_cost(0);
5197 format %{ "[$reg]" %}
5198 interface(MEMORY_INTER) %{
5199 base($reg);
5200 index(0xffffffff);
5201 scale(0x0);
5202 disp(0x0);
5203 %}
5204 %}
5205
5206 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5207 %{
5208 constraint(ALLOC_IN_RC(ptr_reg));
5209 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5210 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5211 op_cost(0);
5212 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5213 interface(MEMORY_INTER) %{
5214 base($reg);
5215 index($ireg);
5216 scale($scale);
5217 disp(0x0);
5218 %}
5219 %}
5220
5221 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5222 %{
5223 constraint(ALLOC_IN_RC(ptr_reg));
5224 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5225 match(AddP reg (LShiftL lreg scale));
5226 op_cost(0);
5227 format %{ "$reg, $lreg lsl($scale)" %}
5228 interface(MEMORY_INTER) %{
5229 base($reg);
5230 index($lreg);
5231 scale($scale);
5232 disp(0x0);
5233 %}
5234 %}
5235
5236 operand indIndexI2L(iRegP reg, iRegI ireg)
5237 %{
5238 constraint(ALLOC_IN_RC(ptr_reg));
5239 match(AddP reg (ConvI2L ireg));
5240 op_cost(0);
5241 format %{ "$reg, $ireg, 0, I2L" %}
5242 interface(MEMORY_INTER) %{
5243 base($reg);
5244 index($ireg);
5245 scale(0x0);
5246 disp(0x0);
5247 %}
5248 %}
5249
5250 operand indIndex(iRegP reg, iRegL lreg)
5251 %{
5252 constraint(ALLOC_IN_RC(ptr_reg));
5253 match(AddP reg lreg);
5254 op_cost(0);
5255 format %{ "$reg, $lreg" %}
5256 interface(MEMORY_INTER) %{
5257 base($reg);
5258 index($lreg);
5259 scale(0x0);
5260 disp(0x0);
5261 %}
5262 %}
5263
5264 operand indOffI1(iRegP reg, immIOffset1 off)
5265 %{
5266 constraint(ALLOC_IN_RC(ptr_reg));
5267 match(AddP reg off);
5268 op_cost(0);
5269 format %{ "[$reg, $off]" %}
5270 interface(MEMORY_INTER) %{
5271 base($reg);
5272 index(0xffffffff);
5273 scale(0x0);
5274 disp($off);
5275 %}
5276 %}
5277
5278 operand indOffI2(iRegP reg, immIOffset2 off)
5279 %{
5280 constraint(ALLOC_IN_RC(ptr_reg));
5281 match(AddP reg off);
5282 op_cost(0);
5283 format %{ "[$reg, $off]" %}
5284 interface(MEMORY_INTER) %{
5285 base($reg);
5286 index(0xffffffff);
5287 scale(0x0);
5288 disp($off);
5289 %}
5290 %}
5291
5292 operand indOffI4(iRegP reg, immIOffset4 off)
5293 %{
5294 constraint(ALLOC_IN_RC(ptr_reg));
5295 match(AddP reg off);
5296 op_cost(0);
5297 format %{ "[$reg, $off]" %}
5298 interface(MEMORY_INTER) %{
5299 base($reg);
5300 index(0xffffffff);
5301 scale(0x0);
5302 disp($off);
5303 %}
5304 %}
5305
5306 operand indOffI8(iRegP reg, immIOffset8 off)
5307 %{
5308 constraint(ALLOC_IN_RC(ptr_reg));
5309 match(AddP reg off);
5310 op_cost(0);
5311 format %{ "[$reg, $off]" %}
5312 interface(MEMORY_INTER) %{
5313 base($reg);
5314 index(0xffffffff);
5315 scale(0x0);
5316 disp($off);
5317 %}
5318 %}
5319
5320 operand indOffI16(iRegP reg, immIOffset16 off)
5321 %{
5322 constraint(ALLOC_IN_RC(ptr_reg));
5323 match(AddP reg off);
5324 op_cost(0);
5325 format %{ "[$reg, $off]" %}
5326 interface(MEMORY_INTER) %{
5327 base($reg);
5328 index(0xffffffff);
5329 scale(0x0);
5330 disp($off);
5331 %}
5332 %}
5333
5334 operand indOffL1(iRegP reg, immLoffset1 off)
5335 %{
5336 constraint(ALLOC_IN_RC(ptr_reg));
5337 match(AddP reg off);
5338 op_cost(0);
5339 format %{ "[$reg, $off]" %}
5340 interface(MEMORY_INTER) %{
5341 base($reg);
5342 index(0xffffffff);
5343 scale(0x0);
5344 disp($off);
5345 %}
5346 %}
5347
5348 operand indOffL2(iRegP reg, immLoffset2 off)
5349 %{
5350 constraint(ALLOC_IN_RC(ptr_reg));
5351 match(AddP reg off);
5352 op_cost(0);
5353 format %{ "[$reg, $off]" %}
5354 interface(MEMORY_INTER) %{
5355 base($reg);
5356 index(0xffffffff);
5357 scale(0x0);
5358 disp($off);
5359 %}
5360 %}
5361
5362 operand indOffL4(iRegP reg, immLoffset4 off)
5363 %{
5364 constraint(ALLOC_IN_RC(ptr_reg));
5365 match(AddP reg off);
5366 op_cost(0);
5367 format %{ "[$reg, $off]" %}
5368 interface(MEMORY_INTER) %{
5369 base($reg);
5370 index(0xffffffff);
5371 scale(0x0);
5372 disp($off);
5373 %}
5374 %}
5375
5376 operand indOffL8(iRegP reg, immLoffset8 off)
5377 %{
5378 constraint(ALLOC_IN_RC(ptr_reg));
5379 match(AddP reg off);
5380 op_cost(0);
5381 format %{ "[$reg, $off]" %}
5382 interface(MEMORY_INTER) %{
5383 base($reg);
5384 index(0xffffffff);
5385 scale(0x0);
5386 disp($off);
5387 %}
5388 %}
5389
5390 operand indOffL16(iRegP reg, immLoffset16 off)
5391 %{
5392 constraint(ALLOC_IN_RC(ptr_reg));
5393 match(AddP reg off);
5394 op_cost(0);
5395 format %{ "[$reg, $off]" %}
5396 interface(MEMORY_INTER) %{
5397 base($reg);
5398 index(0xffffffff);
5399 scale(0x0);
5400 disp($off);
5401 %}
5402 %}
5403
5404 operand indirectX2P(iRegL reg)
5405 %{
5406 constraint(ALLOC_IN_RC(ptr_reg));
5407 match(CastX2P reg);
5408 op_cost(0);
5409 format %{ "[$reg]\t# long -> ptr" %}
5410 interface(MEMORY_INTER) %{
5411 base($reg);
5412 index(0xffffffff);
5413 scale(0x0);
5414 disp(0x0);
5415 %}
5416 %}
5417
5418 operand indOffX2P(iRegL reg, immLOffset off)
5419 %{
5420 constraint(ALLOC_IN_RC(ptr_reg));
5421 match(AddP (CastX2P reg) off);
5422 op_cost(0);
5423 format %{ "[$reg, $off]\t# long -> ptr" %}
5424 interface(MEMORY_INTER) %{
5425 base($reg);
5426 index(0xffffffff);
5427 scale(0x0);
5428 disp($off);
5429 %}
5430 %}
5431
5432 operand indirectN(iRegN reg)
5433 %{
5434 predicate(CompressedOops::shift() == 0);
5435 constraint(ALLOC_IN_RC(ptr_reg));
5436 match(DecodeN reg);
5437 op_cost(0);
5438 format %{ "[$reg]\t# narrow" %}
5439 interface(MEMORY_INTER) %{
5440 base($reg);
5441 index(0xffffffff);
5442 scale(0x0);
5443 disp(0x0);
5444 %}
5445 %}
5446
5447 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5448 %{
5449 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5450 constraint(ALLOC_IN_RC(ptr_reg));
5451 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5452 op_cost(0);
5453 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5454 interface(MEMORY_INTER) %{
5455 base($reg);
5456 index($ireg);
5457 scale($scale);
5458 disp(0x0);
5459 %}
5460 %}
5461
5462 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5463 %{
5464 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5465 constraint(ALLOC_IN_RC(ptr_reg));
5466 match(AddP (DecodeN reg) (LShiftL lreg scale));
5467 op_cost(0);
5468 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5469 interface(MEMORY_INTER) %{
5470 base($reg);
5471 index($lreg);
5472 scale($scale);
5473 disp(0x0);
5474 %}
5475 %}
5476
5477 operand indIndexI2LN(iRegN reg, iRegI ireg)
5478 %{
5479 predicate(CompressedOops::shift() == 0);
5480 constraint(ALLOC_IN_RC(ptr_reg));
5481 match(AddP (DecodeN reg) (ConvI2L ireg));
5482 op_cost(0);
5483 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5484 interface(MEMORY_INTER) %{
5485 base($reg);
5486 index($ireg);
5487 scale(0x0);
5488 disp(0x0);
5489 %}
5490 %}
5491
5492 operand indIndexN(iRegN reg, iRegL lreg)
5493 %{
5494 predicate(CompressedOops::shift() == 0);
5495 constraint(ALLOC_IN_RC(ptr_reg));
5496 match(AddP (DecodeN reg) lreg);
5497 op_cost(0);
5498 format %{ "$reg, $lreg\t# narrow" %}
5499 interface(MEMORY_INTER) %{
5500 base($reg);
5501 index($lreg);
5502 scale(0x0);
5503 disp(0x0);
5504 %}
5505 %}
5506
5507 operand indOffIN(iRegN reg, immIOffset off)
5508 %{
5509 predicate(CompressedOops::shift() == 0);
5510 constraint(ALLOC_IN_RC(ptr_reg));
5511 match(AddP (DecodeN reg) off);
5512 op_cost(0);
5513 format %{ "[$reg, $off]\t# narrow" %}
5514 interface(MEMORY_INTER) %{
5515 base($reg);
5516 index(0xffffffff);
5517 scale(0x0);
5518 disp($off);
5519 %}
5520 %}
5521
5522 operand indOffLN(iRegN reg, immLOffset off)
5523 %{
5524 predicate(CompressedOops::shift() == 0);
5525 constraint(ALLOC_IN_RC(ptr_reg));
5526 match(AddP (DecodeN reg) off);
5527 op_cost(0);
5528 format %{ "[$reg, $off]\t# narrow" %}
5529 interface(MEMORY_INTER) %{
5530 base($reg);
5531 index(0xffffffff);
5532 scale(0x0);
5533 disp($off);
5534 %}
5535 %}
5536
5537
5538 //----------Special Memory Operands--------------------------------------------
5539 // Stack Slot Operand - This operand is used for loading and storing temporary
5540 // values on the stack where a match requires a value to
5541 // flow through memory.
5542 operand stackSlotP(sRegP reg)
5543 %{
5544 constraint(ALLOC_IN_RC(stack_slots));
5545 op_cost(100);
5546 // No match rule because this operand is only generated in matching
5547 // match(RegP);
5548 format %{ "[$reg]" %}
5549 interface(MEMORY_INTER) %{
5550 base(0x1e); // RSP
5551 index(0x0); // No Index
5552 scale(0x0); // No Scale
5553 disp($reg); // Stack Offset
5554 %}
5555 %}
5556
5557 operand stackSlotI(sRegI reg)
5558 %{
5559 constraint(ALLOC_IN_RC(stack_slots));
5560 // No match rule because this operand is only generated in matching
5561 // match(RegI);
5562 format %{ "[$reg]" %}
5563 interface(MEMORY_INTER) %{
5564 base(0x1e); // RSP
5565 index(0x0); // No Index
5566 scale(0x0); // No Scale
5567 disp($reg); // Stack Offset
5568 %}
5569 %}
5570
5571 operand stackSlotF(sRegF reg)
5572 %{
5573 constraint(ALLOC_IN_RC(stack_slots));
5574 // No match rule because this operand is only generated in matching
5575 // match(RegF);
5576 format %{ "[$reg]" %}
5577 interface(MEMORY_INTER) %{
5578 base(0x1e); // RSP
5579 index(0x0); // No Index
5580 scale(0x0); // No Scale
5581 disp($reg); // Stack Offset
5582 %}
5583 %}
5584
5585 operand stackSlotD(sRegD reg)
5586 %{
5587 constraint(ALLOC_IN_RC(stack_slots));
5588 // No match rule because this operand is only generated in matching
5589 // match(RegD);
5590 format %{ "[$reg]" %}
5591 interface(MEMORY_INTER) %{
5592 base(0x1e); // RSP
5593 index(0x0); // No Index
5594 scale(0x0); // No Scale
5595 disp($reg); // Stack Offset
5596 %}
5597 %}
5598
5599 operand stackSlotL(sRegL reg)
5600 %{
5601 constraint(ALLOC_IN_RC(stack_slots));
5602 // No match rule because this operand is only generated in matching
5603 // match(RegL);
5604 format %{ "[$reg]" %}
5605 interface(MEMORY_INTER) %{
5606 base(0x1e); // RSP
5607 index(0x0); // No Index
5608 scale(0x0); // No Scale
5609 disp($reg); // Stack Offset
5610 %}
5611 %}
5612
5613 // Operands for expressing Control Flow
5614 // NOTE: Label is a predefined operand which should not be redefined in
5615 // the AD file. It is generically handled within the ADLC.
5616
5617 //----------Conditional Branch Operands----------------------------------------
5618 // Comparison Op - This is the operation of the comparison, and is limited to
5619 // the following set of codes:
5620 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5621 //
5622 // Other attributes of the comparison, such as unsignedness, are specified
5623 // by the comparison instruction that sets a condition code flags register.
5624 // That result is represented by a flags operand whose subtype is appropriate
5625 // to the unsignedness (etc.) of the comparison.
5626 //
5627 // Later, the instruction which matches both the Comparison Op (a Bool) and
5628 // the flags (produced by the Cmp) specifies the coding of the comparison op
5629 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5630
5631 // used for signed integral comparisons and fp comparisons
5632
5633 operand cmpOp()
5634 %{
5635 match(Bool);
5636
5637 format %{ "" %}
5638 interface(COND_INTER) %{
5639 equal(0x0, "eq");
5640 not_equal(0x1, "ne");
5641 less(0xb, "lt");
5642 greater_equal(0xa, "ge");
5643 less_equal(0xd, "le");
5644 greater(0xc, "gt");
5645 overflow(0x6, "vs");
5646 no_overflow(0x7, "vc");
5647 %}
5648 %}
5649
5650 // used for unsigned integral comparisons
5651
5652 operand cmpOpU()
5653 %{
5654 match(Bool);
5655
5656 format %{ "" %}
5657 interface(COND_INTER) %{
5658 equal(0x0, "eq");
5659 not_equal(0x1, "ne");
5660 less(0x3, "lo");
5661 greater_equal(0x2, "hs");
5662 less_equal(0x9, "ls");
5663 greater(0x8, "hi");
5664 overflow(0x6, "vs");
5665 no_overflow(0x7, "vc");
5666 %}
5667 %}
5668
5669 // used for certain integral comparisons which can be
5670 // converted to cbxx or tbxx instructions
5671
5672 operand cmpOpEqNe()
5673 %{
5674 match(Bool);
5675 op_cost(0);
5676 predicate(n->as_Bool()->_test._test == BoolTest::ne
5677 || n->as_Bool()->_test._test == BoolTest::eq);
5678
5679 format %{ "" %}
5680 interface(COND_INTER) %{
5681 equal(0x0, "eq");
5682 not_equal(0x1, "ne");
5683 less(0xb, "lt");
5684 greater_equal(0xa, "ge");
5685 less_equal(0xd, "le");
5686 greater(0xc, "gt");
5687 overflow(0x6, "vs");
5688 no_overflow(0x7, "vc");
5689 %}
5690 %}
5691
5692 // used for certain integral comparisons which can be
5693 // converted to cbxx or tbxx instructions
5694
5695 operand cmpOpLtGe()
5696 %{
5697 match(Bool);
5698 op_cost(0);
5699
5700 predicate(n->as_Bool()->_test._test == BoolTest::lt
5701 || n->as_Bool()->_test._test == BoolTest::ge);
5702
5703 format %{ "" %}
5704 interface(COND_INTER) %{
5705 equal(0x0, "eq");
5706 not_equal(0x1, "ne");
5707 less(0xb, "lt");
5708 greater_equal(0xa, "ge");
5709 less_equal(0xd, "le");
5710 greater(0xc, "gt");
5711 overflow(0x6, "vs");
5712 no_overflow(0x7, "vc");
5713 %}
5714 %}
5715
5716 // used for certain unsigned integral comparisons which can be
5717 // converted to cbxx or tbxx instructions
5718
5719 operand cmpOpUEqNeLeGt()
5720 %{
5721 match(Bool);
5722 op_cost(0);
5723
5724 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5725 n->as_Bool()->_test._test == BoolTest::ne ||
5726 n->as_Bool()->_test._test == BoolTest::le ||
5727 n->as_Bool()->_test._test == BoolTest::gt);
5728
5729 format %{ "" %}
5730 interface(COND_INTER) %{
5731 equal(0x0, "eq");
5732 not_equal(0x1, "ne");
5733 less(0x3, "lo");
5734 greater_equal(0x2, "hs");
5735 less_equal(0x9, "ls");
5736 greater(0x8, "hi");
5737 overflow(0x6, "vs");
5738 no_overflow(0x7, "vc");
5739 %}
5740 %}
5741
5742 // Special operand allowing long args to int ops to be truncated for free
5743
5744 operand iRegL2I(iRegL reg) %{
5745
5746 op_cost(0);
5747
5748 match(ConvL2I reg);
5749
5750 format %{ "l2i($reg)" %}
5751
5752 interface(REG_INTER)
5753 %}
5754
5755 operand iRegL2P(iRegL reg) %{
5756
5757 op_cost(0);
5758
5759 match(CastX2P reg);
5760
5761 format %{ "l2p($reg)" %}
5762
5763 interface(REG_INTER)
5764 %}
5765
5766 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5767 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5768 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5769 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5770
5771 //----------OPERAND CLASSES----------------------------------------------------
5772 // Operand Classes are groups of operands that are used as to simplify
5773 // instruction definitions by not requiring the AD writer to specify
5774 // separate instructions for every form of operand when the
5775 // instruction accepts multiple operand types with the same basic
5776 // encoding and format. The classic case of this is memory operands.
5777
5778 // memory is used to define read/write location for load/store
5779 // instruction defs. we can turn a memory op into an Address
5780
5781 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5782 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5783
5784 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5785 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5786
5787 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5788 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5789
5790 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5791 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5792
5793 // All of the memory operands. For the pipeline description.
5794 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5795 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5796 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5797
5798
5799 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5800 // operations. it allows the src to be either an iRegI or a (ConvL2I
5801 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5802 // can be elided because the 32-bit instruction will just employ the
5803 // lower 32 bits anyway.
5804 //
5805 // n.b. this does not elide all L2I conversions. if the truncated
5806 // value is consumed by more than one operation then the ConvL2I
5807 // cannot be bundled into the consuming nodes so an l2i gets planted
5808 // (actually a movw $dst $src) and the downstream instructions consume
5809 // the result of the l2i as an iRegI input. That's a shame since the
5810 // movw is actually redundant but its not too costly.
5811
5812 opclass iRegIorL2I(iRegI, iRegL2I);
5813 opclass iRegPorL2P(iRegP, iRegL2P);
5814
5815 //----------PIPELINE-----------------------------------------------------------
5816 // Rules which define the behavior of the target architectures pipeline.
5817
5818 // For specific pipelines, eg A53, define the stages of that pipeline
5819 //pipe_desc(ISS, EX1, EX2, WR);
5820 #define ISS S0
5821 #define EX1 S1
5822 #define EX2 S2
5823 #define WR S3
5824
5825 // Integer ALU reg operation
5826 pipeline %{
5827
5828 attributes %{
5829 // ARM instructions are of fixed length
5830 fixed_size_instructions; // Fixed size instructions TODO does
5831 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5832 // ARM instructions come in 32-bit word units
5833 instruction_unit_size = 4; // An instruction is 4 bytes long
5834 instruction_fetch_unit_size = 64; // The processor fetches one line
5835 instruction_fetch_units = 1; // of 64 bytes
5836
5837 // List of nop instructions
5838 nops( MachNop );
5839 %}
5840
5841 // We don't use an actual pipeline model so don't care about resources
5842 // or description. we do use pipeline classes to introduce fixed
5843 // latencies
5844
5845 //----------RESOURCES----------------------------------------------------------
5846 // Resources are the functional units available to the machine
5847
5848 resources( INS0, INS1, INS01 = INS0 | INS1,
5849 ALU0, ALU1, ALU = ALU0 | ALU1,
5850 MAC,
5851 DIV,
5852 BRANCH,
5853 LDST,
5854 NEON_FP);
5855
5856 //----------PIPELINE DESCRIPTION-----------------------------------------------
5857 // Pipeline Description specifies the stages in the machine's pipeline
5858
5859 // Define the pipeline as a generic 6 stage pipeline
5860 pipe_desc(S0, S1, S2, S3, S4, S5);
5861
5862 //----------PIPELINE CLASSES---------------------------------------------------
5863 // Pipeline Classes describe the stages in which input and output are
5864 // referenced by the hardware pipeline.
5865
5866 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5867 %{
5868 single_instruction;
5869 src1 : S1(read);
5870 src2 : S2(read);
5871 dst : S5(write);
5872 INS01 : ISS;
5873 NEON_FP : S5;
5874 %}
5875
5876 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5877 %{
5878 single_instruction;
5879 src1 : S1(read);
5880 src2 : S2(read);
5881 dst : S5(write);
5882 INS01 : ISS;
5883 NEON_FP : S5;
5884 %}
5885
5886 pipe_class fp_uop_s(vRegF dst, vRegF src)
5887 %{
5888 single_instruction;
5889 src : S1(read);
5890 dst : S5(write);
5891 INS01 : ISS;
5892 NEON_FP : S5;
5893 %}
5894
5895 pipe_class fp_uop_d(vRegD dst, vRegD src)
5896 %{
5897 single_instruction;
5898 src : S1(read);
5899 dst : S5(write);
5900 INS01 : ISS;
5901 NEON_FP : S5;
5902 %}
5903
5904 pipe_class fp_d2f(vRegF dst, vRegD src)
5905 %{
5906 single_instruction;
5907 src : S1(read);
5908 dst : S5(write);
5909 INS01 : ISS;
5910 NEON_FP : S5;
5911 %}
5912
5913 pipe_class fp_f2d(vRegD dst, vRegF src)
5914 %{
5915 single_instruction;
5916 src : S1(read);
5917 dst : S5(write);
5918 INS01 : ISS;
5919 NEON_FP : S5;
5920 %}
5921
5922 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5923 %{
5924 single_instruction;
5925 src : S1(read);
5926 dst : S5(write);
5927 INS01 : ISS;
5928 NEON_FP : S5;
5929 %}
5930
5931 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5932 %{
5933 single_instruction;
5934 src : S1(read);
5935 dst : S5(write);
5936 INS01 : ISS;
5937 NEON_FP : S5;
5938 %}
5939
5940 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5941 %{
5942 single_instruction;
5943 src : S1(read);
5944 dst : S5(write);
5945 INS01 : ISS;
5946 NEON_FP : S5;
5947 %}
5948
5949 pipe_class fp_l2f(vRegF dst, iRegL src)
5950 %{
5951 single_instruction;
5952 src : S1(read);
5953 dst : S5(write);
5954 INS01 : ISS;
5955 NEON_FP : S5;
5956 %}
5957
5958 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5959 %{
5960 single_instruction;
5961 src : S1(read);
5962 dst : S5(write);
5963 INS01 : ISS;
5964 NEON_FP : S5;
5965 %}
5966
5967 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5968 %{
5969 single_instruction;
5970 src : S1(read);
5971 dst : S5(write);
5972 INS01 : ISS;
5973 NEON_FP : S5;
5974 %}
5975
5976 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5977 %{
5978 single_instruction;
5979 src : S1(read);
5980 dst : S5(write);
5981 INS01 : ISS;
5982 NEON_FP : S5;
5983 %}
5984
5985 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5986 %{
5987 single_instruction;
5988 src : S1(read);
5989 dst : S5(write);
5990 INS01 : ISS;
5991 NEON_FP : S5;
5992 %}
5993
5994 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5995 %{
5996 single_instruction;
5997 src1 : S1(read);
5998 src2 : S2(read);
5999 dst : S5(write);
6000 INS0 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6005 %{
6006 single_instruction;
6007 src1 : S1(read);
6008 src2 : S2(read);
6009 dst : S5(write);
6010 INS0 : ISS;
6011 NEON_FP : S5;
6012 %}
6013
6014 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6015 %{
6016 single_instruction;
6017 cr : S1(read);
6018 src1 : S1(read);
6019 src2 : S1(read);
6020 dst : S3(write);
6021 INS01 : ISS;
6022 NEON_FP : S3;
6023 %}
6024
6025 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6026 %{
6027 single_instruction;
6028 cr : S1(read);
6029 src1 : S1(read);
6030 src2 : S1(read);
6031 dst : S3(write);
6032 INS01 : ISS;
6033 NEON_FP : S3;
6034 %}
6035
6036 pipe_class fp_imm_s(vRegF dst)
6037 %{
6038 single_instruction;
6039 dst : S3(write);
6040 INS01 : ISS;
6041 NEON_FP : S3;
6042 %}
6043
6044 pipe_class fp_imm_d(vRegD dst)
6045 %{
6046 single_instruction;
6047 dst : S3(write);
6048 INS01 : ISS;
6049 NEON_FP : S3;
6050 %}
6051
6052 pipe_class fp_load_constant_s(vRegF dst)
6053 %{
6054 single_instruction;
6055 dst : S4(write);
6056 INS01 : ISS;
6057 NEON_FP : S4;
6058 %}
6059
6060 pipe_class fp_load_constant_d(vRegD dst)
6061 %{
6062 single_instruction;
6063 dst : S4(write);
6064 INS01 : ISS;
6065 NEON_FP : S4;
6066 %}
6067
6068 //------- Integer ALU operations --------------------------
6069
6070 // Integer ALU reg-reg operation
6071 // Operands needed in EX1, result generated in EX2
6072 // Eg. ADD x0, x1, x2
6073 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6074 %{
6075 single_instruction;
6076 dst : EX2(write);
6077 src1 : EX1(read);
6078 src2 : EX1(read);
6079 INS01 : ISS; // Dual issue as instruction 0 or 1
6080 ALU : EX2;
6081 %}
6082
6083 // Integer ALU reg-reg operation with constant shift
6084 // Shifted register must be available in LATE_ISS instead of EX1
6085 // Eg. ADD x0, x1, x2, LSL #2
6086 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6087 %{
6088 single_instruction;
6089 dst : EX2(write);
6090 src1 : EX1(read);
6091 src2 : ISS(read);
6092 INS01 : ISS;
6093 ALU : EX2;
6094 %}
6095
6096 // Integer ALU reg operation with constant shift
6097 // Eg. LSL x0, x1, #shift
6098 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6099 %{
6100 single_instruction;
6101 dst : EX2(write);
6102 src1 : ISS(read);
6103 INS01 : ISS;
6104 ALU : EX2;
6105 %}
6106
6107 // Integer ALU reg-reg operation with variable shift
6108 // Both operands must be available in LATE_ISS instead of EX1
6109 // Result is available in EX1 instead of EX2
6110 // Eg. LSLV x0, x1, x2
6111 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6112 %{
6113 single_instruction;
6114 dst : EX1(write);
6115 src1 : ISS(read);
6116 src2 : ISS(read);
6117 INS01 : ISS;
6118 ALU : EX1;
6119 %}
6120
6121 // Integer ALU reg-reg operation with extract
6122 // As for _vshift above, but result generated in EX2
6123 // Eg. EXTR x0, x1, x2, #N
6124 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6125 %{
6126 single_instruction;
6127 dst : EX2(write);
6128 src1 : ISS(read);
6129 src2 : ISS(read);
6130 INS1 : ISS; // Can only dual issue as Instruction 1
6131 ALU : EX1;
6132 %}
6133
6134 // Integer ALU reg operation
6135 // Eg. NEG x0, x1
6136 pipe_class ialu_reg(iRegI dst, iRegI src)
6137 %{
6138 single_instruction;
6139 dst : EX2(write);
6140 src : EX1(read);
6141 INS01 : ISS;
6142 ALU : EX2;
6143 %}
6144
6145 // Integer ALU reg mmediate operation
6146 // Eg. ADD x0, x1, #N
6147 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6148 %{
6149 single_instruction;
6150 dst : EX2(write);
6151 src1 : EX1(read);
6152 INS01 : ISS;
6153 ALU : EX2;
6154 %}
6155
6156 // Integer ALU immediate operation (no source operands)
6157 // Eg. MOV x0, #N
6158 pipe_class ialu_imm(iRegI dst)
6159 %{
6160 single_instruction;
6161 dst : EX1(write);
6162 INS01 : ISS;
6163 ALU : EX1;
6164 %}
6165
6166 //------- Compare operation -------------------------------
6167
6168 // Compare reg-reg
6169 // Eg. CMP x0, x1
6170 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6171 %{
6172 single_instruction;
6173 // fixed_latency(16);
6174 cr : EX2(write);
6175 op1 : EX1(read);
6176 op2 : EX1(read);
6177 INS01 : ISS;
6178 ALU : EX2;
6179 %}
6180
6181 // Compare reg-reg
6182 // Eg. CMP x0, #N
6183 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6184 %{
6185 single_instruction;
6186 // fixed_latency(16);
6187 cr : EX2(write);
6188 op1 : EX1(read);
6189 INS01 : ISS;
6190 ALU : EX2;
6191 %}
6192
6193 //------- Conditional instructions ------------------------
6194
6195 // Conditional no operands
6196 // Eg. CSINC x0, zr, zr, <cond>
6197 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6198 %{
6199 single_instruction;
6200 cr : EX1(read);
6201 dst : EX2(write);
6202 INS01 : ISS;
6203 ALU : EX2;
6204 %}
6205
6206 // Conditional 2 operand
6207 // EG. CSEL X0, X1, X2, <cond>
6208 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6209 %{
6210 single_instruction;
6211 cr : EX1(read);
6212 src1 : EX1(read);
6213 src2 : EX1(read);
6214 dst : EX2(write);
6215 INS01 : ISS;
6216 ALU : EX2;
6217 %}
6218
6219 // Conditional 2 operand
6220 // EG. CSEL X0, X1, X2, <cond>
6221 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6222 %{
6223 single_instruction;
6224 cr : EX1(read);
6225 src : EX1(read);
6226 dst : EX2(write);
6227 INS01 : ISS;
6228 ALU : EX2;
6229 %}
6230
6231 //------- Multiply pipeline operations --------------------
6232
6233 // Multiply reg-reg
6234 // Eg. MUL w0, w1, w2
6235 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6236 %{
6237 single_instruction;
6238 dst : WR(write);
6239 src1 : ISS(read);
6240 src2 : ISS(read);
6241 INS01 : ISS;
6242 MAC : WR;
6243 %}
6244
6245 // Multiply accumulate
6246 // Eg. MADD w0, w1, w2, w3
6247 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6248 %{
6249 single_instruction;
6250 dst : WR(write);
6251 src1 : ISS(read);
6252 src2 : ISS(read);
6253 src3 : ISS(read);
6254 INS01 : ISS;
6255 MAC : WR;
6256 %}
6257
6258 // Eg. MUL w0, w1, w2
6259 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6260 %{
6261 single_instruction;
6262 fixed_latency(3); // Maximum latency for 64 bit mul
6263 dst : WR(write);
6264 src1 : ISS(read);
6265 src2 : ISS(read);
6266 INS01 : ISS;
6267 MAC : WR;
6268 %}
6269
6270 // Multiply accumulate
6271 // Eg. MADD w0, w1, w2, w3
6272 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6273 %{
6274 single_instruction;
6275 fixed_latency(3); // Maximum latency for 64 bit mul
6276 dst : WR(write);
6277 src1 : ISS(read);
6278 src2 : ISS(read);
6279 src3 : ISS(read);
6280 INS01 : ISS;
6281 MAC : WR;
6282 %}
6283
6284 //------- Divide pipeline operations --------------------
6285
6286 // Eg. SDIV w0, w1, w2
6287 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6288 %{
6289 single_instruction;
6290 fixed_latency(8); // Maximum latency for 32 bit divide
6291 dst : WR(write);
6292 src1 : ISS(read);
6293 src2 : ISS(read);
6294 INS0 : ISS; // Can only dual issue as instruction 0
6295 DIV : WR;
6296 %}
6297
6298 // Eg. SDIV x0, x1, x2
6299 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6300 %{
6301 single_instruction;
6302 fixed_latency(16); // Maximum latency for 64 bit divide
6303 dst : WR(write);
6304 src1 : ISS(read);
6305 src2 : ISS(read);
6306 INS0 : ISS; // Can only dual issue as instruction 0
6307 DIV : WR;
6308 %}
6309
6310 //------- Load pipeline operations ------------------------
6311
6312 // Load - prefetch
6313 // Eg. PFRM <mem>
6314 pipe_class iload_prefetch(memory mem)
6315 %{
6316 single_instruction;
6317 mem : ISS(read);
6318 INS01 : ISS;
6319 LDST : WR;
6320 %}
6321
6322 // Load - reg, mem
6323 // Eg. LDR x0, <mem>
6324 pipe_class iload_reg_mem(iRegI dst, memory mem)
6325 %{
6326 single_instruction;
6327 dst : WR(write);
6328 mem : ISS(read);
6329 INS01 : ISS;
6330 LDST : WR;
6331 %}
6332
6333 // Load - reg, reg
6334 // Eg. LDR x0, [sp, x1]
6335 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6336 %{
6337 single_instruction;
6338 dst : WR(write);
6339 src : ISS(read);
6340 INS01 : ISS;
6341 LDST : WR;
6342 %}
6343
6344 //------- Store pipeline operations -----------------------
6345
6346 // Store - zr, mem
6347 // Eg. STR zr, <mem>
6348 pipe_class istore_mem(memory mem)
6349 %{
6350 single_instruction;
6351 mem : ISS(read);
6352 INS01 : ISS;
6353 LDST : WR;
6354 %}
6355
6356 // Store - reg, mem
6357 // Eg. STR x0, <mem>
6358 pipe_class istore_reg_mem(iRegI src, memory mem)
6359 %{
6360 single_instruction;
6361 mem : ISS(read);
6362 src : EX2(read);
6363 INS01 : ISS;
6364 LDST : WR;
6365 %}
6366
6367 // Store - reg, reg
6368 // Eg. STR x0, [sp, x1]
6369 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6370 %{
6371 single_instruction;
6372 dst : ISS(read);
6373 src : EX2(read);
6374 INS01 : ISS;
6375 LDST : WR;
6376 %}
6377
6378 //------- Store pipeline operations -----------------------
6379
6380 // Branch
6381 pipe_class pipe_branch()
6382 %{
6383 single_instruction;
6384 INS01 : ISS;
6385 BRANCH : EX1;
6386 %}
6387
6388 // Conditional branch
6389 pipe_class pipe_branch_cond(rFlagsReg cr)
6390 %{
6391 single_instruction;
6392 cr : EX1(read);
6393 INS01 : ISS;
6394 BRANCH : EX1;
6395 %}
6396
6397 // Compare & Branch
6398 // EG. CBZ/CBNZ
6399 pipe_class pipe_cmp_branch(iRegI op1)
6400 %{
6401 single_instruction;
6402 op1 : EX1(read);
6403 INS01 : ISS;
6404 BRANCH : EX1;
6405 %}
6406
6407 //------- Synchronisation operations ----------------------
6408
6409 // Any operation requiring serialization.
6410 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6411 pipe_class pipe_serial()
6412 %{
6413 single_instruction;
6414 force_serialization;
6415 fixed_latency(16);
6416 INS01 : ISS(2); // Cannot dual issue with any other instruction
6417 LDST : WR;
6418 %}
6419
6420 // Generic big/slow expanded idiom - also serialized
6421 pipe_class pipe_slow()
6422 %{
6423 instruction_count(10);
6424 multiple_bundles;
6425 force_serialization;
6426 fixed_latency(16);
6427 INS01 : ISS(2); // Cannot dual issue with any other instruction
6428 LDST : WR;
6429 %}
6430
6431 // Empty pipeline class
6432 pipe_class pipe_class_empty()
6433 %{
6434 single_instruction;
6435 fixed_latency(0);
6436 %}
6437
6438 // Default pipeline class.
6439 pipe_class pipe_class_default()
6440 %{
6441 single_instruction;
6442 fixed_latency(2);
6443 %}
6444
6445 // Pipeline class for compares.
6446 pipe_class pipe_class_compare()
6447 %{
6448 single_instruction;
6449 fixed_latency(16);
6450 %}
6451
6452 // Pipeline class for memory operations.
6453 pipe_class pipe_class_memory()
6454 %{
6455 single_instruction;
6456 fixed_latency(16);
6457 %}
6458
6459 // Pipeline class for call.
6460 pipe_class pipe_class_call()
6461 %{
6462 single_instruction;
6463 fixed_latency(100);
6464 %}
6465
6466 // Define the class for the Nop node.
6467 define %{
6468 MachNop = pipe_class_empty;
6469 %}
6470
6471 %}
6472 //----------INSTRUCTIONS-------------------------------------------------------
6473 //
6474 // match -- States which machine-independent subtree may be replaced
6475 // by this instruction.
6476 // ins_cost -- The estimated cost of this instruction is used by instruction
6477 // selection to identify a minimum cost tree of machine
6478 // instructions that matches a tree of machine-independent
6479 // instructions.
6480 // format -- A string providing the disassembly for this instruction.
6481 // The value of an instruction's operand may be inserted
6482 // by referring to it with a '$' prefix.
6483 // opcode -- Three instruction opcodes may be provided. These are referred
6484 // to within an encode class as $primary, $secondary, and $tertiary
6485 // rrspectively. The primary opcode is commonly used to
6486 // indicate the type of machine instruction, while secondary
6487 // and tertiary are often used for prefix options or addressing
6488 // modes.
6489 // ins_encode -- A list of encode classes with parameters. The encode class
6490 // name must have been defined in an 'enc_class' specification
6491 // in the encode section of the architecture description.
6492
6493 // ============================================================================
6494 // Memory (Load/Store) Instructions
6495
6496 // Load Instructions
6497
6498 // Load Byte (8 bit signed)
6499 instruct loadB(iRegINoSp dst, memory1 mem)
6500 %{
6501 match(Set dst (LoadB mem));
6502 predicate(!needs_acquiring_load(n));
6503
6504 ins_cost(4 * INSN_COST);
6505 format %{ "ldrsbw $dst, $mem\t# byte" %}
6506
6507 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6508
6509 ins_pipe(iload_reg_mem);
6510 %}
6511
6512 // Load Byte (8 bit signed) into long
6513 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6514 %{
6515 match(Set dst (ConvI2L (LoadB mem)));
6516 predicate(!needs_acquiring_load(n->in(1)));
6517
6518 ins_cost(4 * INSN_COST);
6519 format %{ "ldrsb $dst, $mem\t# byte" %}
6520
6521 ins_encode(aarch64_enc_ldrsb(dst, mem));
6522
6523 ins_pipe(iload_reg_mem);
6524 %}
6525
6526 // Load Byte (8 bit unsigned)
6527 instruct loadUB(iRegINoSp dst, memory1 mem)
6528 %{
6529 match(Set dst (LoadUB mem));
6530 predicate(!needs_acquiring_load(n));
6531
6532 ins_cost(4 * INSN_COST);
6533 format %{ "ldrbw $dst, $mem\t# byte" %}
6534
6535 ins_encode(aarch64_enc_ldrb(dst, mem));
6536
6537 ins_pipe(iload_reg_mem);
6538 %}
6539
6540 // Load Byte (8 bit unsigned) into long
6541 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6542 %{
6543 match(Set dst (ConvI2L (LoadUB mem)));
6544 predicate(!needs_acquiring_load(n->in(1)));
6545
6546 ins_cost(4 * INSN_COST);
6547 format %{ "ldrb $dst, $mem\t# byte" %}
6548
6549 ins_encode(aarch64_enc_ldrb(dst, mem));
6550
6551 ins_pipe(iload_reg_mem);
6552 %}
6553
6554 // Load Short (16 bit signed)
6555 instruct loadS(iRegINoSp dst, memory2 mem)
6556 %{
6557 match(Set dst (LoadS mem));
6558 predicate(!needs_acquiring_load(n));
6559
6560 ins_cost(4 * INSN_COST);
6561 format %{ "ldrshw $dst, $mem\t# short" %}
6562
6563 ins_encode(aarch64_enc_ldrshw(dst, mem));
6564
6565 ins_pipe(iload_reg_mem);
6566 %}
6567
6568 // Load Short (16 bit signed) into long
6569 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6570 %{
6571 match(Set dst (ConvI2L (LoadS mem)));
6572 predicate(!needs_acquiring_load(n->in(1)));
6573
6574 ins_cost(4 * INSN_COST);
6575 format %{ "ldrsh $dst, $mem\t# short" %}
6576
6577 ins_encode(aarch64_enc_ldrsh(dst, mem));
6578
6579 ins_pipe(iload_reg_mem);
6580 %}
6581
6582 // Load Char (16 bit unsigned)
6583 instruct loadUS(iRegINoSp dst, memory2 mem)
6584 %{
6585 match(Set dst (LoadUS mem));
6586 predicate(!needs_acquiring_load(n));
6587
6588 ins_cost(4 * INSN_COST);
6589 format %{ "ldrh $dst, $mem\t# short" %}
6590
6591 ins_encode(aarch64_enc_ldrh(dst, mem));
6592
6593 ins_pipe(iload_reg_mem);
6594 %}
6595
6596 // Load Short/Char (16 bit unsigned) into long
6597 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6598 %{
6599 match(Set dst (ConvI2L (LoadUS mem)));
6600 predicate(!needs_acquiring_load(n->in(1)));
6601
6602 ins_cost(4 * INSN_COST);
6603 format %{ "ldrh $dst, $mem\t# short" %}
6604
6605 ins_encode(aarch64_enc_ldrh(dst, mem));
6606
6607 ins_pipe(iload_reg_mem);
6608 %}
6609
6610 // Load Integer (32 bit signed)
6611 instruct loadI(iRegINoSp dst, memory4 mem)
6612 %{
6613 match(Set dst (LoadI mem));
6614 predicate(!needs_acquiring_load(n));
6615
6616 ins_cost(4 * INSN_COST);
6617 format %{ "ldrw $dst, $mem\t# int" %}
6618
6619 ins_encode(aarch64_enc_ldrw(dst, mem));
6620
6621 ins_pipe(iload_reg_mem);
6622 %}
6623
6624 // Load Integer (32 bit signed) into long
6625 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6626 %{
6627 match(Set dst (ConvI2L (LoadI mem)));
6628 predicate(!needs_acquiring_load(n->in(1)));
6629
6630 ins_cost(4 * INSN_COST);
6631 format %{ "ldrsw $dst, $mem\t# int" %}
6632
6633 ins_encode(aarch64_enc_ldrsw(dst, mem));
6634
6635 ins_pipe(iload_reg_mem);
6636 %}
6637
6638 // Load Integer (32 bit unsigned) into long
6639 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6640 %{
6641 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6642 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6643
6644 ins_cost(4 * INSN_COST);
6645 format %{ "ldrw $dst, $mem\t# int" %}
6646
6647 ins_encode(aarch64_enc_ldrw(dst, mem));
6648
6649 ins_pipe(iload_reg_mem);
6650 %}
6651
6652 // Load Long (64 bit signed)
6653 instruct loadL(iRegLNoSp dst, memory8 mem)
6654 %{
6655 match(Set dst (LoadL mem));
6656 predicate(!needs_acquiring_load(n));
6657
6658 ins_cost(4 * INSN_COST);
6659 format %{ "ldr $dst, $mem\t# int" %}
6660
6661 ins_encode(aarch64_enc_ldr(dst, mem));
6662
6663 ins_pipe(iload_reg_mem);
6664 %}
6665
6666 // Load Range
6667 instruct loadRange(iRegINoSp dst, memory4 mem)
6668 %{
6669 match(Set dst (LoadRange mem));
6670
6671 ins_cost(4 * INSN_COST);
6672 format %{ "ldrw $dst, $mem\t# range" %}
6673
6674 ins_encode(aarch64_enc_ldrw(dst, mem));
6675
6676 ins_pipe(iload_reg_mem);
6677 %}
6678
6679 // Load Pointer
6680 instruct loadP(iRegPNoSp dst, memory8 mem)
6681 %{
6682 match(Set dst (LoadP mem));
6683 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6684
6685 ins_cost(4 * INSN_COST);
6686 format %{ "ldr $dst, $mem\t# ptr" %}
6687
6688 ins_encode(aarch64_enc_ldr(dst, mem));
6689
6690 ins_pipe(iload_reg_mem);
6691 %}
6692
6693 // Load Compressed Pointer
6694 instruct loadN(iRegNNoSp dst, memory4 mem)
6695 %{
6696 match(Set dst (LoadN mem));
6697 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6698
6699 ins_cost(4 * INSN_COST);
6700 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6701
6702 ins_encode(aarch64_enc_ldrw(dst, mem));
6703
6704 ins_pipe(iload_reg_mem);
6705 %}
6706
6707 // Load Klass Pointer
6708 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6709 %{
6710 match(Set dst (LoadKlass mem));
6711 predicate(!needs_acquiring_load(n));
6712
6713 ins_cost(4 * INSN_COST);
6714 format %{ "ldr $dst, $mem\t# class" %}
6715
6716 ins_encode(aarch64_enc_ldr(dst, mem));
6717
6718 ins_pipe(iload_reg_mem);
6719 %}
6720
6721 // Load Narrow Klass Pointer
6722 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6723 %{
6724 match(Set dst (LoadNKlass mem));
6725 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6726
6727 ins_cost(4 * INSN_COST);
6728 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6729
6730 ins_encode(aarch64_enc_ldrw(dst, mem));
6731
6732 ins_pipe(iload_reg_mem);
6733 %}
6734
6735 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6736 %{
6737 match(Set dst (LoadNKlass mem));
6738 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6739
6740 ins_cost(4 * INSN_COST);
6741 format %{
6742 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6743 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6744 %}
6745 ins_encode %{
6746 // inlined aarch64_enc_ldrw
6747 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6748 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6749 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6750 %}
6751 ins_pipe(iload_reg_mem);
6752 %}
6753
6754 // Load Float
6755 instruct loadF(vRegF dst, memory4 mem)
6756 %{
6757 match(Set dst (LoadF mem));
6758 predicate(!needs_acquiring_load(n));
6759
6760 ins_cost(4 * INSN_COST);
6761 format %{ "ldrs $dst, $mem\t# float" %}
6762
6763 ins_encode( aarch64_enc_ldrs(dst, mem) );
6764
6765 ins_pipe(pipe_class_memory);
6766 %}
6767
6768 // Load Double
6769 instruct loadD(vRegD dst, memory8 mem)
6770 %{
6771 match(Set dst (LoadD mem));
6772 predicate(!needs_acquiring_load(n));
6773
6774 ins_cost(4 * INSN_COST);
6775 format %{ "ldrd $dst, $mem\t# double" %}
6776
6777 ins_encode( aarch64_enc_ldrd(dst, mem) );
6778
6779 ins_pipe(pipe_class_memory);
6780 %}
6781
6782
6783 // Load Int Constant
6784 instruct loadConI(iRegINoSp dst, immI src)
6785 %{
6786 match(Set dst src);
6787
6788 ins_cost(INSN_COST);
6789 format %{ "mov $dst, $src\t# int" %}
6790
6791 ins_encode( aarch64_enc_movw_imm(dst, src) );
6792
6793 ins_pipe(ialu_imm);
6794 %}
6795
6796 // Load Long Constant
6797 instruct loadConL(iRegLNoSp dst, immL src)
6798 %{
6799 match(Set dst src);
6800
6801 ins_cost(INSN_COST);
6802 format %{ "mov $dst, $src\t# long" %}
6803
6804 ins_encode( aarch64_enc_mov_imm(dst, src) );
6805
6806 ins_pipe(ialu_imm);
6807 %}
6808
6809 // Load Pointer Constant
6810
6811 instruct loadConP(iRegPNoSp dst, immP con)
6812 %{
6813 match(Set dst con);
6814
6815 ins_cost(INSN_COST * 4);
6816 format %{
6817 "mov $dst, $con\t# ptr"
6818 %}
6819
6820 ins_encode(aarch64_enc_mov_p(dst, con));
6821
6822 ins_pipe(ialu_imm);
6823 %}
6824
6825 // Load Null Pointer Constant
6826
6827 instruct loadConP0(iRegPNoSp dst, immP0 con)
6828 %{
6829 match(Set dst con);
6830
6831 ins_cost(INSN_COST);
6832 format %{ "mov $dst, $con\t# nullptr ptr" %}
6833
6834 ins_encode(aarch64_enc_mov_p0(dst, con));
6835
6836 ins_pipe(ialu_imm);
6837 %}
6838
6839 // Load Pointer Constant One
6840
6841 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6842 %{
6843 match(Set dst con);
6844
6845 ins_cost(INSN_COST);
6846 format %{ "mov $dst, $con\t# nullptr ptr" %}
6847
6848 ins_encode(aarch64_enc_mov_p1(dst, con));
6849
6850 ins_pipe(ialu_imm);
6851 %}
6852
6853 // Load Byte Map Base Constant
6854
6855 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6856 %{
6857 match(Set dst con);
6858
6859 ins_cost(INSN_COST);
6860 format %{ "adr $dst, $con\t# Byte Map Base" %}
6861
6862 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6863
6864 ins_pipe(ialu_imm);
6865 %}
6866
6867 // Load Narrow Pointer Constant
6868
6869 instruct loadConN(iRegNNoSp dst, immN con)
6870 %{
6871 match(Set dst con);
6872
6873 ins_cost(INSN_COST * 4);
6874 format %{ "mov $dst, $con\t# compressed ptr" %}
6875
6876 ins_encode(aarch64_enc_mov_n(dst, con));
6877
6878 ins_pipe(ialu_imm);
6879 %}
6880
6881 // Load Narrow Null Pointer Constant
6882
6883 instruct loadConN0(iRegNNoSp dst, immN0 con)
6884 %{
6885 match(Set dst con);
6886
6887 ins_cost(INSN_COST);
6888 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6889
6890 ins_encode(aarch64_enc_mov_n0(dst, con));
6891
6892 ins_pipe(ialu_imm);
6893 %}
6894
6895 // Load Narrow Klass Constant
6896
6897 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6898 %{
6899 match(Set dst con);
6900
6901 ins_cost(INSN_COST);
6902 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6903
6904 ins_encode(aarch64_enc_mov_nk(dst, con));
6905
6906 ins_pipe(ialu_imm);
6907 %}
6908
6909 // Load Packed Float Constant
6910
6911 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6912 match(Set dst con);
6913 ins_cost(INSN_COST * 4);
6914 format %{ "fmovs $dst, $con"%}
6915 ins_encode %{
6916 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6917 %}
6918
6919 ins_pipe(fp_imm_s);
6920 %}
6921
6922 // Load Float Constant
6923
6924 instruct loadConF(vRegF dst, immF con) %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST * 4);
6928
6929 format %{
6930 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6931 %}
6932
6933 ins_encode %{
6934 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6935 %}
6936
6937 ins_pipe(fp_load_constant_s);
6938 %}
6939
6940 // Load Packed Double Constant
6941
6942 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6943 match(Set dst con);
6944 ins_cost(INSN_COST);
6945 format %{ "fmovd $dst, $con"%}
6946 ins_encode %{
6947 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6948 %}
6949
6950 ins_pipe(fp_imm_d);
6951 %}
6952
6953 // Load Double Constant
6954
6955 instruct loadConD(vRegD dst, immD con) %{
6956 match(Set dst con);
6957
6958 ins_cost(INSN_COST * 5);
6959 format %{
6960 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6961 %}
6962
6963 ins_encode %{
6964 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6965 %}
6966
6967 ins_pipe(fp_load_constant_d);
6968 %}
6969
6970 // Store Instructions
6971
6972 // Store Byte
6973 instruct storeB(iRegIorL2I src, memory1 mem)
6974 %{
6975 match(Set mem (StoreB mem src));
6976 predicate(!needs_releasing_store(n));
6977
6978 ins_cost(INSN_COST);
6979 format %{ "strb $src, $mem\t# byte" %}
6980
6981 ins_encode(aarch64_enc_strb(src, mem));
6982
6983 ins_pipe(istore_reg_mem);
6984 %}
6985
6986
6987 instruct storeimmB0(immI0 zero, memory1 mem)
6988 %{
6989 match(Set mem (StoreB mem zero));
6990 predicate(!needs_releasing_store(n));
6991
6992 ins_cost(INSN_COST);
6993 format %{ "strb rscractch2, $mem\t# byte" %}
6994
6995 ins_encode(aarch64_enc_strb0(mem));
6996
6997 ins_pipe(istore_mem);
6998 %}
6999
7000 // Store Char/Short
7001 instruct storeC(iRegIorL2I src, memory2 mem)
7002 %{
7003 match(Set mem (StoreC mem src));
7004 predicate(!needs_releasing_store(n));
7005
7006 ins_cost(INSN_COST);
7007 format %{ "strh $src, $mem\t# short" %}
7008
7009 ins_encode(aarch64_enc_strh(src, mem));
7010
7011 ins_pipe(istore_reg_mem);
7012 %}
7013
7014 instruct storeimmC0(immI0 zero, memory2 mem)
7015 %{
7016 match(Set mem (StoreC mem zero));
7017 predicate(!needs_releasing_store(n));
7018
7019 ins_cost(INSN_COST);
7020 format %{ "strh zr, $mem\t# short" %}
7021
7022 ins_encode(aarch64_enc_strh0(mem));
7023
7024 ins_pipe(istore_mem);
7025 %}
7026
7027 // Store Integer
7028
7029 instruct storeI(iRegIorL2I src, memory4 mem)
7030 %{
7031 match(Set mem(StoreI mem src));
7032 predicate(!needs_releasing_store(n));
7033
7034 ins_cost(INSN_COST);
7035 format %{ "strw $src, $mem\t# int" %}
7036
7037 ins_encode(aarch64_enc_strw(src, mem));
7038
7039 ins_pipe(istore_reg_mem);
7040 %}
7041
7042 instruct storeimmI0(immI0 zero, memory4 mem)
7043 %{
7044 match(Set mem(StoreI mem zero));
7045 predicate(!needs_releasing_store(n));
7046
7047 ins_cost(INSN_COST);
7048 format %{ "strw zr, $mem\t# int" %}
7049
7050 ins_encode(aarch64_enc_strw0(mem));
7051
7052 ins_pipe(istore_mem);
7053 %}
7054
7055 // Store Long (64 bit signed)
7056 instruct storeL(iRegL src, memory8 mem)
7057 %{
7058 match(Set mem (StoreL mem src));
7059 predicate(!needs_releasing_store(n));
7060
7061 ins_cost(INSN_COST);
7062 format %{ "str $src, $mem\t# int" %}
7063
7064 ins_encode(aarch64_enc_str(src, mem));
7065
7066 ins_pipe(istore_reg_mem);
7067 %}
7068
7069 // Store Long (64 bit signed)
7070 instruct storeimmL0(immL0 zero, memory8 mem)
7071 %{
7072 match(Set mem (StoreL mem zero));
7073 predicate(!needs_releasing_store(n));
7074
7075 ins_cost(INSN_COST);
7076 format %{ "str zr, $mem\t# int" %}
7077
7078 ins_encode(aarch64_enc_str0(mem));
7079
7080 ins_pipe(istore_mem);
7081 %}
7082
7083 // Store Pointer
7084 instruct storeP(iRegP src, memory8 mem)
7085 %{
7086 match(Set mem (StoreP mem src));
7087 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7088
7089 ins_cost(INSN_COST);
7090 format %{ "str $src, $mem\t# ptr" %}
7091
7092 ins_encode(aarch64_enc_str(src, mem));
7093
7094 ins_pipe(istore_reg_mem);
7095 %}
7096
7097 // Store Pointer
7098 instruct storeimmP0(immP0 zero, memory8 mem)
7099 %{
7100 match(Set mem (StoreP mem zero));
7101 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7102
7103 ins_cost(INSN_COST);
7104 format %{ "str zr, $mem\t# ptr" %}
7105
7106 ins_encode(aarch64_enc_str0(mem));
7107
7108 ins_pipe(istore_mem);
7109 %}
7110
7111 // Store Compressed Pointer
7112 instruct storeN(iRegN src, memory4 mem)
7113 %{
7114 match(Set mem (StoreN mem src));
7115 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7116
7117 ins_cost(INSN_COST);
7118 format %{ "strw $src, $mem\t# compressed ptr" %}
7119
7120 ins_encode(aarch64_enc_strw(src, mem));
7121
7122 ins_pipe(istore_reg_mem);
7123 %}
7124
7125 instruct storeImmN0(immN0 zero, memory4 mem)
7126 %{
7127 match(Set mem (StoreN mem zero));
7128 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7129
7130 ins_cost(INSN_COST);
7131 format %{ "strw zr, $mem\t# compressed ptr" %}
7132
7133 ins_encode(aarch64_enc_strw0(mem));
7134
7135 ins_pipe(istore_mem);
7136 %}
7137
7138 // Store Float
7139 instruct storeF(vRegF src, memory4 mem)
7140 %{
7141 match(Set mem (StoreF mem src));
7142 predicate(!needs_releasing_store(n));
7143
7144 ins_cost(INSN_COST);
7145 format %{ "strs $src, $mem\t# float" %}
7146
7147 ins_encode( aarch64_enc_strs(src, mem) );
7148
7149 ins_pipe(pipe_class_memory);
7150 %}
7151
7152 // TODO
7153 // implement storeImmF0 and storeFImmPacked
7154
7155 // Store Double
7156 instruct storeD(vRegD src, memory8 mem)
7157 %{
7158 match(Set mem (StoreD mem src));
7159 predicate(!needs_releasing_store(n));
7160
7161 ins_cost(INSN_COST);
7162 format %{ "strd $src, $mem\t# double" %}
7163
7164 ins_encode( aarch64_enc_strd(src, mem) );
7165
7166 ins_pipe(pipe_class_memory);
7167 %}
7168
7169 // Store Compressed Klass Pointer
7170 instruct storeNKlass(iRegN src, memory4 mem)
7171 %{
7172 predicate(!needs_releasing_store(n));
7173 match(Set mem (StoreNKlass mem src));
7174
7175 ins_cost(INSN_COST);
7176 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7177
7178 ins_encode(aarch64_enc_strw(src, mem));
7179
7180 ins_pipe(istore_reg_mem);
7181 %}
7182
7183 // TODO
7184 // implement storeImmD0 and storeDImmPacked
7185
7186 // prefetch instructions
7187 // Must be safe to execute with invalid address (cannot fault).
7188
7189 instruct prefetchalloc( memory8 mem ) %{
7190 match(PrefetchAllocation mem);
7191
7192 ins_cost(INSN_COST);
7193 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7194
7195 ins_encode( aarch64_enc_prefetchw(mem) );
7196
7197 ins_pipe(iload_prefetch);
7198 %}
7199
7200 // ---------------- volatile loads and stores ----------------
7201
7202 // Load Byte (8 bit signed)
7203 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7204 %{
7205 match(Set dst (LoadB mem));
7206
7207 ins_cost(VOLATILE_REF_COST);
7208 format %{ "ldarsb $dst, $mem\t# byte" %}
7209
7210 ins_encode(aarch64_enc_ldarsb(dst, mem));
7211
7212 ins_pipe(pipe_serial);
7213 %}
7214
7215 // Load Byte (8 bit signed) into long
7216 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7217 %{
7218 match(Set dst (ConvI2L (LoadB mem)));
7219
7220 ins_cost(VOLATILE_REF_COST);
7221 format %{ "ldarsb $dst, $mem\t# byte" %}
7222
7223 ins_encode(aarch64_enc_ldarsb(dst, mem));
7224
7225 ins_pipe(pipe_serial);
7226 %}
7227
7228 // Load Byte (8 bit unsigned)
7229 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7230 %{
7231 match(Set dst (LoadUB mem));
7232
7233 ins_cost(VOLATILE_REF_COST);
7234 format %{ "ldarb $dst, $mem\t# byte" %}
7235
7236 ins_encode(aarch64_enc_ldarb(dst, mem));
7237
7238 ins_pipe(pipe_serial);
7239 %}
7240
7241 // Load Byte (8 bit unsigned) into long
7242 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7243 %{
7244 match(Set dst (ConvI2L (LoadUB mem)));
7245
7246 ins_cost(VOLATILE_REF_COST);
7247 format %{ "ldarb $dst, $mem\t# byte" %}
7248
7249 ins_encode(aarch64_enc_ldarb(dst, mem));
7250
7251 ins_pipe(pipe_serial);
7252 %}
7253
7254 // Load Short (16 bit signed)
7255 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7256 %{
7257 match(Set dst (LoadS mem));
7258
7259 ins_cost(VOLATILE_REF_COST);
7260 format %{ "ldarshw $dst, $mem\t# short" %}
7261
7262 ins_encode(aarch64_enc_ldarshw(dst, mem));
7263
7264 ins_pipe(pipe_serial);
7265 %}
7266
7267 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7268 %{
7269 match(Set dst (LoadUS mem));
7270
7271 ins_cost(VOLATILE_REF_COST);
7272 format %{ "ldarhw $dst, $mem\t# short" %}
7273
7274 ins_encode(aarch64_enc_ldarhw(dst, mem));
7275
7276 ins_pipe(pipe_serial);
7277 %}
7278
7279 // Load Short/Char (16 bit unsigned) into long
7280 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7281 %{
7282 match(Set dst (ConvI2L (LoadUS mem)));
7283
7284 ins_cost(VOLATILE_REF_COST);
7285 format %{ "ldarh $dst, $mem\t# short" %}
7286
7287 ins_encode(aarch64_enc_ldarh(dst, mem));
7288
7289 ins_pipe(pipe_serial);
7290 %}
7291
7292 // Load Short/Char (16 bit signed) into long
7293 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7294 %{
7295 match(Set dst (ConvI2L (LoadS mem)));
7296
7297 ins_cost(VOLATILE_REF_COST);
7298 format %{ "ldarh $dst, $mem\t# short" %}
7299
7300 ins_encode(aarch64_enc_ldarsh(dst, mem));
7301
7302 ins_pipe(pipe_serial);
7303 %}
7304
7305 // Load Integer (32 bit signed)
7306 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7307 %{
7308 match(Set dst (LoadI mem));
7309
7310 ins_cost(VOLATILE_REF_COST);
7311 format %{ "ldarw $dst, $mem\t# int" %}
7312
7313 ins_encode(aarch64_enc_ldarw(dst, mem));
7314
7315 ins_pipe(pipe_serial);
7316 %}
7317
7318 // Load Integer (32 bit unsigned) into long
7319 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7320 %{
7321 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7322
7323 ins_cost(VOLATILE_REF_COST);
7324 format %{ "ldarw $dst, $mem\t# int" %}
7325
7326 ins_encode(aarch64_enc_ldarw(dst, mem));
7327
7328 ins_pipe(pipe_serial);
7329 %}
7330
7331 // Load Long (64 bit signed)
7332 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7333 %{
7334 match(Set dst (LoadL mem));
7335
7336 ins_cost(VOLATILE_REF_COST);
7337 format %{ "ldar $dst, $mem\t# int" %}
7338
7339 ins_encode(aarch64_enc_ldar(dst, mem));
7340
7341 ins_pipe(pipe_serial);
7342 %}
7343
7344 // Load Pointer
7345 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7346 %{
7347 match(Set dst (LoadP mem));
7348 predicate(n->as_Load()->barrier_data() == 0);
7349
7350 ins_cost(VOLATILE_REF_COST);
7351 format %{ "ldar $dst, $mem\t# ptr" %}
7352
7353 ins_encode(aarch64_enc_ldar(dst, mem));
7354
7355 ins_pipe(pipe_serial);
7356 %}
7357
7358 // Load Compressed Pointer
7359 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7360 %{
7361 match(Set dst (LoadN mem));
7362 predicate(n->as_Load()->barrier_data() == 0);
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7366
7367 ins_encode(aarch64_enc_ldarw(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Float
7373 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (LoadF mem));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldars $dst, $mem\t# float" %}
7379
7380 ins_encode( aarch64_enc_fldars(dst, mem) );
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 // Load Double
7386 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set dst (LoadD mem));
7389
7390 ins_cost(VOLATILE_REF_COST);
7391 format %{ "ldard $dst, $mem\t# double" %}
7392
7393 ins_encode( aarch64_enc_fldard(dst, mem) );
7394
7395 ins_pipe(pipe_serial);
7396 %}
7397
7398 // Store Byte
7399 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7400 %{
7401 match(Set mem (StoreB mem src));
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "stlrb $src, $mem\t# byte" %}
7405
7406 ins_encode(aarch64_enc_stlrb(src, mem));
7407
7408 ins_pipe(pipe_class_memory);
7409 %}
7410
7411 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7412 %{
7413 match(Set mem (StoreB mem zero));
7414
7415 ins_cost(VOLATILE_REF_COST);
7416 format %{ "stlrb zr, $mem\t# byte" %}
7417
7418 ins_encode(aarch64_enc_stlrb0(mem));
7419
7420 ins_pipe(pipe_class_memory);
7421 %}
7422
7423 // Store Char/Short
7424 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7425 %{
7426 match(Set mem (StoreC mem src));
7427
7428 ins_cost(VOLATILE_REF_COST);
7429 format %{ "stlrh $src, $mem\t# short" %}
7430
7431 ins_encode(aarch64_enc_stlrh(src, mem));
7432
7433 ins_pipe(pipe_class_memory);
7434 %}
7435
7436 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7437 %{
7438 match(Set mem (StoreC mem zero));
7439
7440 ins_cost(VOLATILE_REF_COST);
7441 format %{ "stlrh zr, $mem\t# short" %}
7442
7443 ins_encode(aarch64_enc_stlrh0(mem));
7444
7445 ins_pipe(pipe_class_memory);
7446 %}
7447
7448 // Store Integer
7449
7450 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7451 %{
7452 match(Set mem(StoreI mem src));
7453
7454 ins_cost(VOLATILE_REF_COST);
7455 format %{ "stlrw $src, $mem\t# int" %}
7456
7457 ins_encode(aarch64_enc_stlrw(src, mem));
7458
7459 ins_pipe(pipe_class_memory);
7460 %}
7461
7462 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7463 %{
7464 match(Set mem(StoreI mem zero));
7465
7466 ins_cost(VOLATILE_REF_COST);
7467 format %{ "stlrw zr, $mem\t# int" %}
7468
7469 ins_encode(aarch64_enc_stlrw0(mem));
7470
7471 ins_pipe(pipe_class_memory);
7472 %}
7473
7474 // Store Long (64 bit signed)
7475 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7476 %{
7477 match(Set mem (StoreL mem src));
7478
7479 ins_cost(VOLATILE_REF_COST);
7480 format %{ "stlr $src, $mem\t# int" %}
7481
7482 ins_encode(aarch64_enc_stlr(src, mem));
7483
7484 ins_pipe(pipe_class_memory);
7485 %}
7486
7487 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7488 %{
7489 match(Set mem (StoreL mem zero));
7490
7491 ins_cost(VOLATILE_REF_COST);
7492 format %{ "stlr zr, $mem\t# int" %}
7493
7494 ins_encode(aarch64_enc_stlr0(mem));
7495
7496 ins_pipe(pipe_class_memory);
7497 %}
7498
7499 // Store Pointer
7500 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7501 %{
7502 match(Set mem (StoreP mem src));
7503 predicate(n->as_Store()->barrier_data() == 0);
7504
7505 ins_cost(VOLATILE_REF_COST);
7506 format %{ "stlr $src, $mem\t# ptr" %}
7507
7508 ins_encode(aarch64_enc_stlr(src, mem));
7509
7510 ins_pipe(pipe_class_memory);
7511 %}
7512
7513 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7514 %{
7515 match(Set mem (StoreP mem zero));
7516 predicate(n->as_Store()->barrier_data() == 0);
7517
7518 ins_cost(VOLATILE_REF_COST);
7519 format %{ "stlr zr, $mem\t# ptr" %}
7520
7521 ins_encode(aarch64_enc_stlr0(mem));
7522
7523 ins_pipe(pipe_class_memory);
7524 %}
7525
7526 // Store Compressed Pointer
7527 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7528 %{
7529 match(Set mem (StoreN mem src));
7530 predicate(n->as_Store()->barrier_data() == 0);
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7534
7535 ins_encode(aarch64_enc_stlrw(src, mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7541 %{
7542 match(Set mem (StoreN mem zero));
7543 predicate(n->as_Store()->barrier_data() == 0);
7544
7545 ins_cost(VOLATILE_REF_COST);
7546 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7547
7548 ins_encode(aarch64_enc_stlrw0(mem));
7549
7550 ins_pipe(pipe_class_memory);
7551 %}
7552
7553 // Store Float
7554 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreF mem src));
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlrs $src, $mem\t# float" %}
7560
7561 ins_encode( aarch64_enc_fstlrs(src, mem) );
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 // TODO
7567 // implement storeImmF0 and storeFImmPacked
7568
7569 // Store Double
7570 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7571 %{
7572 match(Set mem (StoreD mem src));
7573
7574 ins_cost(VOLATILE_REF_COST);
7575 format %{ "stlrd $src, $mem\t# double" %}
7576
7577 ins_encode( aarch64_enc_fstlrd(src, mem) );
7578
7579 ins_pipe(pipe_class_memory);
7580 %}
7581
7582 // ---------------- end of volatile loads and stores ----------------
7583
7584 instruct cacheWB(indirect addr)
7585 %{
7586 predicate(VM_Version::supports_data_cache_line_flush());
7587 match(CacheWB addr);
7588
7589 ins_cost(100);
7590 format %{"cache wb $addr" %}
7591 ins_encode %{
7592 assert($addr->index_position() < 0, "should be");
7593 assert($addr$$disp == 0, "should be");
7594 __ cache_wb(Address($addr$$base$$Register, 0));
7595 %}
7596 ins_pipe(pipe_slow); // XXX
7597 %}
7598
7599 instruct cacheWBPreSync()
7600 %{
7601 predicate(VM_Version::supports_data_cache_line_flush());
7602 match(CacheWBPreSync);
7603
7604 ins_cost(100);
7605 format %{"cache wb presync" %}
7606 ins_encode %{
7607 __ cache_wbsync(true);
7608 %}
7609 ins_pipe(pipe_slow); // XXX
7610 %}
7611
7612 instruct cacheWBPostSync()
7613 %{
7614 predicate(VM_Version::supports_data_cache_line_flush());
7615 match(CacheWBPostSync);
7616
7617 ins_cost(100);
7618 format %{"cache wb postsync" %}
7619 ins_encode %{
7620 __ cache_wbsync(false);
7621 %}
7622 ins_pipe(pipe_slow); // XXX
7623 %}
7624
7625 // ============================================================================
7626 // BSWAP Instructions
7627
7628 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7629 match(Set dst (ReverseBytesI src));
7630
7631 ins_cost(INSN_COST);
7632 format %{ "revw $dst, $src" %}
7633
7634 ins_encode %{
7635 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7636 %}
7637
7638 ins_pipe(ialu_reg);
7639 %}
7640
7641 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7642 match(Set dst (ReverseBytesL src));
7643
7644 ins_cost(INSN_COST);
7645 format %{ "rev $dst, $src" %}
7646
7647 ins_encode %{
7648 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7649 %}
7650
7651 ins_pipe(ialu_reg);
7652 %}
7653
7654 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7655 match(Set dst (ReverseBytesUS src));
7656
7657 ins_cost(INSN_COST);
7658 format %{ "rev16w $dst, $src" %}
7659
7660 ins_encode %{
7661 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7662 %}
7663
7664 ins_pipe(ialu_reg);
7665 %}
7666
7667 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7668 match(Set dst (ReverseBytesS src));
7669
7670 ins_cost(INSN_COST);
7671 format %{ "rev16w $dst, $src\n\t"
7672 "sbfmw $dst, $dst, #0, #15" %}
7673
7674 ins_encode %{
7675 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7676 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7677 %}
7678
7679 ins_pipe(ialu_reg);
7680 %}
7681
7682 // ============================================================================
7683 // Zero Count Instructions
7684
7685 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7686 match(Set dst (CountLeadingZerosI src));
7687
7688 ins_cost(INSN_COST);
7689 format %{ "clzw $dst, $src" %}
7690 ins_encode %{
7691 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7692 %}
7693
7694 ins_pipe(ialu_reg);
7695 %}
7696
7697 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7698 match(Set dst (CountLeadingZerosL src));
7699
7700 ins_cost(INSN_COST);
7701 format %{ "clz $dst, $src" %}
7702 ins_encode %{
7703 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7704 %}
7705
7706 ins_pipe(ialu_reg);
7707 %}
7708
7709 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7710 match(Set dst (CountTrailingZerosI src));
7711
7712 ins_cost(INSN_COST * 2);
7713 format %{ "rbitw $dst, $src\n\t"
7714 "clzw $dst, $dst" %}
7715 ins_encode %{
7716 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7717 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7718 %}
7719
7720 ins_pipe(ialu_reg);
7721 %}
7722
7723 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7724 match(Set dst (CountTrailingZerosL src));
7725
7726 ins_cost(INSN_COST * 2);
7727 format %{ "rbit $dst, $src\n\t"
7728 "clz $dst, $dst" %}
7729 ins_encode %{
7730 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7731 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7732 %}
7733
7734 ins_pipe(ialu_reg);
7735 %}
7736
7737 //---------- Population Count Instructions -------------------------------------
7738 //
7739
7740 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7741 match(Set dst (PopCountI src));
7742 effect(TEMP tmp);
7743 ins_cost(INSN_COST * 13);
7744
7745 format %{ "movw $src, $src\n\t"
7746 "mov $tmp, $src\t# vector (1D)\n\t"
7747 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7748 "addv $tmp, $tmp\t# vector (8B)\n\t"
7749 "mov $dst, $tmp\t# vector (1D)" %}
7750 ins_encode %{
7751 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7752 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7753 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7754 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7755 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7756 %}
7757
7758 ins_pipe(pipe_class_default);
7759 %}
7760
7761 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7762 match(Set dst (PopCountI (LoadI mem)));
7763 effect(TEMP tmp);
7764 ins_cost(INSN_COST * 13);
7765
7766 format %{ "ldrs $tmp, $mem\n\t"
7767 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7768 "addv $tmp, $tmp\t# vector (8B)\n\t"
7769 "mov $dst, $tmp\t# vector (1D)" %}
7770 ins_encode %{
7771 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7772 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7773 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7774 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7775 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7776 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7777 %}
7778
7779 ins_pipe(pipe_class_default);
7780 %}
7781
7782 // Note: Long.bitCount(long) returns an int.
7783 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7784 match(Set dst (PopCountL src));
7785 effect(TEMP tmp);
7786 ins_cost(INSN_COST * 13);
7787
7788 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7789 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7790 "addv $tmp, $tmp\t# vector (8B)\n\t"
7791 "mov $dst, $tmp\t# vector (1D)" %}
7792 ins_encode %{
7793 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7794 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7795 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7796 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7797 %}
7798
7799 ins_pipe(pipe_class_default);
7800 %}
7801
7802 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7803 match(Set dst (PopCountL (LoadL mem)));
7804 effect(TEMP tmp);
7805 ins_cost(INSN_COST * 13);
7806
7807 format %{ "ldrd $tmp, $mem\n\t"
7808 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7809 "addv $tmp, $tmp\t# vector (8B)\n\t"
7810 "mov $dst, $tmp\t# vector (1D)" %}
7811 ins_encode %{
7812 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7813 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7814 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7815 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7816 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7817 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7818 %}
7819
7820 ins_pipe(pipe_class_default);
7821 %}
7822
7823 // ============================================================================
7824 // VerifyVectorAlignment Instruction
7825
7826 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7827 match(Set addr (VerifyVectorAlignment addr mask));
7828 effect(KILL cr);
7829 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7830 ins_encode %{
7831 Label Lskip;
7832 // check if masked bits of addr are zero
7833 __ tst($addr$$Register, $mask$$constant);
7834 __ br(Assembler::EQ, Lskip);
7835 __ stop("verify_vector_alignment found a misaligned vector memory access");
7836 __ bind(Lskip);
7837 %}
7838 ins_pipe(pipe_slow);
7839 %}
7840
7841 // ============================================================================
7842 // MemBar Instruction
7843
7844 instruct load_fence() %{
7845 match(LoadFence);
7846 ins_cost(VOLATILE_REF_COST);
7847
7848 format %{ "load_fence" %}
7849
7850 ins_encode %{
7851 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7852 %}
7853 ins_pipe(pipe_serial);
7854 %}
7855
7856 instruct unnecessary_membar_acquire() %{
7857 predicate(unnecessary_acquire(n));
7858 match(MemBarAcquire);
7859 ins_cost(0);
7860
7861 format %{ "membar_acquire (elided)" %}
7862
7863 ins_encode %{
7864 __ block_comment("membar_acquire (elided)");
7865 %}
7866
7867 ins_pipe(pipe_class_empty);
7868 %}
7869
7870 instruct membar_acquire() %{
7871 match(MemBarAcquire);
7872 ins_cost(VOLATILE_REF_COST);
7873
7874 format %{ "membar_acquire\n\t"
7875 "dmb ishld" %}
7876
7877 ins_encode %{
7878 __ block_comment("membar_acquire");
7879 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7880 %}
7881
7882 ins_pipe(pipe_serial);
7883 %}
7884
7885
7886 instruct membar_acquire_lock() %{
7887 match(MemBarAcquireLock);
7888 ins_cost(VOLATILE_REF_COST);
7889
7890 format %{ "membar_acquire_lock (elided)" %}
7891
7892 ins_encode %{
7893 __ block_comment("membar_acquire_lock (elided)");
7894 %}
7895
7896 ins_pipe(pipe_serial);
7897 %}
7898
7899 instruct store_fence() %{
7900 match(StoreFence);
7901 ins_cost(VOLATILE_REF_COST);
7902
7903 format %{ "store_fence" %}
7904
7905 ins_encode %{
7906 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7907 %}
7908 ins_pipe(pipe_serial);
7909 %}
7910
7911 instruct unnecessary_membar_release() %{
7912 predicate(unnecessary_release(n));
7913 match(MemBarRelease);
7914 ins_cost(0);
7915
7916 format %{ "membar_release (elided)" %}
7917
7918 ins_encode %{
7919 __ block_comment("membar_release (elided)");
7920 %}
7921 ins_pipe(pipe_serial);
7922 %}
7923
7924 instruct membar_release() %{
7925 match(MemBarRelease);
7926 ins_cost(VOLATILE_REF_COST);
7927
7928 format %{ "membar_release\n\t"
7929 "dmb ishst\n\tdmb ishld" %}
7930
7931 ins_encode %{
7932 __ block_comment("membar_release");
7933 // These will be merged if AlwaysMergeDMB is enabled.
7934 __ membar(Assembler::StoreStore);
7935 __ membar(Assembler::LoadStore);
7936 %}
7937 ins_pipe(pipe_serial);
7938 %}
7939
7940 instruct membar_storestore() %{
7941 match(MemBarStoreStore);
7942 match(StoreStoreFence);
7943 ins_cost(VOLATILE_REF_COST);
7944
7945 format %{ "MEMBAR-store-store" %}
7946
7947 ins_encode %{
7948 __ membar(Assembler::StoreStore);
7949 %}
7950 ins_pipe(pipe_serial);
7951 %}
7952
7953 instruct membar_release_lock() %{
7954 match(MemBarReleaseLock);
7955 ins_cost(VOLATILE_REF_COST);
7956
7957 format %{ "membar_release_lock (elided)" %}
7958
7959 ins_encode %{
7960 __ block_comment("membar_release_lock (elided)");
7961 %}
7962
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct unnecessary_membar_volatile() %{
7967 predicate(unnecessary_volatile(n));
7968 match(MemBarVolatile);
7969 ins_cost(0);
7970
7971 format %{ "membar_volatile (elided)" %}
7972
7973 ins_encode %{
7974 __ block_comment("membar_volatile (elided)");
7975 %}
7976
7977 ins_pipe(pipe_serial);
7978 %}
7979
7980 instruct membar_volatile() %{
7981 match(MemBarVolatile);
7982 ins_cost(VOLATILE_REF_COST*100);
7983
7984 format %{ "membar_volatile\n\t"
7985 "dmb ish"%}
7986
7987 ins_encode %{
7988 __ block_comment("membar_volatile");
7989 __ membar(Assembler::StoreLoad);
7990 %}
7991
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 // ============================================================================
7996 // Cast/Convert Instructions
7997
7998 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7999 match(Set dst (CastX2P src));
8000
8001 ins_cost(INSN_COST);
8002 format %{ "mov $dst, $src\t# long -> ptr" %}
8003
8004 ins_encode %{
8005 if ($dst$$reg != $src$$reg) {
8006 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8007 }
8008 %}
8009
8010 ins_pipe(ialu_reg);
8011 %}
8012
8013 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8014 match(Set dst (CastI2N src));
8015
8016 ins_cost(INSN_COST);
8017 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8018
8019 ins_encode %{
8020 if ($dst$$reg != $src$$reg) {
8021 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8022 }
8023 %}
8024
8025 ins_pipe(ialu_reg);
8026 %}
8027
8028 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8029 match(Set dst (CastP2X src));
8030
8031 ins_cost(INSN_COST);
8032 format %{ "mov $dst, $src\t# ptr -> long" %}
8033
8034 ins_encode %{
8035 if ($dst$$reg != $src$$reg) {
8036 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8037 }
8038 %}
8039
8040 ins_pipe(ialu_reg);
8041 %}
8042
8043 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8044 match(Set dst (CastP2X src));
8045
8046 ins_cost(INSN_COST);
8047 format %{ "mov $dst, $src\t# ptr -> long" %}
8048
8049 ins_encode %{
8050 if ($dst$$reg != $src$$reg) {
8051 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8052 }
8053 %}
8054
8055 ins_pipe(ialu_reg);
8056 %}
8057
8058 // Convert oop into int for vectors alignment masking
8059 instruct convP2I(iRegINoSp dst, iRegP src) %{
8060 match(Set dst (ConvL2I (CastP2X src)));
8061
8062 ins_cost(INSN_COST);
8063 format %{ "movw $dst, $src\t# ptr -> int" %}
8064 ins_encode %{
8065 __ movw($dst$$Register, $src$$Register);
8066 %}
8067
8068 ins_pipe(ialu_reg);
8069 %}
8070
8071 // Convert compressed oop into int for vectors alignment masking
8072 // in case of 32bit oops (heap < 4Gb).
8073 instruct convN2I(iRegINoSp dst, iRegN src)
8074 %{
8075 predicate(CompressedOops::shift() == 0);
8076 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8077
8078 ins_cost(INSN_COST);
8079 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8080 ins_encode %{
8081 __ movw($dst$$Register, $src$$Register);
8082 %}
8083
8084 ins_pipe(ialu_reg);
8085 %}
8086
8087
8088 // Convert oop pointer into compressed form
8089 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8090 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8091 match(Set dst (EncodeP src));
8092 effect(KILL cr);
8093 ins_cost(INSN_COST * 3);
8094 format %{ "encode_heap_oop $dst, $src" %}
8095 ins_encode %{
8096 Register s = $src$$Register;
8097 Register d = $dst$$Register;
8098 __ encode_heap_oop(d, s);
8099 %}
8100 ins_pipe(ialu_reg);
8101 %}
8102
8103 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8104 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8105 match(Set dst (EncodeP src));
8106 ins_cost(INSN_COST * 3);
8107 format %{ "encode_heap_oop_not_null $dst, $src" %}
8108 ins_encode %{
8109 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8110 %}
8111 ins_pipe(ialu_reg);
8112 %}
8113
8114 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8115 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8116 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8117 match(Set dst (DecodeN src));
8118 ins_cost(INSN_COST * 3);
8119 format %{ "decode_heap_oop $dst, $src" %}
8120 ins_encode %{
8121 Register s = $src$$Register;
8122 Register d = $dst$$Register;
8123 __ decode_heap_oop(d, s);
8124 %}
8125 ins_pipe(ialu_reg);
8126 %}
8127
8128 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8129 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8130 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8131 match(Set dst (DecodeN src));
8132 ins_cost(INSN_COST * 3);
8133 format %{ "decode_heap_oop_not_null $dst, $src" %}
8134 ins_encode %{
8135 Register s = $src$$Register;
8136 Register d = $dst$$Register;
8137 __ decode_heap_oop_not_null(d, s);
8138 %}
8139 ins_pipe(ialu_reg);
8140 %}
8141
8142 // n.b. AArch64 implementations of encode_klass_not_null and
8143 // decode_klass_not_null do not modify the flags register so, unlike
8144 // Intel, we don't kill CR as a side effect here
8145
8146 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8147 match(Set dst (EncodePKlass src));
8148
8149 ins_cost(INSN_COST * 3);
8150 format %{ "encode_klass_not_null $dst,$src" %}
8151
8152 ins_encode %{
8153 Register src_reg = as_Register($src$$reg);
8154 Register dst_reg = as_Register($dst$$reg);
8155 __ encode_klass_not_null(dst_reg, src_reg);
8156 %}
8157
8158 ins_pipe(ialu_reg);
8159 %}
8160
8161 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8162 match(Set dst (DecodeNKlass src));
8163
8164 ins_cost(INSN_COST * 3);
8165 format %{ "decode_klass_not_null $dst,$src" %}
8166
8167 ins_encode %{
8168 Register src_reg = as_Register($src$$reg);
8169 Register dst_reg = as_Register($dst$$reg);
8170 if (dst_reg != src_reg) {
8171 __ decode_klass_not_null(dst_reg, src_reg);
8172 } else {
8173 __ decode_klass_not_null(dst_reg);
8174 }
8175 %}
8176
8177 ins_pipe(ialu_reg);
8178 %}
8179
8180 instruct checkCastPP(iRegPNoSp dst)
8181 %{
8182 match(Set dst (CheckCastPP dst));
8183
8184 size(0);
8185 format %{ "# checkcastPP of $dst" %}
8186 ins_encode(/* empty encoding */);
8187 ins_pipe(pipe_class_empty);
8188 %}
8189
8190 instruct castPP(iRegPNoSp dst)
8191 %{
8192 match(Set dst (CastPP dst));
8193
8194 size(0);
8195 format %{ "# castPP of $dst" %}
8196 ins_encode(/* empty encoding */);
8197 ins_pipe(pipe_class_empty);
8198 %}
8199
8200 instruct castII(iRegI dst)
8201 %{
8202 match(Set dst (CastII dst));
8203
8204 size(0);
8205 format %{ "# castII of $dst" %}
8206 ins_encode(/* empty encoding */);
8207 ins_cost(0);
8208 ins_pipe(pipe_class_empty);
8209 %}
8210
8211 instruct castLL(iRegL dst)
8212 %{
8213 match(Set dst (CastLL dst));
8214
8215 size(0);
8216 format %{ "# castLL of $dst" %}
8217 ins_encode(/* empty encoding */);
8218 ins_cost(0);
8219 ins_pipe(pipe_class_empty);
8220 %}
8221
8222 instruct castFF(vRegF dst)
8223 %{
8224 match(Set dst (CastFF dst));
8225
8226 size(0);
8227 format %{ "# castFF of $dst" %}
8228 ins_encode(/* empty encoding */);
8229 ins_cost(0);
8230 ins_pipe(pipe_class_empty);
8231 %}
8232
8233 instruct castDD(vRegD dst)
8234 %{
8235 match(Set dst (CastDD dst));
8236
8237 size(0);
8238 format %{ "# castDD of $dst" %}
8239 ins_encode(/* empty encoding */);
8240 ins_cost(0);
8241 ins_pipe(pipe_class_empty);
8242 %}
8243
8244 instruct castVV(vReg dst)
8245 %{
8246 match(Set dst (CastVV dst));
8247
8248 size(0);
8249 format %{ "# castVV of $dst" %}
8250 ins_encode(/* empty encoding */);
8251 ins_cost(0);
8252 ins_pipe(pipe_class_empty);
8253 %}
8254
8255 instruct castVVMask(pRegGov dst)
8256 %{
8257 match(Set dst (CastVV dst));
8258
8259 size(0);
8260 format %{ "# castVV of $dst" %}
8261 ins_encode(/* empty encoding */);
8262 ins_cost(0);
8263 ins_pipe(pipe_class_empty);
8264 %}
8265
8266 // ============================================================================
8267 // Atomic operation instructions
8268 //
8269
8270 // standard CompareAndSwapX when we are using barriers
8271 // these have higher priority than the rules selected by a predicate
8272
8273 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8274 // can't match them
8275
8276 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8277
8278 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8279 ins_cost(2 * VOLATILE_REF_COST);
8280
8281 effect(KILL cr);
8282
8283 format %{
8284 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8285 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8286 %}
8287
8288 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8289 aarch64_enc_cset_eq(res));
8290
8291 ins_pipe(pipe_slow);
8292 %}
8293
8294 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8295
8296 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8297 ins_cost(2 * VOLATILE_REF_COST);
8298
8299 effect(KILL cr);
8300
8301 format %{
8302 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8303 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8304 %}
8305
8306 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8307 aarch64_enc_cset_eq(res));
8308
8309 ins_pipe(pipe_slow);
8310 %}
8311
8312 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8313
8314 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8315 ins_cost(2 * VOLATILE_REF_COST);
8316
8317 effect(KILL cr);
8318
8319 format %{
8320 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8321 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8322 %}
8323
8324 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8325 aarch64_enc_cset_eq(res));
8326
8327 ins_pipe(pipe_slow);
8328 %}
8329
8330 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8331
8332 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8333 ins_cost(2 * VOLATILE_REF_COST);
8334
8335 effect(KILL cr);
8336
8337 format %{
8338 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8339 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8340 %}
8341
8342 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8343 aarch64_enc_cset_eq(res));
8344
8345 ins_pipe(pipe_slow);
8346 %}
8347
8348 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8349
8350 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8351 predicate(n->as_LoadStore()->barrier_data() == 0);
8352 ins_cost(2 * VOLATILE_REF_COST);
8353
8354 effect(KILL cr);
8355
8356 format %{
8357 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8358 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8359 %}
8360
8361 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8362 aarch64_enc_cset_eq(res));
8363
8364 ins_pipe(pipe_slow);
8365 %}
8366
8367 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8368
8369 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8370 predicate(n->as_LoadStore()->barrier_data() == 0);
8371 ins_cost(2 * VOLATILE_REF_COST);
8372
8373 effect(KILL cr);
8374
8375 format %{
8376 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8377 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8378 %}
8379
8380 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8381 aarch64_enc_cset_eq(res));
8382
8383 ins_pipe(pipe_slow);
8384 %}
8385
8386 // alternative CompareAndSwapX when we are eliding barriers
8387
8388 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8389
8390 predicate(needs_acquiring_load_exclusive(n));
8391 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8392 ins_cost(VOLATILE_REF_COST);
8393
8394 effect(KILL cr);
8395
8396 format %{
8397 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8398 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8399 %}
8400
8401 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8402 aarch64_enc_cset_eq(res));
8403
8404 ins_pipe(pipe_slow);
8405 %}
8406
8407 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8408
8409 predicate(needs_acquiring_load_exclusive(n));
8410 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8411 ins_cost(VOLATILE_REF_COST);
8412
8413 effect(KILL cr);
8414
8415 format %{
8416 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8417 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8418 %}
8419
8420 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8421 aarch64_enc_cset_eq(res));
8422
8423 ins_pipe(pipe_slow);
8424 %}
8425
8426 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8427
8428 predicate(needs_acquiring_load_exclusive(n));
8429 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8430 ins_cost(VOLATILE_REF_COST);
8431
8432 effect(KILL cr);
8433
8434 format %{
8435 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8436 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8437 %}
8438
8439 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8440 aarch64_enc_cset_eq(res));
8441
8442 ins_pipe(pipe_slow);
8443 %}
8444
8445 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8446
8447 predicate(needs_acquiring_load_exclusive(n));
8448 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8449 ins_cost(VOLATILE_REF_COST);
8450
8451 effect(KILL cr);
8452
8453 format %{
8454 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8455 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8456 %}
8457
8458 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8459 aarch64_enc_cset_eq(res));
8460
8461 ins_pipe(pipe_slow);
8462 %}
8463
8464 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8465
8466 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8467 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8468 ins_cost(VOLATILE_REF_COST);
8469
8470 effect(KILL cr);
8471
8472 format %{
8473 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8474 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8475 %}
8476
8477 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8478 aarch64_enc_cset_eq(res));
8479
8480 ins_pipe(pipe_slow);
8481 %}
8482
8483 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8484
8485 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8486 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8487 ins_cost(VOLATILE_REF_COST);
8488
8489 effect(KILL cr);
8490
8491 format %{
8492 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8493 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8494 %}
8495
8496 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8497 aarch64_enc_cset_eq(res));
8498
8499 ins_pipe(pipe_slow);
8500 %}
8501
8502
8503 // ---------------------------------------------------------------------
8504
8505 // BEGIN This section of the file is automatically generated. Do not edit --------------
8506
8507 // Sundry CAS operations. Note that release is always true,
8508 // regardless of the memory ordering of the CAS. This is because we
8509 // need the volatile case to be sequentially consistent but there is
8510 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8511 // can't check the type of memory ordering here, so we always emit a
8512 // STLXR.
8513
8514 // This section is generated from cas.m4
8515
8516
8517 // This pattern is generated automatically from cas.m4.
8518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8519 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8520 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8521 ins_cost(2 * VOLATILE_REF_COST);
8522 effect(TEMP_DEF res, KILL cr);
8523 format %{
8524 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8525 %}
8526 ins_encode %{
8527 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8528 Assembler::byte, /*acquire*/ false, /*release*/ true,
8529 /*weak*/ false, $res$$Register);
8530 __ sxtbw($res$$Register, $res$$Register);
8531 %}
8532 ins_pipe(pipe_slow);
8533 %}
8534
8535 // This pattern is generated automatically from cas.m4.
8536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8537 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8538 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8539 ins_cost(2 * VOLATILE_REF_COST);
8540 effect(TEMP_DEF res, KILL cr);
8541 format %{
8542 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8543 %}
8544 ins_encode %{
8545 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8546 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8547 /*weak*/ false, $res$$Register);
8548 __ sxthw($res$$Register, $res$$Register);
8549 %}
8550 ins_pipe(pipe_slow);
8551 %}
8552
8553 // This pattern is generated automatically from cas.m4.
8554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8555 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8556 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8557 ins_cost(2 * VOLATILE_REF_COST);
8558 effect(TEMP_DEF res, KILL cr);
8559 format %{
8560 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8561 %}
8562 ins_encode %{
8563 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8564 Assembler::word, /*acquire*/ false, /*release*/ true,
8565 /*weak*/ false, $res$$Register);
8566 %}
8567 ins_pipe(pipe_slow);
8568 %}
8569
8570 // This pattern is generated automatically from cas.m4.
8571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8572 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8573 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8574 ins_cost(2 * VOLATILE_REF_COST);
8575 effect(TEMP_DEF res, KILL cr);
8576 format %{
8577 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8578 %}
8579 ins_encode %{
8580 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8581 Assembler::xword, /*acquire*/ false, /*release*/ true,
8582 /*weak*/ false, $res$$Register);
8583 %}
8584 ins_pipe(pipe_slow);
8585 %}
8586
8587 // This pattern is generated automatically from cas.m4.
8588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8589 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8590 predicate(n->as_LoadStore()->barrier_data() == 0);
8591 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8592 ins_cost(2 * VOLATILE_REF_COST);
8593 effect(TEMP_DEF res, KILL cr);
8594 format %{
8595 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8596 %}
8597 ins_encode %{
8598 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8599 Assembler::word, /*acquire*/ false, /*release*/ true,
8600 /*weak*/ false, $res$$Register);
8601 %}
8602 ins_pipe(pipe_slow);
8603 %}
8604
8605 // This pattern is generated automatically from cas.m4.
8606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8607 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8608 predicate(n->as_LoadStore()->barrier_data() == 0);
8609 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8610 ins_cost(2 * VOLATILE_REF_COST);
8611 effect(TEMP_DEF res, KILL cr);
8612 format %{
8613 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8614 %}
8615 ins_encode %{
8616 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8617 Assembler::xword, /*acquire*/ false, /*release*/ true,
8618 /*weak*/ false, $res$$Register);
8619 %}
8620 ins_pipe(pipe_slow);
8621 %}
8622
8623 // This pattern is generated automatically from cas.m4.
8624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8625 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8626 predicate(needs_acquiring_load_exclusive(n));
8627 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8628 ins_cost(VOLATILE_REF_COST);
8629 effect(TEMP_DEF res, KILL cr);
8630 format %{
8631 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8632 %}
8633 ins_encode %{
8634 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8635 Assembler::byte, /*acquire*/ true, /*release*/ true,
8636 /*weak*/ false, $res$$Register);
8637 __ sxtbw($res$$Register, $res$$Register);
8638 %}
8639 ins_pipe(pipe_slow);
8640 %}
8641
8642 // This pattern is generated automatically from cas.m4.
8643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8644 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8645 predicate(needs_acquiring_load_exclusive(n));
8646 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8647 ins_cost(VOLATILE_REF_COST);
8648 effect(TEMP_DEF res, KILL cr);
8649 format %{
8650 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8651 %}
8652 ins_encode %{
8653 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8654 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8655 /*weak*/ false, $res$$Register);
8656 __ sxthw($res$$Register, $res$$Register);
8657 %}
8658 ins_pipe(pipe_slow);
8659 %}
8660
8661 // This pattern is generated automatically from cas.m4.
8662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8663 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8664 predicate(needs_acquiring_load_exclusive(n));
8665 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8666 ins_cost(VOLATILE_REF_COST);
8667 effect(TEMP_DEF res, KILL cr);
8668 format %{
8669 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8670 %}
8671 ins_encode %{
8672 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8673 Assembler::word, /*acquire*/ true, /*release*/ true,
8674 /*weak*/ false, $res$$Register);
8675 %}
8676 ins_pipe(pipe_slow);
8677 %}
8678
8679 // This pattern is generated automatically from cas.m4.
8680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8681 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8682 predicate(needs_acquiring_load_exclusive(n));
8683 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8684 ins_cost(VOLATILE_REF_COST);
8685 effect(TEMP_DEF res, KILL cr);
8686 format %{
8687 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8688 %}
8689 ins_encode %{
8690 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8691 Assembler::xword, /*acquire*/ true, /*release*/ true,
8692 /*weak*/ false, $res$$Register);
8693 %}
8694 ins_pipe(pipe_slow);
8695 %}
8696
8697 // This pattern is generated automatically from cas.m4.
8698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8699 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8700 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8701 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8702 ins_cost(VOLATILE_REF_COST);
8703 effect(TEMP_DEF res, KILL cr);
8704 format %{
8705 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8706 %}
8707 ins_encode %{
8708 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8709 Assembler::word, /*acquire*/ true, /*release*/ true,
8710 /*weak*/ false, $res$$Register);
8711 %}
8712 ins_pipe(pipe_slow);
8713 %}
8714
8715 // This pattern is generated automatically from cas.m4.
8716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8717 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8718 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8719 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8720 ins_cost(VOLATILE_REF_COST);
8721 effect(TEMP_DEF res, KILL cr);
8722 format %{
8723 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8724 %}
8725 ins_encode %{
8726 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8727 Assembler::xword, /*acquire*/ true, /*release*/ true,
8728 /*weak*/ false, $res$$Register);
8729 %}
8730 ins_pipe(pipe_slow);
8731 %}
8732
8733 // This pattern is generated automatically from cas.m4.
8734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8735 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8736 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8737 ins_cost(2 * VOLATILE_REF_COST);
8738 effect(KILL cr);
8739 format %{
8740 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8741 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8742 %}
8743 ins_encode %{
8744 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8745 Assembler::byte, /*acquire*/ false, /*release*/ true,
8746 /*weak*/ true, noreg);
8747 __ csetw($res$$Register, Assembler::EQ);
8748 %}
8749 ins_pipe(pipe_slow);
8750 %}
8751
8752 // This pattern is generated automatically from cas.m4.
8753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8754 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8755 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8756 ins_cost(2 * VOLATILE_REF_COST);
8757 effect(KILL cr);
8758 format %{
8759 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8760 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8761 %}
8762 ins_encode %{
8763 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8764 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8765 /*weak*/ true, noreg);
8766 __ csetw($res$$Register, Assembler::EQ);
8767 %}
8768 ins_pipe(pipe_slow);
8769 %}
8770
8771 // This pattern is generated automatically from cas.m4.
8772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8773 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8774 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8775 ins_cost(2 * VOLATILE_REF_COST);
8776 effect(KILL cr);
8777 format %{
8778 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8779 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8780 %}
8781 ins_encode %{
8782 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8783 Assembler::word, /*acquire*/ false, /*release*/ true,
8784 /*weak*/ true, noreg);
8785 __ csetw($res$$Register, Assembler::EQ);
8786 %}
8787 ins_pipe(pipe_slow);
8788 %}
8789
8790 // This pattern is generated automatically from cas.m4.
8791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8792 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8793 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8794 ins_cost(2 * VOLATILE_REF_COST);
8795 effect(KILL cr);
8796 format %{
8797 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8798 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8799 %}
8800 ins_encode %{
8801 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8802 Assembler::xword, /*acquire*/ false, /*release*/ true,
8803 /*weak*/ true, noreg);
8804 __ csetw($res$$Register, Assembler::EQ);
8805 %}
8806 ins_pipe(pipe_slow);
8807 %}
8808
8809 // This pattern is generated automatically from cas.m4.
8810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8811 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8812 predicate(n->as_LoadStore()->barrier_data() == 0);
8813 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8814 ins_cost(2 * VOLATILE_REF_COST);
8815 effect(KILL cr);
8816 format %{
8817 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8818 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8819 %}
8820 ins_encode %{
8821 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8822 Assembler::word, /*acquire*/ false, /*release*/ true,
8823 /*weak*/ true, noreg);
8824 __ csetw($res$$Register, Assembler::EQ);
8825 %}
8826 ins_pipe(pipe_slow);
8827 %}
8828
8829 // This pattern is generated automatically from cas.m4.
8830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8831 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8832 predicate(n->as_LoadStore()->barrier_data() == 0);
8833 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8834 ins_cost(2 * VOLATILE_REF_COST);
8835 effect(KILL cr);
8836 format %{
8837 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8838 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8839 %}
8840 ins_encode %{
8841 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8842 Assembler::xword, /*acquire*/ false, /*release*/ true,
8843 /*weak*/ true, noreg);
8844 __ csetw($res$$Register, Assembler::EQ);
8845 %}
8846 ins_pipe(pipe_slow);
8847 %}
8848
8849 // This pattern is generated automatically from cas.m4.
8850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8851 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8852 predicate(needs_acquiring_load_exclusive(n));
8853 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8854 ins_cost(VOLATILE_REF_COST);
8855 effect(KILL cr);
8856 format %{
8857 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8858 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8859 %}
8860 ins_encode %{
8861 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8862 Assembler::byte, /*acquire*/ true, /*release*/ true,
8863 /*weak*/ true, noreg);
8864 __ csetw($res$$Register, Assembler::EQ);
8865 %}
8866 ins_pipe(pipe_slow);
8867 %}
8868
8869 // This pattern is generated automatically from cas.m4.
8870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8871 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8872 predicate(needs_acquiring_load_exclusive(n));
8873 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8874 ins_cost(VOLATILE_REF_COST);
8875 effect(KILL cr);
8876 format %{
8877 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8878 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8879 %}
8880 ins_encode %{
8881 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8882 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8883 /*weak*/ true, noreg);
8884 __ csetw($res$$Register, Assembler::EQ);
8885 %}
8886 ins_pipe(pipe_slow);
8887 %}
8888
8889 // This pattern is generated automatically from cas.m4.
8890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8891 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8892 predicate(needs_acquiring_load_exclusive(n));
8893 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8894 ins_cost(VOLATILE_REF_COST);
8895 effect(KILL cr);
8896 format %{
8897 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8898 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8899 %}
8900 ins_encode %{
8901 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8902 Assembler::word, /*acquire*/ true, /*release*/ true,
8903 /*weak*/ true, noreg);
8904 __ csetw($res$$Register, Assembler::EQ);
8905 %}
8906 ins_pipe(pipe_slow);
8907 %}
8908
8909 // This pattern is generated automatically from cas.m4.
8910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8911 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8912 predicate(needs_acquiring_load_exclusive(n));
8913 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8914 ins_cost(VOLATILE_REF_COST);
8915 effect(KILL cr);
8916 format %{
8917 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8918 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8919 %}
8920 ins_encode %{
8921 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8922 Assembler::xword, /*acquire*/ true, /*release*/ true,
8923 /*weak*/ true, noreg);
8924 __ csetw($res$$Register, Assembler::EQ);
8925 %}
8926 ins_pipe(pipe_slow);
8927 %}
8928
8929 // This pattern is generated automatically from cas.m4.
8930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8931 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8932 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8933 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8934 ins_cost(VOLATILE_REF_COST);
8935 effect(KILL cr);
8936 format %{
8937 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8938 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8939 %}
8940 ins_encode %{
8941 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8942 Assembler::word, /*acquire*/ true, /*release*/ true,
8943 /*weak*/ true, noreg);
8944 __ csetw($res$$Register, Assembler::EQ);
8945 %}
8946 ins_pipe(pipe_slow);
8947 %}
8948
8949 // This pattern is generated automatically from cas.m4.
8950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8951 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8952 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8953 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8954 ins_cost(VOLATILE_REF_COST);
8955 effect(KILL cr);
8956 format %{
8957 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8958 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8959 %}
8960 ins_encode %{
8961 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8962 Assembler::xword, /*acquire*/ true, /*release*/ true,
8963 /*weak*/ true, noreg);
8964 __ csetw($res$$Register, Assembler::EQ);
8965 %}
8966 ins_pipe(pipe_slow);
8967 %}
8968
8969 // END This section of the file is automatically generated. Do not edit --------------
8970 // ---------------------------------------------------------------------
8971
8972 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8973 match(Set prev (GetAndSetI mem newv));
8974 ins_cost(2 * VOLATILE_REF_COST);
8975 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8976 ins_encode %{
8977 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8978 %}
8979 ins_pipe(pipe_serial);
8980 %}
8981
8982 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8983 match(Set prev (GetAndSetL mem newv));
8984 ins_cost(2 * VOLATILE_REF_COST);
8985 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8986 ins_encode %{
8987 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8988 %}
8989 ins_pipe(pipe_serial);
8990 %}
8991
8992 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8993 predicate(n->as_LoadStore()->barrier_data() == 0);
8994 match(Set prev (GetAndSetN mem newv));
8995 ins_cost(2 * VOLATILE_REF_COST);
8996 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8997 ins_encode %{
8998 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8999 %}
9000 ins_pipe(pipe_serial);
9001 %}
9002
9003 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9004 predicate(n->as_LoadStore()->barrier_data() == 0);
9005 match(Set prev (GetAndSetP mem newv));
9006 ins_cost(2 * VOLATILE_REF_COST);
9007 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9008 ins_encode %{
9009 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9010 %}
9011 ins_pipe(pipe_serial);
9012 %}
9013
9014 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9015 predicate(needs_acquiring_load_exclusive(n));
9016 match(Set prev (GetAndSetI mem newv));
9017 ins_cost(VOLATILE_REF_COST);
9018 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9019 ins_encode %{
9020 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9021 %}
9022 ins_pipe(pipe_serial);
9023 %}
9024
9025 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9026 predicate(needs_acquiring_load_exclusive(n));
9027 match(Set prev (GetAndSetL mem newv));
9028 ins_cost(VOLATILE_REF_COST);
9029 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9030 ins_encode %{
9031 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9032 %}
9033 ins_pipe(pipe_serial);
9034 %}
9035
9036 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9037 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9038 match(Set prev (GetAndSetN mem newv));
9039 ins_cost(VOLATILE_REF_COST);
9040 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9041 ins_encode %{
9042 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9043 %}
9044 ins_pipe(pipe_serial);
9045 %}
9046
9047 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9048 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9049 match(Set prev (GetAndSetP mem newv));
9050 ins_cost(VOLATILE_REF_COST);
9051 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9052 ins_encode %{
9053 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9054 %}
9055 ins_pipe(pipe_serial);
9056 %}
9057
9058
9059 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9060 match(Set newval (GetAndAddL mem incr));
9061 ins_cost(2 * VOLATILE_REF_COST + 1);
9062 format %{ "get_and_addL $newval, [$mem], $incr" %}
9063 ins_encode %{
9064 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9065 %}
9066 ins_pipe(pipe_serial);
9067 %}
9068
9069 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9070 predicate(n->as_LoadStore()->result_not_used());
9071 match(Set dummy (GetAndAddL mem incr));
9072 ins_cost(2 * VOLATILE_REF_COST);
9073 format %{ "get_and_addL [$mem], $incr" %}
9074 ins_encode %{
9075 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9076 %}
9077 ins_pipe(pipe_serial);
9078 %}
9079
9080 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9081 match(Set newval (GetAndAddL mem incr));
9082 ins_cost(2 * VOLATILE_REF_COST + 1);
9083 format %{ "get_and_addL $newval, [$mem], $incr" %}
9084 ins_encode %{
9085 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9086 %}
9087 ins_pipe(pipe_serial);
9088 %}
9089
9090 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9091 predicate(n->as_LoadStore()->result_not_used());
9092 match(Set dummy (GetAndAddL mem incr));
9093 ins_cost(2 * VOLATILE_REF_COST);
9094 format %{ "get_and_addL [$mem], $incr" %}
9095 ins_encode %{
9096 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9097 %}
9098 ins_pipe(pipe_serial);
9099 %}
9100
9101 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9102 match(Set newval (GetAndAddI mem incr));
9103 ins_cost(2 * VOLATILE_REF_COST + 1);
9104 format %{ "get_and_addI $newval, [$mem], $incr" %}
9105 ins_encode %{
9106 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9107 %}
9108 ins_pipe(pipe_serial);
9109 %}
9110
9111 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9112 predicate(n->as_LoadStore()->result_not_used());
9113 match(Set dummy (GetAndAddI mem incr));
9114 ins_cost(2 * VOLATILE_REF_COST);
9115 format %{ "get_and_addI [$mem], $incr" %}
9116 ins_encode %{
9117 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9118 %}
9119 ins_pipe(pipe_serial);
9120 %}
9121
9122 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9123 match(Set newval (GetAndAddI mem incr));
9124 ins_cost(2 * VOLATILE_REF_COST + 1);
9125 format %{ "get_and_addI $newval, [$mem], $incr" %}
9126 ins_encode %{
9127 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9128 %}
9129 ins_pipe(pipe_serial);
9130 %}
9131
9132 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9133 predicate(n->as_LoadStore()->result_not_used());
9134 match(Set dummy (GetAndAddI mem incr));
9135 ins_cost(2 * VOLATILE_REF_COST);
9136 format %{ "get_and_addI [$mem], $incr" %}
9137 ins_encode %{
9138 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9139 %}
9140 ins_pipe(pipe_serial);
9141 %}
9142
9143 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9144 predicate(needs_acquiring_load_exclusive(n));
9145 match(Set newval (GetAndAddL mem incr));
9146 ins_cost(VOLATILE_REF_COST + 1);
9147 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9148 ins_encode %{
9149 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9150 %}
9151 ins_pipe(pipe_serial);
9152 %}
9153
9154 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9155 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9156 match(Set dummy (GetAndAddL mem incr));
9157 ins_cost(VOLATILE_REF_COST);
9158 format %{ "get_and_addL_acq [$mem], $incr" %}
9159 ins_encode %{
9160 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9161 %}
9162 ins_pipe(pipe_serial);
9163 %}
9164
9165 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9166 predicate(needs_acquiring_load_exclusive(n));
9167 match(Set newval (GetAndAddL mem incr));
9168 ins_cost(VOLATILE_REF_COST + 1);
9169 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9170 ins_encode %{
9171 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9172 %}
9173 ins_pipe(pipe_serial);
9174 %}
9175
9176 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9177 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9178 match(Set dummy (GetAndAddL mem incr));
9179 ins_cost(VOLATILE_REF_COST);
9180 format %{ "get_and_addL_acq [$mem], $incr" %}
9181 ins_encode %{
9182 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9183 %}
9184 ins_pipe(pipe_serial);
9185 %}
9186
9187 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9188 predicate(needs_acquiring_load_exclusive(n));
9189 match(Set newval (GetAndAddI mem incr));
9190 ins_cost(VOLATILE_REF_COST + 1);
9191 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9192 ins_encode %{
9193 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9194 %}
9195 ins_pipe(pipe_serial);
9196 %}
9197
9198 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9199 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9200 match(Set dummy (GetAndAddI mem incr));
9201 ins_cost(VOLATILE_REF_COST);
9202 format %{ "get_and_addI_acq [$mem], $incr" %}
9203 ins_encode %{
9204 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9205 %}
9206 ins_pipe(pipe_serial);
9207 %}
9208
9209 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9210 predicate(needs_acquiring_load_exclusive(n));
9211 match(Set newval (GetAndAddI mem incr));
9212 ins_cost(VOLATILE_REF_COST + 1);
9213 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9214 ins_encode %{
9215 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9216 %}
9217 ins_pipe(pipe_serial);
9218 %}
9219
9220 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9221 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9222 match(Set dummy (GetAndAddI mem incr));
9223 ins_cost(VOLATILE_REF_COST);
9224 format %{ "get_and_addI_acq [$mem], $incr" %}
9225 ins_encode %{
9226 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9227 %}
9228 ins_pipe(pipe_serial);
9229 %}
9230
9231 // Manifest a CmpU result in an integer register.
9232 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9233 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9234 %{
9235 match(Set dst (CmpU3 src1 src2));
9236 effect(KILL flags);
9237
9238 ins_cost(INSN_COST * 3);
9239 format %{
9240 "cmpw $src1, $src2\n\t"
9241 "csetw $dst, ne\n\t"
9242 "cnegw $dst, lo\t# CmpU3(reg)"
9243 %}
9244 ins_encode %{
9245 __ cmpw($src1$$Register, $src2$$Register);
9246 __ csetw($dst$$Register, Assembler::NE);
9247 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9248 %}
9249
9250 ins_pipe(pipe_class_default);
9251 %}
9252
9253 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9254 %{
9255 match(Set dst (CmpU3 src1 src2));
9256 effect(KILL flags);
9257
9258 ins_cost(INSN_COST * 3);
9259 format %{
9260 "subsw zr, $src1, $src2\n\t"
9261 "csetw $dst, ne\n\t"
9262 "cnegw $dst, lo\t# CmpU3(imm)"
9263 %}
9264 ins_encode %{
9265 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9266 __ csetw($dst$$Register, Assembler::NE);
9267 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9268 %}
9269
9270 ins_pipe(pipe_class_default);
9271 %}
9272
9273 // Manifest a CmpUL result in an integer register.
9274 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9275 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9276 %{
9277 match(Set dst (CmpUL3 src1 src2));
9278 effect(KILL flags);
9279
9280 ins_cost(INSN_COST * 3);
9281 format %{
9282 "cmp $src1, $src2\n\t"
9283 "csetw $dst, ne\n\t"
9284 "cnegw $dst, lo\t# CmpUL3(reg)"
9285 %}
9286 ins_encode %{
9287 __ cmp($src1$$Register, $src2$$Register);
9288 __ csetw($dst$$Register, Assembler::NE);
9289 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9290 %}
9291
9292 ins_pipe(pipe_class_default);
9293 %}
9294
9295 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9296 %{
9297 match(Set dst (CmpUL3 src1 src2));
9298 effect(KILL flags);
9299
9300 ins_cost(INSN_COST * 3);
9301 format %{
9302 "subs zr, $src1, $src2\n\t"
9303 "csetw $dst, ne\n\t"
9304 "cnegw $dst, lo\t# CmpUL3(imm)"
9305 %}
9306 ins_encode %{
9307 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9308 __ csetw($dst$$Register, Assembler::NE);
9309 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9310 %}
9311
9312 ins_pipe(pipe_class_default);
9313 %}
9314
9315 // Manifest a CmpL result in an integer register.
9316 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9317 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9318 %{
9319 match(Set dst (CmpL3 src1 src2));
9320 effect(KILL flags);
9321
9322 ins_cost(INSN_COST * 3);
9323 format %{
9324 "cmp $src1, $src2\n\t"
9325 "csetw $dst, ne\n\t"
9326 "cnegw $dst, lt\t# CmpL3(reg)"
9327 %}
9328 ins_encode %{
9329 __ cmp($src1$$Register, $src2$$Register);
9330 __ csetw($dst$$Register, Assembler::NE);
9331 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9332 %}
9333
9334 ins_pipe(pipe_class_default);
9335 %}
9336
9337 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9338 %{
9339 match(Set dst (CmpL3 src1 src2));
9340 effect(KILL flags);
9341
9342 ins_cost(INSN_COST * 3);
9343 format %{
9344 "subs zr, $src1, $src2\n\t"
9345 "csetw $dst, ne\n\t"
9346 "cnegw $dst, lt\t# CmpL3(imm)"
9347 %}
9348 ins_encode %{
9349 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9350 __ csetw($dst$$Register, Assembler::NE);
9351 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9352 %}
9353
9354 ins_pipe(pipe_class_default);
9355 %}
9356
9357 // ============================================================================
9358 // Conditional Move Instructions
9359
9360 // n.b. we have identical rules for both a signed compare op (cmpOp)
9361 // and an unsigned compare op (cmpOpU). it would be nice if we could
9362 // define an op class which merged both inputs and use it to type the
9363 // argument to a single rule. unfortunatelyt his fails because the
9364 // opclass does not live up to the COND_INTER interface of its
9365 // component operands. When the generic code tries to negate the
9366 // operand it ends up running the generci Machoper::negate method
9367 // which throws a ShouldNotHappen. So, we have to provide two flavours
9368 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9369
9370 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9371 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9372
9373 ins_cost(INSN_COST * 2);
9374 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9375
9376 ins_encode %{
9377 __ cselw(as_Register($dst$$reg),
9378 as_Register($src2$$reg),
9379 as_Register($src1$$reg),
9380 (Assembler::Condition)$cmp$$cmpcode);
9381 %}
9382
9383 ins_pipe(icond_reg_reg);
9384 %}
9385
9386 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9387 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9388
9389 ins_cost(INSN_COST * 2);
9390 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9391
9392 ins_encode %{
9393 __ cselw(as_Register($dst$$reg),
9394 as_Register($src2$$reg),
9395 as_Register($src1$$reg),
9396 (Assembler::Condition)$cmp$$cmpcode);
9397 %}
9398
9399 ins_pipe(icond_reg_reg);
9400 %}
9401
9402 // special cases where one arg is zero
9403
9404 // n.b. this is selected in preference to the rule above because it
9405 // avoids loading constant 0 into a source register
9406
9407 // TODO
9408 // we ought only to be able to cull one of these variants as the ideal
9409 // transforms ought always to order the zero consistently (to left/right?)
9410
9411 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9412 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9413
9414 ins_cost(INSN_COST * 2);
9415 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9416
9417 ins_encode %{
9418 __ cselw(as_Register($dst$$reg),
9419 as_Register($src$$reg),
9420 zr,
9421 (Assembler::Condition)$cmp$$cmpcode);
9422 %}
9423
9424 ins_pipe(icond_reg);
9425 %}
9426
9427 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9428 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9429
9430 ins_cost(INSN_COST * 2);
9431 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9432
9433 ins_encode %{
9434 __ cselw(as_Register($dst$$reg),
9435 as_Register($src$$reg),
9436 zr,
9437 (Assembler::Condition)$cmp$$cmpcode);
9438 %}
9439
9440 ins_pipe(icond_reg);
9441 %}
9442
9443 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9444 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9445
9446 ins_cost(INSN_COST * 2);
9447 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9448
9449 ins_encode %{
9450 __ cselw(as_Register($dst$$reg),
9451 zr,
9452 as_Register($src$$reg),
9453 (Assembler::Condition)$cmp$$cmpcode);
9454 %}
9455
9456 ins_pipe(icond_reg);
9457 %}
9458
9459 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9460 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9461
9462 ins_cost(INSN_COST * 2);
9463 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9464
9465 ins_encode %{
9466 __ cselw(as_Register($dst$$reg),
9467 zr,
9468 as_Register($src$$reg),
9469 (Assembler::Condition)$cmp$$cmpcode);
9470 %}
9471
9472 ins_pipe(icond_reg);
9473 %}
9474
9475 // special case for creating a boolean 0 or 1
9476
9477 // n.b. this is selected in preference to the rule above because it
9478 // avoids loading constants 0 and 1 into a source register
9479
9480 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9481 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9482
9483 ins_cost(INSN_COST * 2);
9484 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9485
9486 ins_encode %{
9487 // equivalently
9488 // cset(as_Register($dst$$reg),
9489 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9490 __ csincw(as_Register($dst$$reg),
9491 zr,
9492 zr,
9493 (Assembler::Condition)$cmp$$cmpcode);
9494 %}
9495
9496 ins_pipe(icond_none);
9497 %}
9498
9499 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9500 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9501
9502 ins_cost(INSN_COST * 2);
9503 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9504
9505 ins_encode %{
9506 // equivalently
9507 // cset(as_Register($dst$$reg),
9508 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9509 __ csincw(as_Register($dst$$reg),
9510 zr,
9511 zr,
9512 (Assembler::Condition)$cmp$$cmpcode);
9513 %}
9514
9515 ins_pipe(icond_none);
9516 %}
9517
9518 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9519 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9520
9521 ins_cost(INSN_COST * 2);
9522 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9523
9524 ins_encode %{
9525 __ csel(as_Register($dst$$reg),
9526 as_Register($src2$$reg),
9527 as_Register($src1$$reg),
9528 (Assembler::Condition)$cmp$$cmpcode);
9529 %}
9530
9531 ins_pipe(icond_reg_reg);
9532 %}
9533
9534 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9535 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9536
9537 ins_cost(INSN_COST * 2);
9538 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9539
9540 ins_encode %{
9541 __ csel(as_Register($dst$$reg),
9542 as_Register($src2$$reg),
9543 as_Register($src1$$reg),
9544 (Assembler::Condition)$cmp$$cmpcode);
9545 %}
9546
9547 ins_pipe(icond_reg_reg);
9548 %}
9549
9550 // special cases where one arg is zero
9551
9552 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9553 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9554
9555 ins_cost(INSN_COST * 2);
9556 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9557
9558 ins_encode %{
9559 __ csel(as_Register($dst$$reg),
9560 zr,
9561 as_Register($src$$reg),
9562 (Assembler::Condition)$cmp$$cmpcode);
9563 %}
9564
9565 ins_pipe(icond_reg);
9566 %}
9567
9568 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9569 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9570
9571 ins_cost(INSN_COST * 2);
9572 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9573
9574 ins_encode %{
9575 __ csel(as_Register($dst$$reg),
9576 zr,
9577 as_Register($src$$reg),
9578 (Assembler::Condition)$cmp$$cmpcode);
9579 %}
9580
9581 ins_pipe(icond_reg);
9582 %}
9583
9584 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9585 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9586
9587 ins_cost(INSN_COST * 2);
9588 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9589
9590 ins_encode %{
9591 __ csel(as_Register($dst$$reg),
9592 as_Register($src$$reg),
9593 zr,
9594 (Assembler::Condition)$cmp$$cmpcode);
9595 %}
9596
9597 ins_pipe(icond_reg);
9598 %}
9599
9600 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9601 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9602
9603 ins_cost(INSN_COST * 2);
9604 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9605
9606 ins_encode %{
9607 __ csel(as_Register($dst$$reg),
9608 as_Register($src$$reg),
9609 zr,
9610 (Assembler::Condition)$cmp$$cmpcode);
9611 %}
9612
9613 ins_pipe(icond_reg);
9614 %}
9615
9616 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9617 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9618
9619 ins_cost(INSN_COST * 2);
9620 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9621
9622 ins_encode %{
9623 __ csel(as_Register($dst$$reg),
9624 as_Register($src2$$reg),
9625 as_Register($src1$$reg),
9626 (Assembler::Condition)$cmp$$cmpcode);
9627 %}
9628
9629 ins_pipe(icond_reg_reg);
9630 %}
9631
9632 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9633 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9634
9635 ins_cost(INSN_COST * 2);
9636 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9637
9638 ins_encode %{
9639 __ csel(as_Register($dst$$reg),
9640 as_Register($src2$$reg),
9641 as_Register($src1$$reg),
9642 (Assembler::Condition)$cmp$$cmpcode);
9643 %}
9644
9645 ins_pipe(icond_reg_reg);
9646 %}
9647
9648 // special cases where one arg is zero
9649
9650 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9651 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9652
9653 ins_cost(INSN_COST * 2);
9654 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9655
9656 ins_encode %{
9657 __ csel(as_Register($dst$$reg),
9658 zr,
9659 as_Register($src$$reg),
9660 (Assembler::Condition)$cmp$$cmpcode);
9661 %}
9662
9663 ins_pipe(icond_reg);
9664 %}
9665
9666 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9667 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9668
9669 ins_cost(INSN_COST * 2);
9670 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9671
9672 ins_encode %{
9673 __ csel(as_Register($dst$$reg),
9674 zr,
9675 as_Register($src$$reg),
9676 (Assembler::Condition)$cmp$$cmpcode);
9677 %}
9678
9679 ins_pipe(icond_reg);
9680 %}
9681
9682 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9683 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9684
9685 ins_cost(INSN_COST * 2);
9686 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9687
9688 ins_encode %{
9689 __ csel(as_Register($dst$$reg),
9690 as_Register($src$$reg),
9691 zr,
9692 (Assembler::Condition)$cmp$$cmpcode);
9693 %}
9694
9695 ins_pipe(icond_reg);
9696 %}
9697
9698 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9699 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9700
9701 ins_cost(INSN_COST * 2);
9702 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9703
9704 ins_encode %{
9705 __ csel(as_Register($dst$$reg),
9706 as_Register($src$$reg),
9707 zr,
9708 (Assembler::Condition)$cmp$$cmpcode);
9709 %}
9710
9711 ins_pipe(icond_reg);
9712 %}
9713
9714 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9715 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9716
9717 ins_cost(INSN_COST * 2);
9718 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9719
9720 ins_encode %{
9721 __ cselw(as_Register($dst$$reg),
9722 as_Register($src2$$reg),
9723 as_Register($src1$$reg),
9724 (Assembler::Condition)$cmp$$cmpcode);
9725 %}
9726
9727 ins_pipe(icond_reg_reg);
9728 %}
9729
9730 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9731 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9732
9733 ins_cost(INSN_COST * 2);
9734 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9735
9736 ins_encode %{
9737 __ cselw(as_Register($dst$$reg),
9738 as_Register($src2$$reg),
9739 as_Register($src1$$reg),
9740 (Assembler::Condition)$cmp$$cmpcode);
9741 %}
9742
9743 ins_pipe(icond_reg_reg);
9744 %}
9745
9746 // special cases where one arg is zero
9747
9748 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9749 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9750
9751 ins_cost(INSN_COST * 2);
9752 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9753
9754 ins_encode %{
9755 __ cselw(as_Register($dst$$reg),
9756 zr,
9757 as_Register($src$$reg),
9758 (Assembler::Condition)$cmp$$cmpcode);
9759 %}
9760
9761 ins_pipe(icond_reg);
9762 %}
9763
9764 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9765 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9766
9767 ins_cost(INSN_COST * 2);
9768 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9769
9770 ins_encode %{
9771 __ cselw(as_Register($dst$$reg),
9772 zr,
9773 as_Register($src$$reg),
9774 (Assembler::Condition)$cmp$$cmpcode);
9775 %}
9776
9777 ins_pipe(icond_reg);
9778 %}
9779
9780 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9781 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9782
9783 ins_cost(INSN_COST * 2);
9784 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9785
9786 ins_encode %{
9787 __ cselw(as_Register($dst$$reg),
9788 as_Register($src$$reg),
9789 zr,
9790 (Assembler::Condition)$cmp$$cmpcode);
9791 %}
9792
9793 ins_pipe(icond_reg);
9794 %}
9795
9796 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9797 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9798
9799 ins_cost(INSN_COST * 2);
9800 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9801
9802 ins_encode %{
9803 __ cselw(as_Register($dst$$reg),
9804 as_Register($src$$reg),
9805 zr,
9806 (Assembler::Condition)$cmp$$cmpcode);
9807 %}
9808
9809 ins_pipe(icond_reg);
9810 %}
9811
9812 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9813 %{
9814 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9815
9816 ins_cost(INSN_COST * 3);
9817
9818 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9819 ins_encode %{
9820 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9821 __ fcsels(as_FloatRegister($dst$$reg),
9822 as_FloatRegister($src2$$reg),
9823 as_FloatRegister($src1$$reg),
9824 cond);
9825 %}
9826
9827 ins_pipe(fp_cond_reg_reg_s);
9828 %}
9829
9830 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9831 %{
9832 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9833
9834 ins_cost(INSN_COST * 3);
9835
9836 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9837 ins_encode %{
9838 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9839 __ fcsels(as_FloatRegister($dst$$reg),
9840 as_FloatRegister($src2$$reg),
9841 as_FloatRegister($src1$$reg),
9842 cond);
9843 %}
9844
9845 ins_pipe(fp_cond_reg_reg_s);
9846 %}
9847
9848 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9849 %{
9850 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9851
9852 ins_cost(INSN_COST * 3);
9853
9854 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9855 ins_encode %{
9856 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9857 __ fcseld(as_FloatRegister($dst$$reg),
9858 as_FloatRegister($src2$$reg),
9859 as_FloatRegister($src1$$reg),
9860 cond);
9861 %}
9862
9863 ins_pipe(fp_cond_reg_reg_d);
9864 %}
9865
9866 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9867 %{
9868 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9869
9870 ins_cost(INSN_COST * 3);
9871
9872 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9873 ins_encode %{
9874 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9875 __ fcseld(as_FloatRegister($dst$$reg),
9876 as_FloatRegister($src2$$reg),
9877 as_FloatRegister($src1$$reg),
9878 cond);
9879 %}
9880
9881 ins_pipe(fp_cond_reg_reg_d);
9882 %}
9883
9884 // ============================================================================
9885 // Arithmetic Instructions
9886 //
9887
9888 // Integer Addition
9889
9890 // TODO
9891 // these currently employ operations which do not set CR and hence are
9892 // not flagged as killing CR but we would like to isolate the cases
9893 // where we want to set flags from those where we don't. need to work
9894 // out how to do that.
9895
9896 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9897 match(Set dst (AddI src1 src2));
9898
9899 ins_cost(INSN_COST);
9900 format %{ "addw $dst, $src1, $src2" %}
9901
9902 ins_encode %{
9903 __ addw(as_Register($dst$$reg),
9904 as_Register($src1$$reg),
9905 as_Register($src2$$reg));
9906 %}
9907
9908 ins_pipe(ialu_reg_reg);
9909 %}
9910
9911 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9912 match(Set dst (AddI src1 src2));
9913
9914 ins_cost(INSN_COST);
9915 format %{ "addw $dst, $src1, $src2" %}
9916
9917 // use opcode to indicate that this is an add not a sub
9918 opcode(0x0);
9919
9920 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9921
9922 ins_pipe(ialu_reg_imm);
9923 %}
9924
9925 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9926 match(Set dst (AddI (ConvL2I src1) src2));
9927
9928 ins_cost(INSN_COST);
9929 format %{ "addw $dst, $src1, $src2" %}
9930
9931 // use opcode to indicate that this is an add not a sub
9932 opcode(0x0);
9933
9934 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9935
9936 ins_pipe(ialu_reg_imm);
9937 %}
9938
9939 // Pointer Addition
9940 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9941 match(Set dst (AddP src1 src2));
9942
9943 ins_cost(INSN_COST);
9944 format %{ "add $dst, $src1, $src2\t# ptr" %}
9945
9946 ins_encode %{
9947 __ add(as_Register($dst$$reg),
9948 as_Register($src1$$reg),
9949 as_Register($src2$$reg));
9950 %}
9951
9952 ins_pipe(ialu_reg_reg);
9953 %}
9954
9955 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9956 match(Set dst (AddP src1 (ConvI2L src2)));
9957
9958 ins_cost(1.9 * INSN_COST);
9959 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9960
9961 ins_encode %{
9962 __ add(as_Register($dst$$reg),
9963 as_Register($src1$$reg),
9964 as_Register($src2$$reg), ext::sxtw);
9965 %}
9966
9967 ins_pipe(ialu_reg_reg);
9968 %}
9969
9970 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9971 match(Set dst (AddP src1 (LShiftL src2 scale)));
9972
9973 ins_cost(1.9 * INSN_COST);
9974 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9975
9976 ins_encode %{
9977 __ lea(as_Register($dst$$reg),
9978 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9979 Address::lsl($scale$$constant)));
9980 %}
9981
9982 ins_pipe(ialu_reg_reg_shift);
9983 %}
9984
9985 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9986 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9987
9988 ins_cost(1.9 * INSN_COST);
9989 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9990
9991 ins_encode %{
9992 __ lea(as_Register($dst$$reg),
9993 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9994 Address::sxtw($scale$$constant)));
9995 %}
9996
9997 ins_pipe(ialu_reg_reg_shift);
9998 %}
9999
10000 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10001 match(Set dst (LShiftL (ConvI2L src) scale));
10002
10003 ins_cost(INSN_COST);
10004 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10005
10006 ins_encode %{
10007 __ sbfiz(as_Register($dst$$reg),
10008 as_Register($src$$reg),
10009 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10010 %}
10011
10012 ins_pipe(ialu_reg_shift);
10013 %}
10014
10015 // Pointer Immediate Addition
10016 // n.b. this needs to be more expensive than using an indirect memory
10017 // operand
10018 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10019 match(Set dst (AddP src1 src2));
10020
10021 ins_cost(INSN_COST);
10022 format %{ "add $dst, $src1, $src2\t# ptr" %}
10023
10024 // use opcode to indicate that this is an add not a sub
10025 opcode(0x0);
10026
10027 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10028
10029 ins_pipe(ialu_reg_imm);
10030 %}
10031
10032 // Long Addition
10033 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10034
10035 match(Set dst (AddL src1 src2));
10036
10037 ins_cost(INSN_COST);
10038 format %{ "add $dst, $src1, $src2" %}
10039
10040 ins_encode %{
10041 __ add(as_Register($dst$$reg),
10042 as_Register($src1$$reg),
10043 as_Register($src2$$reg));
10044 %}
10045
10046 ins_pipe(ialu_reg_reg);
10047 %}
10048
10049 // No constant pool entries requiredLong Immediate Addition.
10050 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10051 match(Set dst (AddL src1 src2));
10052
10053 ins_cost(INSN_COST);
10054 format %{ "add $dst, $src1, $src2" %}
10055
10056 // use opcode to indicate that this is an add not a sub
10057 opcode(0x0);
10058
10059 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10060
10061 ins_pipe(ialu_reg_imm);
10062 %}
10063
10064 // Integer Subtraction
10065 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10066 match(Set dst (SubI src1 src2));
10067
10068 ins_cost(INSN_COST);
10069 format %{ "subw $dst, $src1, $src2" %}
10070
10071 ins_encode %{
10072 __ subw(as_Register($dst$$reg),
10073 as_Register($src1$$reg),
10074 as_Register($src2$$reg));
10075 %}
10076
10077 ins_pipe(ialu_reg_reg);
10078 %}
10079
10080 // Immediate Subtraction
10081 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10082 match(Set dst (SubI src1 src2));
10083
10084 ins_cost(INSN_COST);
10085 format %{ "subw $dst, $src1, $src2" %}
10086
10087 // use opcode to indicate that this is a sub not an add
10088 opcode(0x1);
10089
10090 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10091
10092 ins_pipe(ialu_reg_imm);
10093 %}
10094
10095 // Long Subtraction
10096 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10097
10098 match(Set dst (SubL src1 src2));
10099
10100 ins_cost(INSN_COST);
10101 format %{ "sub $dst, $src1, $src2" %}
10102
10103 ins_encode %{
10104 __ sub(as_Register($dst$$reg),
10105 as_Register($src1$$reg),
10106 as_Register($src2$$reg));
10107 %}
10108
10109 ins_pipe(ialu_reg_reg);
10110 %}
10111
10112 // No constant pool entries requiredLong Immediate Subtraction.
10113 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10114 match(Set dst (SubL src1 src2));
10115
10116 ins_cost(INSN_COST);
10117 format %{ "sub$dst, $src1, $src2" %}
10118
10119 // use opcode to indicate that this is a sub not an add
10120 opcode(0x1);
10121
10122 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10123
10124 ins_pipe(ialu_reg_imm);
10125 %}
10126
10127 // Integer Negation (special case for sub)
10128
10129 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10130 match(Set dst (SubI zero src));
10131
10132 ins_cost(INSN_COST);
10133 format %{ "negw $dst, $src\t# int" %}
10134
10135 ins_encode %{
10136 __ negw(as_Register($dst$$reg),
10137 as_Register($src$$reg));
10138 %}
10139
10140 ins_pipe(ialu_reg);
10141 %}
10142
10143 // Long Negation
10144
10145 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10146 match(Set dst (SubL zero src));
10147
10148 ins_cost(INSN_COST);
10149 format %{ "neg $dst, $src\t# long" %}
10150
10151 ins_encode %{
10152 __ neg(as_Register($dst$$reg),
10153 as_Register($src$$reg));
10154 %}
10155
10156 ins_pipe(ialu_reg);
10157 %}
10158
10159 // Integer Multiply
10160
10161 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10162 match(Set dst (MulI src1 src2));
10163
10164 ins_cost(INSN_COST * 3);
10165 format %{ "mulw $dst, $src1, $src2" %}
10166
10167 ins_encode %{
10168 __ mulw(as_Register($dst$$reg),
10169 as_Register($src1$$reg),
10170 as_Register($src2$$reg));
10171 %}
10172
10173 ins_pipe(imul_reg_reg);
10174 %}
10175
10176 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10177 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10178
10179 ins_cost(INSN_COST * 3);
10180 format %{ "smull $dst, $src1, $src2" %}
10181
10182 ins_encode %{
10183 __ smull(as_Register($dst$$reg),
10184 as_Register($src1$$reg),
10185 as_Register($src2$$reg));
10186 %}
10187
10188 ins_pipe(imul_reg_reg);
10189 %}
10190
10191 // Long Multiply
10192
10193 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10194 match(Set dst (MulL src1 src2));
10195
10196 ins_cost(INSN_COST * 5);
10197 format %{ "mul $dst, $src1, $src2" %}
10198
10199 ins_encode %{
10200 __ mul(as_Register($dst$$reg),
10201 as_Register($src1$$reg),
10202 as_Register($src2$$reg));
10203 %}
10204
10205 ins_pipe(lmul_reg_reg);
10206 %}
10207
10208 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10209 %{
10210 match(Set dst (MulHiL src1 src2));
10211
10212 ins_cost(INSN_COST * 7);
10213 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10214
10215 ins_encode %{
10216 __ smulh(as_Register($dst$$reg),
10217 as_Register($src1$$reg),
10218 as_Register($src2$$reg));
10219 %}
10220
10221 ins_pipe(lmul_reg_reg);
10222 %}
10223
10224 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10225 %{
10226 match(Set dst (UMulHiL src1 src2));
10227
10228 ins_cost(INSN_COST * 7);
10229 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10230
10231 ins_encode %{
10232 __ umulh(as_Register($dst$$reg),
10233 as_Register($src1$$reg),
10234 as_Register($src2$$reg));
10235 %}
10236
10237 ins_pipe(lmul_reg_reg);
10238 %}
10239
10240 // Combined Integer Multiply & Add/Sub
10241
10242 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10243 match(Set dst (AddI src3 (MulI src1 src2)));
10244
10245 ins_cost(INSN_COST * 3);
10246 format %{ "madd $dst, $src1, $src2, $src3" %}
10247
10248 ins_encode %{
10249 __ maddw(as_Register($dst$$reg),
10250 as_Register($src1$$reg),
10251 as_Register($src2$$reg),
10252 as_Register($src3$$reg));
10253 %}
10254
10255 ins_pipe(imac_reg_reg);
10256 %}
10257
10258 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10259 match(Set dst (SubI src3 (MulI src1 src2)));
10260
10261 ins_cost(INSN_COST * 3);
10262 format %{ "msub $dst, $src1, $src2, $src3" %}
10263
10264 ins_encode %{
10265 __ msubw(as_Register($dst$$reg),
10266 as_Register($src1$$reg),
10267 as_Register($src2$$reg),
10268 as_Register($src3$$reg));
10269 %}
10270
10271 ins_pipe(imac_reg_reg);
10272 %}
10273
10274 // Combined Integer Multiply & Neg
10275
10276 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10277 match(Set dst (MulI (SubI zero src1) src2));
10278
10279 ins_cost(INSN_COST * 3);
10280 format %{ "mneg $dst, $src1, $src2" %}
10281
10282 ins_encode %{
10283 __ mnegw(as_Register($dst$$reg),
10284 as_Register($src1$$reg),
10285 as_Register($src2$$reg));
10286 %}
10287
10288 ins_pipe(imac_reg_reg);
10289 %}
10290
10291 // Combined Long Multiply & Add/Sub
10292
10293 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10294 match(Set dst (AddL src3 (MulL src1 src2)));
10295
10296 ins_cost(INSN_COST * 5);
10297 format %{ "madd $dst, $src1, $src2, $src3" %}
10298
10299 ins_encode %{
10300 __ madd(as_Register($dst$$reg),
10301 as_Register($src1$$reg),
10302 as_Register($src2$$reg),
10303 as_Register($src3$$reg));
10304 %}
10305
10306 ins_pipe(lmac_reg_reg);
10307 %}
10308
10309 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10310 match(Set dst (SubL src3 (MulL src1 src2)));
10311
10312 ins_cost(INSN_COST * 5);
10313 format %{ "msub $dst, $src1, $src2, $src3" %}
10314
10315 ins_encode %{
10316 __ msub(as_Register($dst$$reg),
10317 as_Register($src1$$reg),
10318 as_Register($src2$$reg),
10319 as_Register($src3$$reg));
10320 %}
10321
10322 ins_pipe(lmac_reg_reg);
10323 %}
10324
10325 // Combined Long Multiply & Neg
10326
10327 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10328 match(Set dst (MulL (SubL zero src1) src2));
10329
10330 ins_cost(INSN_COST * 5);
10331 format %{ "mneg $dst, $src1, $src2" %}
10332
10333 ins_encode %{
10334 __ mneg(as_Register($dst$$reg),
10335 as_Register($src1$$reg),
10336 as_Register($src2$$reg));
10337 %}
10338
10339 ins_pipe(lmac_reg_reg);
10340 %}
10341
10342 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10343
10344 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10345 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10346
10347 ins_cost(INSN_COST * 3);
10348 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10349
10350 ins_encode %{
10351 __ smaddl(as_Register($dst$$reg),
10352 as_Register($src1$$reg),
10353 as_Register($src2$$reg),
10354 as_Register($src3$$reg));
10355 %}
10356
10357 ins_pipe(imac_reg_reg);
10358 %}
10359
10360 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10361 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10362
10363 ins_cost(INSN_COST * 3);
10364 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10365
10366 ins_encode %{
10367 __ smsubl(as_Register($dst$$reg),
10368 as_Register($src1$$reg),
10369 as_Register($src2$$reg),
10370 as_Register($src3$$reg));
10371 %}
10372
10373 ins_pipe(imac_reg_reg);
10374 %}
10375
10376 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10377 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10378
10379 ins_cost(INSN_COST * 3);
10380 format %{ "smnegl $dst, $src1, $src2" %}
10381
10382 ins_encode %{
10383 __ smnegl(as_Register($dst$$reg),
10384 as_Register($src1$$reg),
10385 as_Register($src2$$reg));
10386 %}
10387
10388 ins_pipe(imac_reg_reg);
10389 %}
10390
10391 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10392
10393 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10394 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10395
10396 ins_cost(INSN_COST * 5);
10397 format %{ "mulw rscratch1, $src1, $src2\n\t"
10398 "maddw $dst, $src3, $src4, rscratch1" %}
10399
10400 ins_encode %{
10401 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10402 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10403
10404 ins_pipe(imac_reg_reg);
10405 %}
10406
10407 // Integer Divide
10408
10409 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10410 match(Set dst (DivI src1 src2));
10411
10412 ins_cost(INSN_COST * 19);
10413 format %{ "sdivw $dst, $src1, $src2" %}
10414
10415 ins_encode(aarch64_enc_divw(dst, src1, src2));
10416 ins_pipe(idiv_reg_reg);
10417 %}
10418
10419 // Long Divide
10420
10421 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10422 match(Set dst (DivL src1 src2));
10423
10424 ins_cost(INSN_COST * 35);
10425 format %{ "sdiv $dst, $src1, $src2" %}
10426
10427 ins_encode(aarch64_enc_div(dst, src1, src2));
10428 ins_pipe(ldiv_reg_reg);
10429 %}
10430
10431 // Integer Remainder
10432
10433 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10434 match(Set dst (ModI src1 src2));
10435
10436 ins_cost(INSN_COST * 22);
10437 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10438 "msubw $dst, rscratch1, $src2, $src1" %}
10439
10440 ins_encode(aarch64_enc_modw(dst, src1, src2));
10441 ins_pipe(idiv_reg_reg);
10442 %}
10443
10444 // Long Remainder
10445
10446 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10447 match(Set dst (ModL src1 src2));
10448
10449 ins_cost(INSN_COST * 38);
10450 format %{ "sdiv rscratch1, $src1, $src2\n"
10451 "msub $dst, rscratch1, $src2, $src1" %}
10452
10453 ins_encode(aarch64_enc_mod(dst, src1, src2));
10454 ins_pipe(ldiv_reg_reg);
10455 %}
10456
10457 // Unsigned Integer Divide
10458
10459 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10460 match(Set dst (UDivI src1 src2));
10461
10462 ins_cost(INSN_COST * 19);
10463 format %{ "udivw $dst, $src1, $src2" %}
10464
10465 ins_encode %{
10466 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10467 %}
10468
10469 ins_pipe(idiv_reg_reg);
10470 %}
10471
10472 // Unsigned Long Divide
10473
10474 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10475 match(Set dst (UDivL src1 src2));
10476
10477 ins_cost(INSN_COST * 35);
10478 format %{ "udiv $dst, $src1, $src2" %}
10479
10480 ins_encode %{
10481 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10482 %}
10483
10484 ins_pipe(ldiv_reg_reg);
10485 %}
10486
10487 // Unsigned Integer Remainder
10488
10489 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10490 match(Set dst (UModI src1 src2));
10491
10492 ins_cost(INSN_COST * 22);
10493 format %{ "udivw rscratch1, $src1, $src2\n\t"
10494 "msubw $dst, rscratch1, $src2, $src1" %}
10495
10496 ins_encode %{
10497 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10498 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10499 %}
10500
10501 ins_pipe(idiv_reg_reg);
10502 %}
10503
10504 // Unsigned Long Remainder
10505
10506 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10507 match(Set dst (UModL src1 src2));
10508
10509 ins_cost(INSN_COST * 38);
10510 format %{ "udiv rscratch1, $src1, $src2\n"
10511 "msub $dst, rscratch1, $src2, $src1" %}
10512
10513 ins_encode %{
10514 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10515 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10516 %}
10517
10518 ins_pipe(ldiv_reg_reg);
10519 %}
10520
10521 // Integer Shifts
10522
10523 // Shift Left Register
10524 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10525 match(Set dst (LShiftI src1 src2));
10526
10527 ins_cost(INSN_COST * 2);
10528 format %{ "lslvw $dst, $src1, $src2" %}
10529
10530 ins_encode %{
10531 __ lslvw(as_Register($dst$$reg),
10532 as_Register($src1$$reg),
10533 as_Register($src2$$reg));
10534 %}
10535
10536 ins_pipe(ialu_reg_reg_vshift);
10537 %}
10538
10539 // Shift Left Immediate
10540 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10541 match(Set dst (LShiftI src1 src2));
10542
10543 ins_cost(INSN_COST);
10544 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10545
10546 ins_encode %{
10547 __ lslw(as_Register($dst$$reg),
10548 as_Register($src1$$reg),
10549 $src2$$constant & 0x1f);
10550 %}
10551
10552 ins_pipe(ialu_reg_shift);
10553 %}
10554
10555 // Shift Right Logical Register
10556 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10557 match(Set dst (URShiftI src1 src2));
10558
10559 ins_cost(INSN_COST * 2);
10560 format %{ "lsrvw $dst, $src1, $src2" %}
10561
10562 ins_encode %{
10563 __ lsrvw(as_Register($dst$$reg),
10564 as_Register($src1$$reg),
10565 as_Register($src2$$reg));
10566 %}
10567
10568 ins_pipe(ialu_reg_reg_vshift);
10569 %}
10570
10571 // Shift Right Logical Immediate
10572 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10573 match(Set dst (URShiftI src1 src2));
10574
10575 ins_cost(INSN_COST);
10576 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10577
10578 ins_encode %{
10579 __ lsrw(as_Register($dst$$reg),
10580 as_Register($src1$$reg),
10581 $src2$$constant & 0x1f);
10582 %}
10583
10584 ins_pipe(ialu_reg_shift);
10585 %}
10586
10587 // Shift Right Arithmetic Register
10588 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10589 match(Set dst (RShiftI src1 src2));
10590
10591 ins_cost(INSN_COST * 2);
10592 format %{ "asrvw $dst, $src1, $src2" %}
10593
10594 ins_encode %{
10595 __ asrvw(as_Register($dst$$reg),
10596 as_Register($src1$$reg),
10597 as_Register($src2$$reg));
10598 %}
10599
10600 ins_pipe(ialu_reg_reg_vshift);
10601 %}
10602
10603 // Shift Right Arithmetic Immediate
10604 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10605 match(Set dst (RShiftI src1 src2));
10606
10607 ins_cost(INSN_COST);
10608 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10609
10610 ins_encode %{
10611 __ asrw(as_Register($dst$$reg),
10612 as_Register($src1$$reg),
10613 $src2$$constant & 0x1f);
10614 %}
10615
10616 ins_pipe(ialu_reg_shift);
10617 %}
10618
10619 // Combined Int Mask and Right Shift (using UBFM)
10620 // TODO
10621
10622 // Long Shifts
10623
10624 // Shift Left Register
10625 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10626 match(Set dst (LShiftL src1 src2));
10627
10628 ins_cost(INSN_COST * 2);
10629 format %{ "lslv $dst, $src1, $src2" %}
10630
10631 ins_encode %{
10632 __ lslv(as_Register($dst$$reg),
10633 as_Register($src1$$reg),
10634 as_Register($src2$$reg));
10635 %}
10636
10637 ins_pipe(ialu_reg_reg_vshift);
10638 %}
10639
10640 // Shift Left Immediate
10641 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10642 match(Set dst (LShiftL src1 src2));
10643
10644 ins_cost(INSN_COST);
10645 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10646
10647 ins_encode %{
10648 __ lsl(as_Register($dst$$reg),
10649 as_Register($src1$$reg),
10650 $src2$$constant & 0x3f);
10651 %}
10652
10653 ins_pipe(ialu_reg_shift);
10654 %}
10655
10656 // Shift Right Logical Register
10657 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10658 match(Set dst (URShiftL src1 src2));
10659
10660 ins_cost(INSN_COST * 2);
10661 format %{ "lsrv $dst, $src1, $src2" %}
10662
10663 ins_encode %{
10664 __ lsrv(as_Register($dst$$reg),
10665 as_Register($src1$$reg),
10666 as_Register($src2$$reg));
10667 %}
10668
10669 ins_pipe(ialu_reg_reg_vshift);
10670 %}
10671
10672 // Shift Right Logical Immediate
10673 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10674 match(Set dst (URShiftL src1 src2));
10675
10676 ins_cost(INSN_COST);
10677 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10678
10679 ins_encode %{
10680 __ lsr(as_Register($dst$$reg),
10681 as_Register($src1$$reg),
10682 $src2$$constant & 0x3f);
10683 %}
10684
10685 ins_pipe(ialu_reg_shift);
10686 %}
10687
10688 // A special-case pattern for card table stores.
10689 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10690 match(Set dst (URShiftL (CastP2X src1) src2));
10691
10692 ins_cost(INSN_COST);
10693 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10694
10695 ins_encode %{
10696 __ lsr(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 $src2$$constant & 0x3f);
10699 %}
10700
10701 ins_pipe(ialu_reg_shift);
10702 %}
10703
10704 // Shift Right Arithmetic Register
10705 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10706 match(Set dst (RShiftL src1 src2));
10707
10708 ins_cost(INSN_COST * 2);
10709 format %{ "asrv $dst, $src1, $src2" %}
10710
10711 ins_encode %{
10712 __ asrv(as_Register($dst$$reg),
10713 as_Register($src1$$reg),
10714 as_Register($src2$$reg));
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_vshift);
10718 %}
10719
10720 // Shift Right Arithmetic Immediate
10721 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10722 match(Set dst (RShiftL src1 src2));
10723
10724 ins_cost(INSN_COST);
10725 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10726
10727 ins_encode %{
10728 __ asr(as_Register($dst$$reg),
10729 as_Register($src1$$reg),
10730 $src2$$constant & 0x3f);
10731 %}
10732
10733 ins_pipe(ialu_reg_shift);
10734 %}
10735
10736 // BEGIN This section of the file is automatically generated. Do not edit --------------
10737 // This section is generated from aarch64_ad.m4
10738
10739 // This pattern is automatically generated from aarch64_ad.m4
10740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10741 instruct regL_not_reg(iRegLNoSp dst,
10742 iRegL src1, immL_M1 m1,
10743 rFlagsReg cr) %{
10744 match(Set dst (XorL src1 m1));
10745 ins_cost(INSN_COST);
10746 format %{ "eon $dst, $src1, zr" %}
10747
10748 ins_encode %{
10749 __ eon(as_Register($dst$$reg),
10750 as_Register($src1$$reg),
10751 zr,
10752 Assembler::LSL, 0);
10753 %}
10754
10755 ins_pipe(ialu_reg);
10756 %}
10757
10758 // This pattern is automatically generated from aarch64_ad.m4
10759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10760 instruct regI_not_reg(iRegINoSp dst,
10761 iRegIorL2I src1, immI_M1 m1,
10762 rFlagsReg cr) %{
10763 match(Set dst (XorI src1 m1));
10764 ins_cost(INSN_COST);
10765 format %{ "eonw $dst, $src1, zr" %}
10766
10767 ins_encode %{
10768 __ eonw(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 zr,
10771 Assembler::LSL, 0);
10772 %}
10773
10774 ins_pipe(ialu_reg);
10775 %}
10776
10777 // This pattern is automatically generated from aarch64_ad.m4
10778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10779 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10780 immI0 zero, iRegIorL2I src1, immI src2) %{
10781 match(Set dst (SubI zero (URShiftI src1 src2)));
10782
10783 ins_cost(1.9 * INSN_COST);
10784 format %{ "negw $dst, $src1, LSR $src2" %}
10785
10786 ins_encode %{
10787 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10788 Assembler::LSR, $src2$$constant & 0x1f);
10789 %}
10790
10791 ins_pipe(ialu_reg_shift);
10792 %}
10793
10794 // This pattern is automatically generated from aarch64_ad.m4
10795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10796 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10797 immI0 zero, iRegIorL2I src1, immI src2) %{
10798 match(Set dst (SubI zero (RShiftI src1 src2)));
10799
10800 ins_cost(1.9 * INSN_COST);
10801 format %{ "negw $dst, $src1, ASR $src2" %}
10802
10803 ins_encode %{
10804 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10805 Assembler::ASR, $src2$$constant & 0x1f);
10806 %}
10807
10808 ins_pipe(ialu_reg_shift);
10809 %}
10810
10811 // This pattern is automatically generated from aarch64_ad.m4
10812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10813 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10814 immI0 zero, iRegIorL2I src1, immI src2) %{
10815 match(Set dst (SubI zero (LShiftI src1 src2)));
10816
10817 ins_cost(1.9 * INSN_COST);
10818 format %{ "negw $dst, $src1, LSL $src2" %}
10819
10820 ins_encode %{
10821 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10822 Assembler::LSL, $src2$$constant & 0x1f);
10823 %}
10824
10825 ins_pipe(ialu_reg_shift);
10826 %}
10827
10828 // This pattern is automatically generated from aarch64_ad.m4
10829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10830 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10831 immL0 zero, iRegL src1, immI src2) %{
10832 match(Set dst (SubL zero (URShiftL src1 src2)));
10833
10834 ins_cost(1.9 * INSN_COST);
10835 format %{ "neg $dst, $src1, LSR $src2" %}
10836
10837 ins_encode %{
10838 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10839 Assembler::LSR, $src2$$constant & 0x3f);
10840 %}
10841
10842 ins_pipe(ialu_reg_shift);
10843 %}
10844
10845 // This pattern is automatically generated from aarch64_ad.m4
10846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10847 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10848 immL0 zero, iRegL src1, immI src2) %{
10849 match(Set dst (SubL zero (RShiftL src1 src2)));
10850
10851 ins_cost(1.9 * INSN_COST);
10852 format %{ "neg $dst, $src1, ASR $src2" %}
10853
10854 ins_encode %{
10855 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10856 Assembler::ASR, $src2$$constant & 0x3f);
10857 %}
10858
10859 ins_pipe(ialu_reg_shift);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10865 immL0 zero, iRegL src1, immI src2) %{
10866 match(Set dst (SubL zero (LShiftL src1 src2)));
10867
10868 ins_cost(1.9 * INSN_COST);
10869 format %{ "neg $dst, $src1, LSL $src2" %}
10870
10871 ins_encode %{
10872 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10873 Assembler::LSL, $src2$$constant & 0x3f);
10874 %}
10875
10876 ins_pipe(ialu_reg_shift);
10877 %}
10878
10879 // This pattern is automatically generated from aarch64_ad.m4
10880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10881 instruct AndI_reg_not_reg(iRegINoSp dst,
10882 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10883 match(Set dst (AndI src1 (XorI src2 m1)));
10884 ins_cost(INSN_COST);
10885 format %{ "bicw $dst, $src1, $src2" %}
10886
10887 ins_encode %{
10888 __ bicw(as_Register($dst$$reg),
10889 as_Register($src1$$reg),
10890 as_Register($src2$$reg),
10891 Assembler::LSL, 0);
10892 %}
10893
10894 ins_pipe(ialu_reg_reg);
10895 %}
10896
10897 // This pattern is automatically generated from aarch64_ad.m4
10898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10899 instruct AndL_reg_not_reg(iRegLNoSp dst,
10900 iRegL src1, iRegL src2, immL_M1 m1) %{
10901 match(Set dst (AndL src1 (XorL src2 m1)));
10902 ins_cost(INSN_COST);
10903 format %{ "bic $dst, $src1, $src2" %}
10904
10905 ins_encode %{
10906 __ bic(as_Register($dst$$reg),
10907 as_Register($src1$$reg),
10908 as_Register($src2$$reg),
10909 Assembler::LSL, 0);
10910 %}
10911
10912 ins_pipe(ialu_reg_reg);
10913 %}
10914
10915 // This pattern is automatically generated from aarch64_ad.m4
10916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10917 instruct OrI_reg_not_reg(iRegINoSp dst,
10918 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10919 match(Set dst (OrI src1 (XorI src2 m1)));
10920 ins_cost(INSN_COST);
10921 format %{ "ornw $dst, $src1, $src2" %}
10922
10923 ins_encode %{
10924 __ ornw(as_Register($dst$$reg),
10925 as_Register($src1$$reg),
10926 as_Register($src2$$reg),
10927 Assembler::LSL, 0);
10928 %}
10929
10930 ins_pipe(ialu_reg_reg);
10931 %}
10932
10933 // This pattern is automatically generated from aarch64_ad.m4
10934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10935 instruct OrL_reg_not_reg(iRegLNoSp dst,
10936 iRegL src1, iRegL src2, immL_M1 m1) %{
10937 match(Set dst (OrL src1 (XorL src2 m1)));
10938 ins_cost(INSN_COST);
10939 format %{ "orn $dst, $src1, $src2" %}
10940
10941 ins_encode %{
10942 __ orn(as_Register($dst$$reg),
10943 as_Register($src1$$reg),
10944 as_Register($src2$$reg),
10945 Assembler::LSL, 0);
10946 %}
10947
10948 ins_pipe(ialu_reg_reg);
10949 %}
10950
10951 // This pattern is automatically generated from aarch64_ad.m4
10952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10953 instruct XorI_reg_not_reg(iRegINoSp dst,
10954 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10955 match(Set dst (XorI m1 (XorI src2 src1)));
10956 ins_cost(INSN_COST);
10957 format %{ "eonw $dst, $src1, $src2" %}
10958
10959 ins_encode %{
10960 __ eonw(as_Register($dst$$reg),
10961 as_Register($src1$$reg),
10962 as_Register($src2$$reg),
10963 Assembler::LSL, 0);
10964 %}
10965
10966 ins_pipe(ialu_reg_reg);
10967 %}
10968
10969 // This pattern is automatically generated from aarch64_ad.m4
10970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10971 instruct XorL_reg_not_reg(iRegLNoSp dst,
10972 iRegL src1, iRegL src2, immL_M1 m1) %{
10973 match(Set dst (XorL m1 (XorL src2 src1)));
10974 ins_cost(INSN_COST);
10975 format %{ "eon $dst, $src1, $src2" %}
10976
10977 ins_encode %{
10978 __ eon(as_Register($dst$$reg),
10979 as_Register($src1$$reg),
10980 as_Register($src2$$reg),
10981 Assembler::LSL, 0);
10982 %}
10983
10984 ins_pipe(ialu_reg_reg);
10985 %}
10986
10987 // This pattern is automatically generated from aarch64_ad.m4
10988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10989 // val & (-1 ^ (val >>> shift)) ==> bicw
10990 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10991 iRegIorL2I src1, iRegIorL2I src2,
10992 immI src3, immI_M1 src4) %{
10993 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10994 ins_cost(1.9 * INSN_COST);
10995 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10996
10997 ins_encode %{
10998 __ bicw(as_Register($dst$$reg),
10999 as_Register($src1$$reg),
11000 as_Register($src2$$reg),
11001 Assembler::LSR,
11002 $src3$$constant & 0x1f);
11003 %}
11004
11005 ins_pipe(ialu_reg_reg_shift);
11006 %}
11007
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 // val & (-1 ^ (val >>> shift)) ==> bic
11011 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11012 iRegL src1, iRegL src2,
11013 immI src3, immL_M1 src4) %{
11014 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11015 ins_cost(1.9 * INSN_COST);
11016 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11017
11018 ins_encode %{
11019 __ bic(as_Register($dst$$reg),
11020 as_Register($src1$$reg),
11021 as_Register($src2$$reg),
11022 Assembler::LSR,
11023 $src3$$constant & 0x3f);
11024 %}
11025
11026 ins_pipe(ialu_reg_reg_shift);
11027 %}
11028
11029 // This pattern is automatically generated from aarch64_ad.m4
11030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11031 // val & (-1 ^ (val >> shift)) ==> bicw
11032 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11033 iRegIorL2I src1, iRegIorL2I src2,
11034 immI src3, immI_M1 src4) %{
11035 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11036 ins_cost(1.9 * INSN_COST);
11037 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11038
11039 ins_encode %{
11040 __ bicw(as_Register($dst$$reg),
11041 as_Register($src1$$reg),
11042 as_Register($src2$$reg),
11043 Assembler::ASR,
11044 $src3$$constant & 0x1f);
11045 %}
11046
11047 ins_pipe(ialu_reg_reg_shift);
11048 %}
11049
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 // val & (-1 ^ (val >> shift)) ==> bic
11053 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11054 iRegL src1, iRegL src2,
11055 immI src3, immL_M1 src4) %{
11056 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11057 ins_cost(1.9 * INSN_COST);
11058 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11059
11060 ins_encode %{
11061 __ bic(as_Register($dst$$reg),
11062 as_Register($src1$$reg),
11063 as_Register($src2$$reg),
11064 Assembler::ASR,
11065 $src3$$constant & 0x3f);
11066 %}
11067
11068 ins_pipe(ialu_reg_reg_shift);
11069 %}
11070
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 // val & (-1 ^ (val ror shift)) ==> bicw
11074 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11075 iRegIorL2I src1, iRegIorL2I src2,
11076 immI src3, immI_M1 src4) %{
11077 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11078 ins_cost(1.9 * INSN_COST);
11079 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11080
11081 ins_encode %{
11082 __ bicw(as_Register($dst$$reg),
11083 as_Register($src1$$reg),
11084 as_Register($src2$$reg),
11085 Assembler::ROR,
11086 $src3$$constant & 0x1f);
11087 %}
11088
11089 ins_pipe(ialu_reg_reg_shift);
11090 %}
11091
11092 // This pattern is automatically generated from aarch64_ad.m4
11093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11094 // val & (-1 ^ (val ror shift)) ==> bic
11095 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11096 iRegL src1, iRegL src2,
11097 immI src3, immL_M1 src4) %{
11098 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11099 ins_cost(1.9 * INSN_COST);
11100 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11101
11102 ins_encode %{
11103 __ bic(as_Register($dst$$reg),
11104 as_Register($src1$$reg),
11105 as_Register($src2$$reg),
11106 Assembler::ROR,
11107 $src3$$constant & 0x3f);
11108 %}
11109
11110 ins_pipe(ialu_reg_reg_shift);
11111 %}
11112
11113 // This pattern is automatically generated from aarch64_ad.m4
11114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11115 // val & (-1 ^ (val << shift)) ==> bicw
11116 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11117 iRegIorL2I src1, iRegIorL2I src2,
11118 immI src3, immI_M1 src4) %{
11119 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11120 ins_cost(1.9 * INSN_COST);
11121 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11122
11123 ins_encode %{
11124 __ bicw(as_Register($dst$$reg),
11125 as_Register($src1$$reg),
11126 as_Register($src2$$reg),
11127 Assembler::LSL,
11128 $src3$$constant & 0x1f);
11129 %}
11130
11131 ins_pipe(ialu_reg_reg_shift);
11132 %}
11133
11134 // This pattern is automatically generated from aarch64_ad.m4
11135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11136 // val & (-1 ^ (val << shift)) ==> bic
11137 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11138 iRegL src1, iRegL src2,
11139 immI src3, immL_M1 src4) %{
11140 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11141 ins_cost(1.9 * INSN_COST);
11142 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11143
11144 ins_encode %{
11145 __ bic(as_Register($dst$$reg),
11146 as_Register($src1$$reg),
11147 as_Register($src2$$reg),
11148 Assembler::LSL,
11149 $src3$$constant & 0x3f);
11150 %}
11151
11152 ins_pipe(ialu_reg_reg_shift);
11153 %}
11154
11155 // This pattern is automatically generated from aarch64_ad.m4
11156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11157 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11158 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11159 iRegIorL2I src1, iRegIorL2I src2,
11160 immI src3, immI_M1 src4) %{
11161 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11162 ins_cost(1.9 * INSN_COST);
11163 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11164
11165 ins_encode %{
11166 __ eonw(as_Register($dst$$reg),
11167 as_Register($src1$$reg),
11168 as_Register($src2$$reg),
11169 Assembler::LSR,
11170 $src3$$constant & 0x1f);
11171 %}
11172
11173 ins_pipe(ialu_reg_reg_shift);
11174 %}
11175
11176 // This pattern is automatically generated from aarch64_ad.m4
11177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11178 // val ^ (-1 ^ (val >>> shift)) ==> eon
11179 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11180 iRegL src1, iRegL src2,
11181 immI src3, immL_M1 src4) %{
11182 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11183 ins_cost(1.9 * INSN_COST);
11184 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11185
11186 ins_encode %{
11187 __ eon(as_Register($dst$$reg),
11188 as_Register($src1$$reg),
11189 as_Register($src2$$reg),
11190 Assembler::LSR,
11191 $src3$$constant & 0x3f);
11192 %}
11193
11194 ins_pipe(ialu_reg_reg_shift);
11195 %}
11196
11197 // This pattern is automatically generated from aarch64_ad.m4
11198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11199 // val ^ (-1 ^ (val >> shift)) ==> eonw
11200 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11201 iRegIorL2I src1, iRegIorL2I src2,
11202 immI src3, immI_M1 src4) %{
11203 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11204 ins_cost(1.9 * INSN_COST);
11205 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11206
11207 ins_encode %{
11208 __ eonw(as_Register($dst$$reg),
11209 as_Register($src1$$reg),
11210 as_Register($src2$$reg),
11211 Assembler::ASR,
11212 $src3$$constant & 0x1f);
11213 %}
11214
11215 ins_pipe(ialu_reg_reg_shift);
11216 %}
11217
11218 // This pattern is automatically generated from aarch64_ad.m4
11219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11220 // val ^ (-1 ^ (val >> shift)) ==> eon
11221 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11222 iRegL src1, iRegL src2,
11223 immI src3, immL_M1 src4) %{
11224 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11225 ins_cost(1.9 * INSN_COST);
11226 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11227
11228 ins_encode %{
11229 __ eon(as_Register($dst$$reg),
11230 as_Register($src1$$reg),
11231 as_Register($src2$$reg),
11232 Assembler::ASR,
11233 $src3$$constant & 0x3f);
11234 %}
11235
11236 ins_pipe(ialu_reg_reg_shift);
11237 %}
11238
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 // val ^ (-1 ^ (val ror shift)) ==> eonw
11242 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11243 iRegIorL2I src1, iRegIorL2I src2,
11244 immI src3, immI_M1 src4) %{
11245 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11246 ins_cost(1.9 * INSN_COST);
11247 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11248
11249 ins_encode %{
11250 __ eonw(as_Register($dst$$reg),
11251 as_Register($src1$$reg),
11252 as_Register($src2$$reg),
11253 Assembler::ROR,
11254 $src3$$constant & 0x1f);
11255 %}
11256
11257 ins_pipe(ialu_reg_reg_shift);
11258 %}
11259
11260 // This pattern is automatically generated from aarch64_ad.m4
11261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11262 // val ^ (-1 ^ (val ror shift)) ==> eon
11263 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11264 iRegL src1, iRegL src2,
11265 immI src3, immL_M1 src4) %{
11266 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11267 ins_cost(1.9 * INSN_COST);
11268 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11269
11270 ins_encode %{
11271 __ eon(as_Register($dst$$reg),
11272 as_Register($src1$$reg),
11273 as_Register($src2$$reg),
11274 Assembler::ROR,
11275 $src3$$constant & 0x3f);
11276 %}
11277
11278 ins_pipe(ialu_reg_reg_shift);
11279 %}
11280
11281 // This pattern is automatically generated from aarch64_ad.m4
11282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11283 // val ^ (-1 ^ (val << shift)) ==> eonw
11284 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11285 iRegIorL2I src1, iRegIorL2I src2,
11286 immI src3, immI_M1 src4) %{
11287 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11288 ins_cost(1.9 * INSN_COST);
11289 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11290
11291 ins_encode %{
11292 __ eonw(as_Register($dst$$reg),
11293 as_Register($src1$$reg),
11294 as_Register($src2$$reg),
11295 Assembler::LSL,
11296 $src3$$constant & 0x1f);
11297 %}
11298
11299 ins_pipe(ialu_reg_reg_shift);
11300 %}
11301
11302 // This pattern is automatically generated from aarch64_ad.m4
11303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11304 // val ^ (-1 ^ (val << shift)) ==> eon
11305 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11306 iRegL src1, iRegL src2,
11307 immI src3, immL_M1 src4) %{
11308 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11309 ins_cost(1.9 * INSN_COST);
11310 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11311
11312 ins_encode %{
11313 __ eon(as_Register($dst$$reg),
11314 as_Register($src1$$reg),
11315 as_Register($src2$$reg),
11316 Assembler::LSL,
11317 $src3$$constant & 0x3f);
11318 %}
11319
11320 ins_pipe(ialu_reg_reg_shift);
11321 %}
11322
11323 // This pattern is automatically generated from aarch64_ad.m4
11324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11325 // val | (-1 ^ (val >>> shift)) ==> ornw
11326 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11327 iRegIorL2I src1, iRegIorL2I src2,
11328 immI src3, immI_M1 src4) %{
11329 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11330 ins_cost(1.9 * INSN_COST);
11331 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11332
11333 ins_encode %{
11334 __ ornw(as_Register($dst$$reg),
11335 as_Register($src1$$reg),
11336 as_Register($src2$$reg),
11337 Assembler::LSR,
11338 $src3$$constant & 0x1f);
11339 %}
11340
11341 ins_pipe(ialu_reg_reg_shift);
11342 %}
11343
11344 // This pattern is automatically generated from aarch64_ad.m4
11345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11346 // val | (-1 ^ (val >>> shift)) ==> orn
11347 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11348 iRegL src1, iRegL src2,
11349 immI src3, immL_M1 src4) %{
11350 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11351 ins_cost(1.9 * INSN_COST);
11352 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11353
11354 ins_encode %{
11355 __ orn(as_Register($dst$$reg),
11356 as_Register($src1$$reg),
11357 as_Register($src2$$reg),
11358 Assembler::LSR,
11359 $src3$$constant & 0x3f);
11360 %}
11361
11362 ins_pipe(ialu_reg_reg_shift);
11363 %}
11364
11365 // This pattern is automatically generated from aarch64_ad.m4
11366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11367 // val | (-1 ^ (val >> shift)) ==> ornw
11368 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11369 iRegIorL2I src1, iRegIorL2I src2,
11370 immI src3, immI_M1 src4) %{
11371 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11372 ins_cost(1.9 * INSN_COST);
11373 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11374
11375 ins_encode %{
11376 __ ornw(as_Register($dst$$reg),
11377 as_Register($src1$$reg),
11378 as_Register($src2$$reg),
11379 Assembler::ASR,
11380 $src3$$constant & 0x1f);
11381 %}
11382
11383 ins_pipe(ialu_reg_reg_shift);
11384 %}
11385
11386 // This pattern is automatically generated from aarch64_ad.m4
11387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11388 // val | (-1 ^ (val >> shift)) ==> orn
11389 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11390 iRegL src1, iRegL src2,
11391 immI src3, immL_M1 src4) %{
11392 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11393 ins_cost(1.9 * INSN_COST);
11394 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11395
11396 ins_encode %{
11397 __ orn(as_Register($dst$$reg),
11398 as_Register($src1$$reg),
11399 as_Register($src2$$reg),
11400 Assembler::ASR,
11401 $src3$$constant & 0x3f);
11402 %}
11403
11404 ins_pipe(ialu_reg_reg_shift);
11405 %}
11406
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409 // val | (-1 ^ (val ror shift)) ==> ornw
11410 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11411 iRegIorL2I src1, iRegIorL2I src2,
11412 immI src3, immI_M1 src4) %{
11413 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11414 ins_cost(1.9 * INSN_COST);
11415 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11416
11417 ins_encode %{
11418 __ ornw(as_Register($dst$$reg),
11419 as_Register($src1$$reg),
11420 as_Register($src2$$reg),
11421 Assembler::ROR,
11422 $src3$$constant & 0x1f);
11423 %}
11424
11425 ins_pipe(ialu_reg_reg_shift);
11426 %}
11427
11428 // This pattern is automatically generated from aarch64_ad.m4
11429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11430 // val | (-1 ^ (val ror shift)) ==> orn
11431 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11432 iRegL src1, iRegL src2,
11433 immI src3, immL_M1 src4) %{
11434 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11435 ins_cost(1.9 * INSN_COST);
11436 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11437
11438 ins_encode %{
11439 __ orn(as_Register($dst$$reg),
11440 as_Register($src1$$reg),
11441 as_Register($src2$$reg),
11442 Assembler::ROR,
11443 $src3$$constant & 0x3f);
11444 %}
11445
11446 ins_pipe(ialu_reg_reg_shift);
11447 %}
11448
11449 // This pattern is automatically generated from aarch64_ad.m4
11450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11451 // val | (-1 ^ (val << shift)) ==> ornw
11452 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11453 iRegIorL2I src1, iRegIorL2I src2,
11454 immI src3, immI_M1 src4) %{
11455 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11456 ins_cost(1.9 * INSN_COST);
11457 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11458
11459 ins_encode %{
11460 __ ornw(as_Register($dst$$reg),
11461 as_Register($src1$$reg),
11462 as_Register($src2$$reg),
11463 Assembler::LSL,
11464 $src3$$constant & 0x1f);
11465 %}
11466
11467 ins_pipe(ialu_reg_reg_shift);
11468 %}
11469
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 // val | (-1 ^ (val << shift)) ==> orn
11473 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11474 iRegL src1, iRegL src2,
11475 immI src3, immL_M1 src4) %{
11476 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11477 ins_cost(1.9 * INSN_COST);
11478 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11479
11480 ins_encode %{
11481 __ orn(as_Register($dst$$reg),
11482 as_Register($src1$$reg),
11483 as_Register($src2$$reg),
11484 Assembler::LSL,
11485 $src3$$constant & 0x3f);
11486 %}
11487
11488 ins_pipe(ialu_reg_reg_shift);
11489 %}
11490
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11494 iRegIorL2I src1, iRegIorL2I src2,
11495 immI src3) %{
11496 match(Set dst (AndI src1 (URShiftI src2 src3)));
11497
11498 ins_cost(1.9 * INSN_COST);
11499 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11500
11501 ins_encode %{
11502 __ andw(as_Register($dst$$reg),
11503 as_Register($src1$$reg),
11504 as_Register($src2$$reg),
11505 Assembler::LSR,
11506 $src3$$constant & 0x1f);
11507 %}
11508
11509 ins_pipe(ialu_reg_reg_shift);
11510 %}
11511
11512 // This pattern is automatically generated from aarch64_ad.m4
11513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11514 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11515 iRegL src1, iRegL src2,
11516 immI src3) %{
11517 match(Set dst (AndL src1 (URShiftL src2 src3)));
11518
11519 ins_cost(1.9 * INSN_COST);
11520 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11521
11522 ins_encode %{
11523 __ andr(as_Register($dst$$reg),
11524 as_Register($src1$$reg),
11525 as_Register($src2$$reg),
11526 Assembler::LSR,
11527 $src3$$constant & 0x3f);
11528 %}
11529
11530 ins_pipe(ialu_reg_reg_shift);
11531 %}
11532
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11536 iRegIorL2I src1, iRegIorL2I src2,
11537 immI src3) %{
11538 match(Set dst (AndI src1 (RShiftI src2 src3)));
11539
11540 ins_cost(1.9 * INSN_COST);
11541 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11542
11543 ins_encode %{
11544 __ andw(as_Register($dst$$reg),
11545 as_Register($src1$$reg),
11546 as_Register($src2$$reg),
11547 Assembler::ASR,
11548 $src3$$constant & 0x1f);
11549 %}
11550
11551 ins_pipe(ialu_reg_reg_shift);
11552 %}
11553
11554 // This pattern is automatically generated from aarch64_ad.m4
11555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11556 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11557 iRegL src1, iRegL src2,
11558 immI src3) %{
11559 match(Set dst (AndL src1 (RShiftL src2 src3)));
11560
11561 ins_cost(1.9 * INSN_COST);
11562 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11563
11564 ins_encode %{
11565 __ andr(as_Register($dst$$reg),
11566 as_Register($src1$$reg),
11567 as_Register($src2$$reg),
11568 Assembler::ASR,
11569 $src3$$constant & 0x3f);
11570 %}
11571
11572 ins_pipe(ialu_reg_reg_shift);
11573 %}
11574
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11578 iRegIorL2I src1, iRegIorL2I src2,
11579 immI src3) %{
11580 match(Set dst (AndI src1 (LShiftI src2 src3)));
11581
11582 ins_cost(1.9 * INSN_COST);
11583 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11584
11585 ins_encode %{
11586 __ andw(as_Register($dst$$reg),
11587 as_Register($src1$$reg),
11588 as_Register($src2$$reg),
11589 Assembler::LSL,
11590 $src3$$constant & 0x1f);
11591 %}
11592
11593 ins_pipe(ialu_reg_reg_shift);
11594 %}
11595
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11599 iRegL src1, iRegL src2,
11600 immI src3) %{
11601 match(Set dst (AndL src1 (LShiftL src2 src3)));
11602
11603 ins_cost(1.9 * INSN_COST);
11604 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11605
11606 ins_encode %{
11607 __ andr(as_Register($dst$$reg),
11608 as_Register($src1$$reg),
11609 as_Register($src2$$reg),
11610 Assembler::LSL,
11611 $src3$$constant & 0x3f);
11612 %}
11613
11614 ins_pipe(ialu_reg_reg_shift);
11615 %}
11616
11617 // This pattern is automatically generated from aarch64_ad.m4
11618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11619 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11620 iRegIorL2I src1, iRegIorL2I src2,
11621 immI src3) %{
11622 match(Set dst (AndI src1 (RotateRight src2 src3)));
11623
11624 ins_cost(1.9 * INSN_COST);
11625 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11626
11627 ins_encode %{
11628 __ andw(as_Register($dst$$reg),
11629 as_Register($src1$$reg),
11630 as_Register($src2$$reg),
11631 Assembler::ROR,
11632 $src3$$constant & 0x1f);
11633 %}
11634
11635 ins_pipe(ialu_reg_reg_shift);
11636 %}
11637
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11641 iRegL src1, iRegL src2,
11642 immI src3) %{
11643 match(Set dst (AndL src1 (RotateRight src2 src3)));
11644
11645 ins_cost(1.9 * INSN_COST);
11646 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11647
11648 ins_encode %{
11649 __ andr(as_Register($dst$$reg),
11650 as_Register($src1$$reg),
11651 as_Register($src2$$reg),
11652 Assembler::ROR,
11653 $src3$$constant & 0x3f);
11654 %}
11655
11656 ins_pipe(ialu_reg_reg_shift);
11657 %}
11658
11659 // This pattern is automatically generated from aarch64_ad.m4
11660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11661 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11662 iRegIorL2I src1, iRegIorL2I src2,
11663 immI src3) %{
11664 match(Set dst (XorI src1 (URShiftI src2 src3)));
11665
11666 ins_cost(1.9 * INSN_COST);
11667 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11668
11669 ins_encode %{
11670 __ eorw(as_Register($dst$$reg),
11671 as_Register($src1$$reg),
11672 as_Register($src2$$reg),
11673 Assembler::LSR,
11674 $src3$$constant & 0x1f);
11675 %}
11676
11677 ins_pipe(ialu_reg_reg_shift);
11678 %}
11679
11680 // This pattern is automatically generated from aarch64_ad.m4
11681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11682 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11683 iRegL src1, iRegL src2,
11684 immI src3) %{
11685 match(Set dst (XorL src1 (URShiftL src2 src3)));
11686
11687 ins_cost(1.9 * INSN_COST);
11688 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11689
11690 ins_encode %{
11691 __ eor(as_Register($dst$$reg),
11692 as_Register($src1$$reg),
11693 as_Register($src2$$reg),
11694 Assembler::LSR,
11695 $src3$$constant & 0x3f);
11696 %}
11697
11698 ins_pipe(ialu_reg_reg_shift);
11699 %}
11700
11701 // This pattern is automatically generated from aarch64_ad.m4
11702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11703 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11704 iRegIorL2I src1, iRegIorL2I src2,
11705 immI src3) %{
11706 match(Set dst (XorI src1 (RShiftI src2 src3)));
11707
11708 ins_cost(1.9 * INSN_COST);
11709 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11710
11711 ins_encode %{
11712 __ eorw(as_Register($dst$$reg),
11713 as_Register($src1$$reg),
11714 as_Register($src2$$reg),
11715 Assembler::ASR,
11716 $src3$$constant & 0x1f);
11717 %}
11718
11719 ins_pipe(ialu_reg_reg_shift);
11720 %}
11721
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11725 iRegL src1, iRegL src2,
11726 immI src3) %{
11727 match(Set dst (XorL src1 (RShiftL src2 src3)));
11728
11729 ins_cost(1.9 * INSN_COST);
11730 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11731
11732 ins_encode %{
11733 __ eor(as_Register($dst$$reg),
11734 as_Register($src1$$reg),
11735 as_Register($src2$$reg),
11736 Assembler::ASR,
11737 $src3$$constant & 0x3f);
11738 %}
11739
11740 ins_pipe(ialu_reg_reg_shift);
11741 %}
11742
11743 // This pattern is automatically generated from aarch64_ad.m4
11744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11745 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11746 iRegIorL2I src1, iRegIorL2I src2,
11747 immI src3) %{
11748 match(Set dst (XorI src1 (LShiftI src2 src3)));
11749
11750 ins_cost(1.9 * INSN_COST);
11751 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11752
11753 ins_encode %{
11754 __ eorw(as_Register($dst$$reg),
11755 as_Register($src1$$reg),
11756 as_Register($src2$$reg),
11757 Assembler::LSL,
11758 $src3$$constant & 0x1f);
11759 %}
11760
11761 ins_pipe(ialu_reg_reg_shift);
11762 %}
11763
11764 // This pattern is automatically generated from aarch64_ad.m4
11765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11766 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11767 iRegL src1, iRegL src2,
11768 immI src3) %{
11769 match(Set dst (XorL src1 (LShiftL src2 src3)));
11770
11771 ins_cost(1.9 * INSN_COST);
11772 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11773
11774 ins_encode %{
11775 __ eor(as_Register($dst$$reg),
11776 as_Register($src1$$reg),
11777 as_Register($src2$$reg),
11778 Assembler::LSL,
11779 $src3$$constant & 0x3f);
11780 %}
11781
11782 ins_pipe(ialu_reg_reg_shift);
11783 %}
11784
11785 // This pattern is automatically generated from aarch64_ad.m4
11786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11787 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11788 iRegIorL2I src1, iRegIorL2I src2,
11789 immI src3) %{
11790 match(Set dst (XorI src1 (RotateRight src2 src3)));
11791
11792 ins_cost(1.9 * INSN_COST);
11793 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11794
11795 ins_encode %{
11796 __ eorw(as_Register($dst$$reg),
11797 as_Register($src1$$reg),
11798 as_Register($src2$$reg),
11799 Assembler::ROR,
11800 $src3$$constant & 0x1f);
11801 %}
11802
11803 ins_pipe(ialu_reg_reg_shift);
11804 %}
11805
11806 // This pattern is automatically generated from aarch64_ad.m4
11807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11808 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11809 iRegL src1, iRegL src2,
11810 immI src3) %{
11811 match(Set dst (XorL src1 (RotateRight src2 src3)));
11812
11813 ins_cost(1.9 * INSN_COST);
11814 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11815
11816 ins_encode %{
11817 __ eor(as_Register($dst$$reg),
11818 as_Register($src1$$reg),
11819 as_Register($src2$$reg),
11820 Assembler::ROR,
11821 $src3$$constant & 0x3f);
11822 %}
11823
11824 ins_pipe(ialu_reg_reg_shift);
11825 %}
11826
11827 // This pattern is automatically generated from aarch64_ad.m4
11828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11829 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11830 iRegIorL2I src1, iRegIorL2I src2,
11831 immI src3) %{
11832 match(Set dst (OrI src1 (URShiftI src2 src3)));
11833
11834 ins_cost(1.9 * INSN_COST);
11835 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11836
11837 ins_encode %{
11838 __ orrw(as_Register($dst$$reg),
11839 as_Register($src1$$reg),
11840 as_Register($src2$$reg),
11841 Assembler::LSR,
11842 $src3$$constant & 0x1f);
11843 %}
11844
11845 ins_pipe(ialu_reg_reg_shift);
11846 %}
11847
11848 // This pattern is automatically generated from aarch64_ad.m4
11849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11850 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11851 iRegL src1, iRegL src2,
11852 immI src3) %{
11853 match(Set dst (OrL src1 (URShiftL src2 src3)));
11854
11855 ins_cost(1.9 * INSN_COST);
11856 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11857
11858 ins_encode %{
11859 __ orr(as_Register($dst$$reg),
11860 as_Register($src1$$reg),
11861 as_Register($src2$$reg),
11862 Assembler::LSR,
11863 $src3$$constant & 0x3f);
11864 %}
11865
11866 ins_pipe(ialu_reg_reg_shift);
11867 %}
11868
11869 // This pattern is automatically generated from aarch64_ad.m4
11870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11871 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11872 iRegIorL2I src1, iRegIorL2I src2,
11873 immI src3) %{
11874 match(Set dst (OrI src1 (RShiftI src2 src3)));
11875
11876 ins_cost(1.9 * INSN_COST);
11877 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11878
11879 ins_encode %{
11880 __ orrw(as_Register($dst$$reg),
11881 as_Register($src1$$reg),
11882 as_Register($src2$$reg),
11883 Assembler::ASR,
11884 $src3$$constant & 0x1f);
11885 %}
11886
11887 ins_pipe(ialu_reg_reg_shift);
11888 %}
11889
11890 // This pattern is automatically generated from aarch64_ad.m4
11891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11892 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11893 iRegL src1, iRegL src2,
11894 immI src3) %{
11895 match(Set dst (OrL src1 (RShiftL src2 src3)));
11896
11897 ins_cost(1.9 * INSN_COST);
11898 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11899
11900 ins_encode %{
11901 __ orr(as_Register($dst$$reg),
11902 as_Register($src1$$reg),
11903 as_Register($src2$$reg),
11904 Assembler::ASR,
11905 $src3$$constant & 0x3f);
11906 %}
11907
11908 ins_pipe(ialu_reg_reg_shift);
11909 %}
11910
11911 // This pattern is automatically generated from aarch64_ad.m4
11912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11913 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11914 iRegIorL2I src1, iRegIorL2I src2,
11915 immI src3) %{
11916 match(Set dst (OrI src1 (LShiftI src2 src3)));
11917
11918 ins_cost(1.9 * INSN_COST);
11919 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11920
11921 ins_encode %{
11922 __ orrw(as_Register($dst$$reg),
11923 as_Register($src1$$reg),
11924 as_Register($src2$$reg),
11925 Assembler::LSL,
11926 $src3$$constant & 0x1f);
11927 %}
11928
11929 ins_pipe(ialu_reg_reg_shift);
11930 %}
11931
11932 // This pattern is automatically generated from aarch64_ad.m4
11933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11934 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11935 iRegL src1, iRegL src2,
11936 immI src3) %{
11937 match(Set dst (OrL src1 (LShiftL src2 src3)));
11938
11939 ins_cost(1.9 * INSN_COST);
11940 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11941
11942 ins_encode %{
11943 __ orr(as_Register($dst$$reg),
11944 as_Register($src1$$reg),
11945 as_Register($src2$$reg),
11946 Assembler::LSL,
11947 $src3$$constant & 0x3f);
11948 %}
11949
11950 ins_pipe(ialu_reg_reg_shift);
11951 %}
11952
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11956 iRegIorL2I src1, iRegIorL2I src2,
11957 immI src3) %{
11958 match(Set dst (OrI src1 (RotateRight src2 src3)));
11959
11960 ins_cost(1.9 * INSN_COST);
11961 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11962
11963 ins_encode %{
11964 __ orrw(as_Register($dst$$reg),
11965 as_Register($src1$$reg),
11966 as_Register($src2$$reg),
11967 Assembler::ROR,
11968 $src3$$constant & 0x1f);
11969 %}
11970
11971 ins_pipe(ialu_reg_reg_shift);
11972 %}
11973
11974 // This pattern is automatically generated from aarch64_ad.m4
11975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11976 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11977 iRegL src1, iRegL src2,
11978 immI src3) %{
11979 match(Set dst (OrL src1 (RotateRight src2 src3)));
11980
11981 ins_cost(1.9 * INSN_COST);
11982 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11983
11984 ins_encode %{
11985 __ orr(as_Register($dst$$reg),
11986 as_Register($src1$$reg),
11987 as_Register($src2$$reg),
11988 Assembler::ROR,
11989 $src3$$constant & 0x3f);
11990 %}
11991
11992 ins_pipe(ialu_reg_reg_shift);
11993 %}
11994
11995 // This pattern is automatically generated from aarch64_ad.m4
11996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11997 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11998 iRegIorL2I src1, iRegIorL2I src2,
11999 immI src3) %{
12000 match(Set dst (AddI src1 (URShiftI src2 src3)));
12001
12002 ins_cost(1.9 * INSN_COST);
12003 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12004
12005 ins_encode %{
12006 __ addw(as_Register($dst$$reg),
12007 as_Register($src1$$reg),
12008 as_Register($src2$$reg),
12009 Assembler::LSR,
12010 $src3$$constant & 0x1f);
12011 %}
12012
12013 ins_pipe(ialu_reg_reg_shift);
12014 %}
12015
12016 // This pattern is automatically generated from aarch64_ad.m4
12017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12018 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12019 iRegL src1, iRegL src2,
12020 immI src3) %{
12021 match(Set dst (AddL src1 (URShiftL src2 src3)));
12022
12023 ins_cost(1.9 * INSN_COST);
12024 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12025
12026 ins_encode %{
12027 __ add(as_Register($dst$$reg),
12028 as_Register($src1$$reg),
12029 as_Register($src2$$reg),
12030 Assembler::LSR,
12031 $src3$$constant & 0x3f);
12032 %}
12033
12034 ins_pipe(ialu_reg_reg_shift);
12035 %}
12036
12037 // This pattern is automatically generated from aarch64_ad.m4
12038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12039 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12040 iRegIorL2I src1, iRegIorL2I src2,
12041 immI src3) %{
12042 match(Set dst (AddI src1 (RShiftI src2 src3)));
12043
12044 ins_cost(1.9 * INSN_COST);
12045 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12046
12047 ins_encode %{
12048 __ addw(as_Register($dst$$reg),
12049 as_Register($src1$$reg),
12050 as_Register($src2$$reg),
12051 Assembler::ASR,
12052 $src3$$constant & 0x1f);
12053 %}
12054
12055 ins_pipe(ialu_reg_reg_shift);
12056 %}
12057
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12061 iRegL src1, iRegL src2,
12062 immI src3) %{
12063 match(Set dst (AddL src1 (RShiftL src2 src3)));
12064
12065 ins_cost(1.9 * INSN_COST);
12066 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12067
12068 ins_encode %{
12069 __ add(as_Register($dst$$reg),
12070 as_Register($src1$$reg),
12071 as_Register($src2$$reg),
12072 Assembler::ASR,
12073 $src3$$constant & 0x3f);
12074 %}
12075
12076 ins_pipe(ialu_reg_reg_shift);
12077 %}
12078
12079 // This pattern is automatically generated from aarch64_ad.m4
12080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12081 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12082 iRegIorL2I src1, iRegIorL2I src2,
12083 immI src3) %{
12084 match(Set dst (AddI src1 (LShiftI src2 src3)));
12085
12086 ins_cost(1.9 * INSN_COST);
12087 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12088
12089 ins_encode %{
12090 __ addw(as_Register($dst$$reg),
12091 as_Register($src1$$reg),
12092 as_Register($src2$$reg),
12093 Assembler::LSL,
12094 $src3$$constant & 0x1f);
12095 %}
12096
12097 ins_pipe(ialu_reg_reg_shift);
12098 %}
12099
12100 // This pattern is automatically generated from aarch64_ad.m4
12101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12102 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12103 iRegL src1, iRegL src2,
12104 immI src3) %{
12105 match(Set dst (AddL src1 (LShiftL src2 src3)));
12106
12107 ins_cost(1.9 * INSN_COST);
12108 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12109
12110 ins_encode %{
12111 __ add(as_Register($dst$$reg),
12112 as_Register($src1$$reg),
12113 as_Register($src2$$reg),
12114 Assembler::LSL,
12115 $src3$$constant & 0x3f);
12116 %}
12117
12118 ins_pipe(ialu_reg_reg_shift);
12119 %}
12120
12121 // This pattern is automatically generated from aarch64_ad.m4
12122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12123 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12124 iRegIorL2I src1, iRegIorL2I src2,
12125 immI src3) %{
12126 match(Set dst (SubI src1 (URShiftI src2 src3)));
12127
12128 ins_cost(1.9 * INSN_COST);
12129 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12130
12131 ins_encode %{
12132 __ subw(as_Register($dst$$reg),
12133 as_Register($src1$$reg),
12134 as_Register($src2$$reg),
12135 Assembler::LSR,
12136 $src3$$constant & 0x1f);
12137 %}
12138
12139 ins_pipe(ialu_reg_reg_shift);
12140 %}
12141
12142 // This pattern is automatically generated from aarch64_ad.m4
12143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12144 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12145 iRegL src1, iRegL src2,
12146 immI src3) %{
12147 match(Set dst (SubL src1 (URShiftL src2 src3)));
12148
12149 ins_cost(1.9 * INSN_COST);
12150 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12151
12152 ins_encode %{
12153 __ sub(as_Register($dst$$reg),
12154 as_Register($src1$$reg),
12155 as_Register($src2$$reg),
12156 Assembler::LSR,
12157 $src3$$constant & 0x3f);
12158 %}
12159
12160 ins_pipe(ialu_reg_reg_shift);
12161 %}
12162
12163 // This pattern is automatically generated from aarch64_ad.m4
12164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12165 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12166 iRegIorL2I src1, iRegIorL2I src2,
12167 immI src3) %{
12168 match(Set dst (SubI src1 (RShiftI src2 src3)));
12169
12170 ins_cost(1.9 * INSN_COST);
12171 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12172
12173 ins_encode %{
12174 __ subw(as_Register($dst$$reg),
12175 as_Register($src1$$reg),
12176 as_Register($src2$$reg),
12177 Assembler::ASR,
12178 $src3$$constant & 0x1f);
12179 %}
12180
12181 ins_pipe(ialu_reg_reg_shift);
12182 %}
12183
12184 // This pattern is automatically generated from aarch64_ad.m4
12185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12186 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12187 iRegL src1, iRegL src2,
12188 immI src3) %{
12189 match(Set dst (SubL src1 (RShiftL src2 src3)));
12190
12191 ins_cost(1.9 * INSN_COST);
12192 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12193
12194 ins_encode %{
12195 __ sub(as_Register($dst$$reg),
12196 as_Register($src1$$reg),
12197 as_Register($src2$$reg),
12198 Assembler::ASR,
12199 $src3$$constant & 0x3f);
12200 %}
12201
12202 ins_pipe(ialu_reg_reg_shift);
12203 %}
12204
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12208 iRegIorL2I src1, iRegIorL2I src2,
12209 immI src3) %{
12210 match(Set dst (SubI src1 (LShiftI src2 src3)));
12211
12212 ins_cost(1.9 * INSN_COST);
12213 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12214
12215 ins_encode %{
12216 __ subw(as_Register($dst$$reg),
12217 as_Register($src1$$reg),
12218 as_Register($src2$$reg),
12219 Assembler::LSL,
12220 $src3$$constant & 0x1f);
12221 %}
12222
12223 ins_pipe(ialu_reg_reg_shift);
12224 %}
12225
12226 // This pattern is automatically generated from aarch64_ad.m4
12227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12228 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12229 iRegL src1, iRegL src2,
12230 immI src3) %{
12231 match(Set dst (SubL src1 (LShiftL src2 src3)));
12232
12233 ins_cost(1.9 * INSN_COST);
12234 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12235
12236 ins_encode %{
12237 __ sub(as_Register($dst$$reg),
12238 as_Register($src1$$reg),
12239 as_Register($src2$$reg),
12240 Assembler::LSL,
12241 $src3$$constant & 0x3f);
12242 %}
12243
12244 ins_pipe(ialu_reg_reg_shift);
12245 %}
12246
12247 // This pattern is automatically generated from aarch64_ad.m4
12248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12249
12250 // Shift Left followed by Shift Right.
12251 // This idiom is used by the compiler for the i2b bytecode etc.
12252 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12253 %{
12254 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12255 ins_cost(INSN_COST * 2);
12256 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12257 ins_encode %{
12258 int lshift = $lshift_count$$constant & 63;
12259 int rshift = $rshift_count$$constant & 63;
12260 int s = 63 - lshift;
12261 int r = (rshift - lshift) & 63;
12262 __ sbfm(as_Register($dst$$reg),
12263 as_Register($src$$reg),
12264 r, s);
12265 %}
12266
12267 ins_pipe(ialu_reg_shift);
12268 %}
12269
12270 // This pattern is automatically generated from aarch64_ad.m4
12271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12272
12273 // Shift Left followed by Shift Right.
12274 // This idiom is used by the compiler for the i2b bytecode etc.
12275 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12276 %{
12277 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12278 ins_cost(INSN_COST * 2);
12279 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12280 ins_encode %{
12281 int lshift = $lshift_count$$constant & 31;
12282 int rshift = $rshift_count$$constant & 31;
12283 int s = 31 - lshift;
12284 int r = (rshift - lshift) & 31;
12285 __ sbfmw(as_Register($dst$$reg),
12286 as_Register($src$$reg),
12287 r, s);
12288 %}
12289
12290 ins_pipe(ialu_reg_shift);
12291 %}
12292
12293 // This pattern is automatically generated from aarch64_ad.m4
12294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12295
12296 // Shift Left followed by Shift Right.
12297 // This idiom is used by the compiler for the i2b bytecode etc.
12298 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12299 %{
12300 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12301 ins_cost(INSN_COST * 2);
12302 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12303 ins_encode %{
12304 int lshift = $lshift_count$$constant & 63;
12305 int rshift = $rshift_count$$constant & 63;
12306 int s = 63 - lshift;
12307 int r = (rshift - lshift) & 63;
12308 __ ubfm(as_Register($dst$$reg),
12309 as_Register($src$$reg),
12310 r, s);
12311 %}
12312
12313 ins_pipe(ialu_reg_shift);
12314 %}
12315
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318
12319 // Shift Left followed by Shift Right.
12320 // This idiom is used by the compiler for the i2b bytecode etc.
12321 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12322 %{
12323 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12324 ins_cost(INSN_COST * 2);
12325 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12326 ins_encode %{
12327 int lshift = $lshift_count$$constant & 31;
12328 int rshift = $rshift_count$$constant & 31;
12329 int s = 31 - lshift;
12330 int r = (rshift - lshift) & 31;
12331 __ ubfmw(as_Register($dst$$reg),
12332 as_Register($src$$reg),
12333 r, s);
12334 %}
12335
12336 ins_pipe(ialu_reg_shift);
12337 %}
12338
12339 // Bitfield extract with shift & mask
12340
12341 // This pattern is automatically generated from aarch64_ad.m4
12342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12343 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12344 %{
12345 match(Set dst (AndI (URShiftI src rshift) mask));
12346 // Make sure we are not going to exceed what ubfxw can do.
12347 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12348
12349 ins_cost(INSN_COST);
12350 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12351 ins_encode %{
12352 int rshift = $rshift$$constant & 31;
12353 intptr_t mask = $mask$$constant;
12354 int width = exact_log2(mask+1);
12355 __ ubfxw(as_Register($dst$$reg),
12356 as_Register($src$$reg), rshift, width);
12357 %}
12358 ins_pipe(ialu_reg_shift);
12359 %}
12360
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12364 %{
12365 match(Set dst (AndL (URShiftL src rshift) mask));
12366 // Make sure we are not going to exceed what ubfx can do.
12367 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12368
12369 ins_cost(INSN_COST);
12370 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12371 ins_encode %{
12372 int rshift = $rshift$$constant & 63;
12373 intptr_t mask = $mask$$constant;
12374 int width = exact_log2_long(mask+1);
12375 __ ubfx(as_Register($dst$$reg),
12376 as_Register($src$$reg), rshift, width);
12377 %}
12378 ins_pipe(ialu_reg_shift);
12379 %}
12380
12381
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384
12385 // We can use ubfx when extending an And with a mask when we know mask
12386 // is positive. We know that because immI_bitmask guarantees it.
12387 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12388 %{
12389 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12390 // Make sure we are not going to exceed what ubfxw can do.
12391 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12392
12393 ins_cost(INSN_COST * 2);
12394 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12395 ins_encode %{
12396 int rshift = $rshift$$constant & 31;
12397 intptr_t mask = $mask$$constant;
12398 int width = exact_log2(mask+1);
12399 __ ubfx(as_Register($dst$$reg),
12400 as_Register($src$$reg), rshift, width);
12401 %}
12402 ins_pipe(ialu_reg_shift);
12403 %}
12404
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408
12409 // We can use ubfiz when masking by a positive number and then left shifting the result.
12410 // We know that the mask is positive because immI_bitmask guarantees it.
12411 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12412 %{
12413 match(Set dst (LShiftI (AndI src mask) lshift));
12414 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12415
12416 ins_cost(INSN_COST);
12417 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12418 ins_encode %{
12419 int lshift = $lshift$$constant & 31;
12420 intptr_t mask = $mask$$constant;
12421 int width = exact_log2(mask+1);
12422 __ ubfizw(as_Register($dst$$reg),
12423 as_Register($src$$reg), lshift, width);
12424 %}
12425 ins_pipe(ialu_reg_shift);
12426 %}
12427
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430
12431 // We can use ubfiz when masking by a positive number and then left shifting the result.
12432 // We know that the mask is positive because immL_bitmask guarantees it.
12433 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12434 %{
12435 match(Set dst (LShiftL (AndL src mask) lshift));
12436 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12437
12438 ins_cost(INSN_COST);
12439 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12440 ins_encode %{
12441 int lshift = $lshift$$constant & 63;
12442 intptr_t mask = $mask$$constant;
12443 int width = exact_log2_long(mask+1);
12444 __ ubfiz(as_Register($dst$$reg),
12445 as_Register($src$$reg), lshift, width);
12446 %}
12447 ins_pipe(ialu_reg_shift);
12448 %}
12449
12450 // This pattern is automatically generated from aarch64_ad.m4
12451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12452
12453 // We can use ubfiz when masking by a positive number and then left shifting the result.
12454 // We know that the mask is positive because immI_bitmask guarantees it.
12455 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12456 %{
12457 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12458 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12459
12460 ins_cost(INSN_COST);
12461 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12462 ins_encode %{
12463 int lshift = $lshift$$constant & 31;
12464 intptr_t mask = $mask$$constant;
12465 int width = exact_log2(mask+1);
12466 __ ubfizw(as_Register($dst$$reg),
12467 as_Register($src$$reg), lshift, width);
12468 %}
12469 ins_pipe(ialu_reg_shift);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474
12475 // We can use ubfiz when masking by a positive number and then left shifting the result.
12476 // We know that the mask is positive because immL_bitmask guarantees it.
12477 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12478 %{
12479 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12480 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12481
12482 ins_cost(INSN_COST);
12483 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12484 ins_encode %{
12485 int lshift = $lshift$$constant & 63;
12486 intptr_t mask = $mask$$constant;
12487 int width = exact_log2_long(mask+1);
12488 __ ubfiz(as_Register($dst$$reg),
12489 as_Register($src$$reg), lshift, width);
12490 %}
12491 ins_pipe(ialu_reg_shift);
12492 %}
12493
12494
12495 // This pattern is automatically generated from aarch64_ad.m4
12496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12497
12498 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12499 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12500 %{
12501 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12502 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12503
12504 ins_cost(INSN_COST);
12505 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12506 ins_encode %{
12507 int lshift = $lshift$$constant & 63;
12508 intptr_t mask = $mask$$constant;
12509 int width = exact_log2(mask+1);
12510 __ ubfiz(as_Register($dst$$reg),
12511 as_Register($src$$reg), lshift, width);
12512 %}
12513 ins_pipe(ialu_reg_shift);
12514 %}
12515
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518
12519 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12520 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12521 %{
12522 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12523 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12524
12525 ins_cost(INSN_COST);
12526 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12527 ins_encode %{
12528 int lshift = $lshift$$constant & 31;
12529 intptr_t mask = $mask$$constant;
12530 int width = exact_log2(mask+1);
12531 __ ubfiz(as_Register($dst$$reg),
12532 as_Register($src$$reg), lshift, width);
12533 %}
12534 ins_pipe(ialu_reg_shift);
12535 %}
12536
12537 // This pattern is automatically generated from aarch64_ad.m4
12538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12539
12540 // Can skip int2long conversions after AND with small bitmask
12541 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12542 %{
12543 match(Set dst (ConvI2L (AndI src msk)));
12544 ins_cost(INSN_COST);
12545 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12546 ins_encode %{
12547 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12548 %}
12549 ins_pipe(ialu_reg_shift);
12550 %}
12551
12552
12553 // Rotations
12554
12555 // This pattern is automatically generated from aarch64_ad.m4
12556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12557 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12558 %{
12559 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12560 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12561
12562 ins_cost(INSN_COST);
12563 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12564
12565 ins_encode %{
12566 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12567 $rshift$$constant & 63);
12568 %}
12569 ins_pipe(ialu_reg_reg_extr);
12570 %}
12571
12572
12573 // This pattern is automatically generated from aarch64_ad.m4
12574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12575 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12576 %{
12577 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12578 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12579
12580 ins_cost(INSN_COST);
12581 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12582
12583 ins_encode %{
12584 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12585 $rshift$$constant & 31);
12586 %}
12587 ins_pipe(ialu_reg_reg_extr);
12588 %}
12589
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12594 %{
12595 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12596 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12597
12598 ins_cost(INSN_COST);
12599 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12600
12601 ins_encode %{
12602 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12603 $rshift$$constant & 63);
12604 %}
12605 ins_pipe(ialu_reg_reg_extr);
12606 %}
12607
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12612 %{
12613 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12614 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12615
12616 ins_cost(INSN_COST);
12617 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12618
12619 ins_encode %{
12620 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12621 $rshift$$constant & 31);
12622 %}
12623 ins_pipe(ialu_reg_reg_extr);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12629 %{
12630 match(Set dst (RotateRight src shift));
12631
12632 ins_cost(INSN_COST);
12633 format %{ "ror $dst, $src, $shift" %}
12634
12635 ins_encode %{
12636 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12637 $shift$$constant & 0x1f);
12638 %}
12639 ins_pipe(ialu_reg_reg_vshift);
12640 %}
12641
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12645 %{
12646 match(Set dst (RotateRight src shift));
12647
12648 ins_cost(INSN_COST);
12649 format %{ "ror $dst, $src, $shift" %}
12650
12651 ins_encode %{
12652 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12653 $shift$$constant & 0x3f);
12654 %}
12655 ins_pipe(ialu_reg_reg_vshift);
12656 %}
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12661 %{
12662 match(Set dst (RotateRight src shift));
12663
12664 ins_cost(INSN_COST);
12665 format %{ "ror $dst, $src, $shift" %}
12666
12667 ins_encode %{
12668 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12669 %}
12670 ins_pipe(ialu_reg_reg_vshift);
12671 %}
12672
12673 // This pattern is automatically generated from aarch64_ad.m4
12674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12675 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12676 %{
12677 match(Set dst (RotateRight src shift));
12678
12679 ins_cost(INSN_COST);
12680 format %{ "ror $dst, $src, $shift" %}
12681
12682 ins_encode %{
12683 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12684 %}
12685 ins_pipe(ialu_reg_reg_vshift);
12686 %}
12687
12688 // This pattern is automatically generated from aarch64_ad.m4
12689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12690 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12691 %{
12692 match(Set dst (RotateLeft src shift));
12693
12694 ins_cost(INSN_COST);
12695 format %{ "rol $dst, $src, $shift" %}
12696
12697 ins_encode %{
12698 __ subw(rscratch1, zr, as_Register($shift$$reg));
12699 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12700 %}
12701 ins_pipe(ialu_reg_reg_vshift);
12702 %}
12703
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12707 %{
12708 match(Set dst (RotateLeft src shift));
12709
12710 ins_cost(INSN_COST);
12711 format %{ "rol $dst, $src, $shift" %}
12712
12713 ins_encode %{
12714 __ subw(rscratch1, zr, as_Register($shift$$reg));
12715 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12716 %}
12717 ins_pipe(ialu_reg_reg_vshift);
12718 %}
12719
12720
12721 // Add/subtract (extended)
12722
12723 // This pattern is automatically generated from aarch64_ad.m4
12724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12725 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12726 %{
12727 match(Set dst (AddL src1 (ConvI2L src2)));
12728 ins_cost(INSN_COST);
12729 format %{ "add $dst, $src1, $src2, sxtw" %}
12730
12731 ins_encode %{
12732 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12733 as_Register($src2$$reg), ext::sxtw);
12734 %}
12735 ins_pipe(ialu_reg_reg);
12736 %}
12737
12738 // This pattern is automatically generated from aarch64_ad.m4
12739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12740 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12741 %{
12742 match(Set dst (SubL src1 (ConvI2L src2)));
12743 ins_cost(INSN_COST);
12744 format %{ "sub $dst, $src1, $src2, sxtw" %}
12745
12746 ins_encode %{
12747 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12748 as_Register($src2$$reg), ext::sxtw);
12749 %}
12750 ins_pipe(ialu_reg_reg);
12751 %}
12752
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12756 %{
12757 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12758 ins_cost(INSN_COST);
12759 format %{ "add $dst, $src1, $src2, sxth" %}
12760
12761 ins_encode %{
12762 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12763 as_Register($src2$$reg), ext::sxth);
12764 %}
12765 ins_pipe(ialu_reg_reg);
12766 %}
12767
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12771 %{
12772 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12773 ins_cost(INSN_COST);
12774 format %{ "add $dst, $src1, $src2, sxtb" %}
12775
12776 ins_encode %{
12777 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12778 as_Register($src2$$reg), ext::sxtb);
12779 %}
12780 ins_pipe(ialu_reg_reg);
12781 %}
12782
12783 // This pattern is automatically generated from aarch64_ad.m4
12784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12785 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12786 %{
12787 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12788 ins_cost(INSN_COST);
12789 format %{ "add $dst, $src1, $src2, uxtb" %}
12790
12791 ins_encode %{
12792 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12793 as_Register($src2$$reg), ext::uxtb);
12794 %}
12795 ins_pipe(ialu_reg_reg);
12796 %}
12797
12798 // This pattern is automatically generated from aarch64_ad.m4
12799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12800 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12801 %{
12802 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12803 ins_cost(INSN_COST);
12804 format %{ "add $dst, $src1, $src2, sxth" %}
12805
12806 ins_encode %{
12807 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12808 as_Register($src2$$reg), ext::sxth);
12809 %}
12810 ins_pipe(ialu_reg_reg);
12811 %}
12812
12813 // This pattern is automatically generated from aarch64_ad.m4
12814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12815 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12816 %{
12817 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12818 ins_cost(INSN_COST);
12819 format %{ "add $dst, $src1, $src2, sxtw" %}
12820
12821 ins_encode %{
12822 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12823 as_Register($src2$$reg), ext::sxtw);
12824 %}
12825 ins_pipe(ialu_reg_reg);
12826 %}
12827
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12831 %{
12832 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12833 ins_cost(INSN_COST);
12834 format %{ "add $dst, $src1, $src2, sxtb" %}
12835
12836 ins_encode %{
12837 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12838 as_Register($src2$$reg), ext::sxtb);
12839 %}
12840 ins_pipe(ialu_reg_reg);
12841 %}
12842
12843 // This pattern is automatically generated from aarch64_ad.m4
12844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12845 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12846 %{
12847 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12848 ins_cost(INSN_COST);
12849 format %{ "add $dst, $src1, $src2, uxtb" %}
12850
12851 ins_encode %{
12852 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12853 as_Register($src2$$reg), ext::uxtb);
12854 %}
12855 ins_pipe(ialu_reg_reg);
12856 %}
12857
12858 // This pattern is automatically generated from aarch64_ad.m4
12859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12860 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12861 %{
12862 match(Set dst (AddI src1 (AndI src2 mask)));
12863 ins_cost(INSN_COST);
12864 format %{ "addw $dst, $src1, $src2, uxtb" %}
12865
12866 ins_encode %{
12867 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12868 as_Register($src2$$reg), ext::uxtb);
12869 %}
12870 ins_pipe(ialu_reg_reg);
12871 %}
12872
12873 // This pattern is automatically generated from aarch64_ad.m4
12874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12875 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12876 %{
12877 match(Set dst (AddI src1 (AndI src2 mask)));
12878 ins_cost(INSN_COST);
12879 format %{ "addw $dst, $src1, $src2, uxth" %}
12880
12881 ins_encode %{
12882 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12883 as_Register($src2$$reg), ext::uxth);
12884 %}
12885 ins_pipe(ialu_reg_reg);
12886 %}
12887
12888 // This pattern is automatically generated from aarch64_ad.m4
12889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12890 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12891 %{
12892 match(Set dst (AddL src1 (AndL src2 mask)));
12893 ins_cost(INSN_COST);
12894 format %{ "add $dst, $src1, $src2, uxtb" %}
12895
12896 ins_encode %{
12897 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12898 as_Register($src2$$reg), ext::uxtb);
12899 %}
12900 ins_pipe(ialu_reg_reg);
12901 %}
12902
12903 // This pattern is automatically generated from aarch64_ad.m4
12904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12905 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12906 %{
12907 match(Set dst (AddL src1 (AndL src2 mask)));
12908 ins_cost(INSN_COST);
12909 format %{ "add $dst, $src1, $src2, uxth" %}
12910
12911 ins_encode %{
12912 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12913 as_Register($src2$$reg), ext::uxth);
12914 %}
12915 ins_pipe(ialu_reg_reg);
12916 %}
12917
12918 // This pattern is automatically generated from aarch64_ad.m4
12919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12920 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12921 %{
12922 match(Set dst (AddL src1 (AndL src2 mask)));
12923 ins_cost(INSN_COST);
12924 format %{ "add $dst, $src1, $src2, uxtw" %}
12925
12926 ins_encode %{
12927 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12928 as_Register($src2$$reg), ext::uxtw);
12929 %}
12930 ins_pipe(ialu_reg_reg);
12931 %}
12932
12933 // This pattern is automatically generated from aarch64_ad.m4
12934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12935 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12936 %{
12937 match(Set dst (SubI src1 (AndI src2 mask)));
12938 ins_cost(INSN_COST);
12939 format %{ "subw $dst, $src1, $src2, uxtb" %}
12940
12941 ins_encode %{
12942 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12943 as_Register($src2$$reg), ext::uxtb);
12944 %}
12945 ins_pipe(ialu_reg_reg);
12946 %}
12947
12948 // This pattern is automatically generated from aarch64_ad.m4
12949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12950 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12951 %{
12952 match(Set dst (SubI src1 (AndI src2 mask)));
12953 ins_cost(INSN_COST);
12954 format %{ "subw $dst, $src1, $src2, uxth" %}
12955
12956 ins_encode %{
12957 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12958 as_Register($src2$$reg), ext::uxth);
12959 %}
12960 ins_pipe(ialu_reg_reg);
12961 %}
12962
12963 // This pattern is automatically generated from aarch64_ad.m4
12964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12965 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12966 %{
12967 match(Set dst (SubL src1 (AndL src2 mask)));
12968 ins_cost(INSN_COST);
12969 format %{ "sub $dst, $src1, $src2, uxtb" %}
12970
12971 ins_encode %{
12972 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12973 as_Register($src2$$reg), ext::uxtb);
12974 %}
12975 ins_pipe(ialu_reg_reg);
12976 %}
12977
12978 // This pattern is automatically generated from aarch64_ad.m4
12979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12980 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12981 %{
12982 match(Set dst (SubL src1 (AndL src2 mask)));
12983 ins_cost(INSN_COST);
12984 format %{ "sub $dst, $src1, $src2, uxth" %}
12985
12986 ins_encode %{
12987 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12988 as_Register($src2$$reg), ext::uxth);
12989 %}
12990 ins_pipe(ialu_reg_reg);
12991 %}
12992
12993 // This pattern is automatically generated from aarch64_ad.m4
12994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12995 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12996 %{
12997 match(Set dst (SubL src1 (AndL src2 mask)));
12998 ins_cost(INSN_COST);
12999 format %{ "sub $dst, $src1, $src2, uxtw" %}
13000
13001 ins_encode %{
13002 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13003 as_Register($src2$$reg), ext::uxtw);
13004 %}
13005 ins_pipe(ialu_reg_reg);
13006 %}
13007
13008
13009 // This pattern is automatically generated from aarch64_ad.m4
13010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13011 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13012 %{
13013 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13014 ins_cost(1.9 * INSN_COST);
13015 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13016
13017 ins_encode %{
13018 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13019 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13020 %}
13021 ins_pipe(ialu_reg_reg_shift);
13022 %}
13023
13024 // This pattern is automatically generated from aarch64_ad.m4
13025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13026 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13027 %{
13028 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13029 ins_cost(1.9 * INSN_COST);
13030 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13031
13032 ins_encode %{
13033 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13034 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13035 %}
13036 ins_pipe(ialu_reg_reg_shift);
13037 %}
13038
13039 // This pattern is automatically generated from aarch64_ad.m4
13040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13041 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13042 %{
13043 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13044 ins_cost(1.9 * INSN_COST);
13045 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13046
13047 ins_encode %{
13048 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13049 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13050 %}
13051 ins_pipe(ialu_reg_reg_shift);
13052 %}
13053
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13057 %{
13058 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13059 ins_cost(1.9 * INSN_COST);
13060 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13061
13062 ins_encode %{
13063 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13064 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13065 %}
13066 ins_pipe(ialu_reg_reg_shift);
13067 %}
13068
13069 // This pattern is automatically generated from aarch64_ad.m4
13070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13071 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13072 %{
13073 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13074 ins_cost(1.9 * INSN_COST);
13075 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13076
13077 ins_encode %{
13078 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13079 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13080 %}
13081 ins_pipe(ialu_reg_reg_shift);
13082 %}
13083
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13087 %{
13088 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13089 ins_cost(1.9 * INSN_COST);
13090 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13091
13092 ins_encode %{
13093 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13094 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13095 %}
13096 ins_pipe(ialu_reg_reg_shift);
13097 %}
13098
13099 // This pattern is automatically generated from aarch64_ad.m4
13100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13101 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13102 %{
13103 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13104 ins_cost(1.9 * INSN_COST);
13105 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13106
13107 ins_encode %{
13108 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13109 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13110 %}
13111 ins_pipe(ialu_reg_reg_shift);
13112 %}
13113
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13117 %{
13118 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13119 ins_cost(1.9 * INSN_COST);
13120 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13121
13122 ins_encode %{
13123 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13124 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13125 %}
13126 ins_pipe(ialu_reg_reg_shift);
13127 %}
13128
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13132 %{
13133 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13134 ins_cost(1.9 * INSN_COST);
13135 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13136
13137 ins_encode %{
13138 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13139 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13140 %}
13141 ins_pipe(ialu_reg_reg_shift);
13142 %}
13143
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13147 %{
13148 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13149 ins_cost(1.9 * INSN_COST);
13150 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13151
13152 ins_encode %{
13153 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13154 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13155 %}
13156 ins_pipe(ialu_reg_reg_shift);
13157 %}
13158
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13162 %{
13163 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13164 ins_cost(1.9 * INSN_COST);
13165 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13166
13167 ins_encode %{
13168 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13170 %}
13171 ins_pipe(ialu_reg_reg_shift);
13172 %}
13173
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13177 %{
13178 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13179 ins_cost(1.9 * INSN_COST);
13180 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13181
13182 ins_encode %{
13183 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13184 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13185 %}
13186 ins_pipe(ialu_reg_reg_shift);
13187 %}
13188
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13192 %{
13193 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13194 ins_cost(1.9 * INSN_COST);
13195 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13196
13197 ins_encode %{
13198 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13199 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13200 %}
13201 ins_pipe(ialu_reg_reg_shift);
13202 %}
13203
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13207 %{
13208 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13209 ins_cost(1.9 * INSN_COST);
13210 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13211
13212 ins_encode %{
13213 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13214 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13215 %}
13216 ins_pipe(ialu_reg_reg_shift);
13217 %}
13218
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13222 %{
13223 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13224 ins_cost(1.9 * INSN_COST);
13225 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13226
13227 ins_encode %{
13228 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13229 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13230 %}
13231 ins_pipe(ialu_reg_reg_shift);
13232 %}
13233
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13237 %{
13238 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13239 ins_cost(1.9 * INSN_COST);
13240 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13241
13242 ins_encode %{
13243 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13244 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13245 %}
13246 ins_pipe(ialu_reg_reg_shift);
13247 %}
13248
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13252 %{
13253 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13254 ins_cost(1.9 * INSN_COST);
13255 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13256
13257 ins_encode %{
13258 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13259 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13260 %}
13261 ins_pipe(ialu_reg_reg_shift);
13262 %}
13263
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13267 %{
13268 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13269 ins_cost(1.9 * INSN_COST);
13270 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13271
13272 ins_encode %{
13273 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13274 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13275 %}
13276 ins_pipe(ialu_reg_reg_shift);
13277 %}
13278
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13282 %{
13283 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13284 ins_cost(1.9 * INSN_COST);
13285 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13286
13287 ins_encode %{
13288 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13289 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13290 %}
13291 ins_pipe(ialu_reg_reg_shift);
13292 %}
13293
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13297 %{
13298 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13299 ins_cost(1.9 * INSN_COST);
13300 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13301
13302 ins_encode %{
13303 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13304 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13305 %}
13306 ins_pipe(ialu_reg_reg_shift);
13307 %}
13308
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13312 %{
13313 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13314 ins_cost(1.9 * INSN_COST);
13315 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13316
13317 ins_encode %{
13318 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13319 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13320 %}
13321 ins_pipe(ialu_reg_reg_shift);
13322 %}
13323
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13327 %{
13328 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13329 ins_cost(1.9 * INSN_COST);
13330 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13331
13332 ins_encode %{
13333 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13334 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13335 %}
13336 ins_pipe(ialu_reg_reg_shift);
13337 %}
13338
13339 // This pattern is automatically generated from aarch64_ad.m4
13340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13341 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13342 %{
13343 effect(DEF dst, USE src1, USE src2, USE cr);
13344 ins_cost(INSN_COST * 2);
13345 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13346
13347 ins_encode %{
13348 __ cselw($dst$$Register,
13349 $src1$$Register,
13350 $src2$$Register,
13351 Assembler::LT);
13352 %}
13353 ins_pipe(icond_reg_reg);
13354 %}
13355
13356 // This pattern is automatically generated from aarch64_ad.m4
13357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13358 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13359 %{
13360 effect(DEF dst, USE src1, USE src2, USE cr);
13361 ins_cost(INSN_COST * 2);
13362 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13363
13364 ins_encode %{
13365 __ cselw($dst$$Register,
13366 $src1$$Register,
13367 $src2$$Register,
13368 Assembler::GT);
13369 %}
13370 ins_pipe(icond_reg_reg);
13371 %}
13372
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13376 %{
13377 effect(DEF dst, USE src1, USE cr);
13378 ins_cost(INSN_COST * 2);
13379 format %{ "cselw $dst, $src1, zr lt\t" %}
13380
13381 ins_encode %{
13382 __ cselw($dst$$Register,
13383 $src1$$Register,
13384 zr,
13385 Assembler::LT);
13386 %}
13387 ins_pipe(icond_reg);
13388 %}
13389
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13393 %{
13394 effect(DEF dst, USE src1, USE cr);
13395 ins_cost(INSN_COST * 2);
13396 format %{ "cselw $dst, $src1, zr gt\t" %}
13397
13398 ins_encode %{
13399 __ cselw($dst$$Register,
13400 $src1$$Register,
13401 zr,
13402 Assembler::GT);
13403 %}
13404 ins_pipe(icond_reg);
13405 %}
13406
13407 // This pattern is automatically generated from aarch64_ad.m4
13408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13409 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13410 %{
13411 effect(DEF dst, USE src1, USE cr);
13412 ins_cost(INSN_COST * 2);
13413 format %{ "csincw $dst, $src1, zr le\t" %}
13414
13415 ins_encode %{
13416 __ csincw($dst$$Register,
13417 $src1$$Register,
13418 zr,
13419 Assembler::LE);
13420 %}
13421 ins_pipe(icond_reg);
13422 %}
13423
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13427 %{
13428 effect(DEF dst, USE src1, USE cr);
13429 ins_cost(INSN_COST * 2);
13430 format %{ "csincw $dst, $src1, zr gt\t" %}
13431
13432 ins_encode %{
13433 __ csincw($dst$$Register,
13434 $src1$$Register,
13435 zr,
13436 Assembler::GT);
13437 %}
13438 ins_pipe(icond_reg);
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13444 %{
13445 effect(DEF dst, USE src1, USE cr);
13446 ins_cost(INSN_COST * 2);
13447 format %{ "csinvw $dst, $src1, zr lt\t" %}
13448
13449 ins_encode %{
13450 __ csinvw($dst$$Register,
13451 $src1$$Register,
13452 zr,
13453 Assembler::LT);
13454 %}
13455 ins_pipe(icond_reg);
13456 %}
13457
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13461 %{
13462 effect(DEF dst, USE src1, USE cr);
13463 ins_cost(INSN_COST * 2);
13464 format %{ "csinvw $dst, $src1, zr ge\t" %}
13465
13466 ins_encode %{
13467 __ csinvw($dst$$Register,
13468 $src1$$Register,
13469 zr,
13470 Assembler::GE);
13471 %}
13472 ins_pipe(icond_reg);
13473 %}
13474
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13478 %{
13479 match(Set dst (MinI src imm));
13480 ins_cost(INSN_COST * 3);
13481 expand %{
13482 rFlagsReg cr;
13483 compI_reg_imm0(cr, src);
13484 cmovI_reg_imm0_lt(dst, src, cr);
13485 %}
13486 %}
13487
13488 // This pattern is automatically generated from aarch64_ad.m4
13489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13490 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13491 %{
13492 match(Set dst (MinI imm src));
13493 ins_cost(INSN_COST * 3);
13494 expand %{
13495 rFlagsReg cr;
13496 compI_reg_imm0(cr, src);
13497 cmovI_reg_imm0_lt(dst, src, cr);
13498 %}
13499 %}
13500
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13504 %{
13505 match(Set dst (MinI src imm));
13506 ins_cost(INSN_COST * 3);
13507 expand %{
13508 rFlagsReg cr;
13509 compI_reg_imm0(cr, src);
13510 cmovI_reg_imm1_le(dst, src, cr);
13511 %}
13512 %}
13513
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13517 %{
13518 match(Set dst (MinI imm src));
13519 ins_cost(INSN_COST * 3);
13520 expand %{
13521 rFlagsReg cr;
13522 compI_reg_imm0(cr, src);
13523 cmovI_reg_imm1_le(dst, src, cr);
13524 %}
13525 %}
13526
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13530 %{
13531 match(Set dst (MinI src imm));
13532 ins_cost(INSN_COST * 3);
13533 expand %{
13534 rFlagsReg cr;
13535 compI_reg_imm0(cr, src);
13536 cmovI_reg_immM1_lt(dst, src, cr);
13537 %}
13538 %}
13539
13540 // This pattern is automatically generated from aarch64_ad.m4
13541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13542 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13543 %{
13544 match(Set dst (MinI imm src));
13545 ins_cost(INSN_COST * 3);
13546 expand %{
13547 rFlagsReg cr;
13548 compI_reg_imm0(cr, src);
13549 cmovI_reg_immM1_lt(dst, src, cr);
13550 %}
13551 %}
13552
13553 // This pattern is automatically generated from aarch64_ad.m4
13554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13555 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13556 %{
13557 match(Set dst (MaxI src imm));
13558 ins_cost(INSN_COST * 3);
13559 expand %{
13560 rFlagsReg cr;
13561 compI_reg_imm0(cr, src);
13562 cmovI_reg_imm0_gt(dst, src, cr);
13563 %}
13564 %}
13565
13566 // This pattern is automatically generated from aarch64_ad.m4
13567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13568 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13569 %{
13570 match(Set dst (MaxI imm src));
13571 ins_cost(INSN_COST * 3);
13572 expand %{
13573 rFlagsReg cr;
13574 compI_reg_imm0(cr, src);
13575 cmovI_reg_imm0_gt(dst, src, cr);
13576 %}
13577 %}
13578
13579 // This pattern is automatically generated from aarch64_ad.m4
13580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13581 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13582 %{
13583 match(Set dst (MaxI src imm));
13584 ins_cost(INSN_COST * 3);
13585 expand %{
13586 rFlagsReg cr;
13587 compI_reg_imm0(cr, src);
13588 cmovI_reg_imm1_gt(dst, src, cr);
13589 %}
13590 %}
13591
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13595 %{
13596 match(Set dst (MaxI imm src));
13597 ins_cost(INSN_COST * 3);
13598 expand %{
13599 rFlagsReg cr;
13600 compI_reg_imm0(cr, src);
13601 cmovI_reg_imm1_gt(dst, src, cr);
13602 %}
13603 %}
13604
13605 // This pattern is automatically generated from aarch64_ad.m4
13606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13607 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13608 %{
13609 match(Set dst (MaxI src imm));
13610 ins_cost(INSN_COST * 3);
13611 expand %{
13612 rFlagsReg cr;
13613 compI_reg_imm0(cr, src);
13614 cmovI_reg_immM1_ge(dst, src, cr);
13615 %}
13616 %}
13617
13618 // This pattern is automatically generated from aarch64_ad.m4
13619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13620 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13621 %{
13622 match(Set dst (MaxI imm src));
13623 ins_cost(INSN_COST * 3);
13624 expand %{
13625 rFlagsReg cr;
13626 compI_reg_imm0(cr, src);
13627 cmovI_reg_immM1_ge(dst, src, cr);
13628 %}
13629 %}
13630
13631 // This pattern is automatically generated from aarch64_ad.m4
13632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13633 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13634 %{
13635 match(Set dst (ReverseI src));
13636 ins_cost(INSN_COST);
13637 format %{ "rbitw $dst, $src" %}
13638 ins_encode %{
13639 __ rbitw($dst$$Register, $src$$Register);
13640 %}
13641 ins_pipe(ialu_reg);
13642 %}
13643
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13647 %{
13648 match(Set dst (ReverseL src));
13649 ins_cost(INSN_COST);
13650 format %{ "rbit $dst, $src" %}
13651 ins_encode %{
13652 __ rbit($dst$$Register, $src$$Register);
13653 %}
13654 ins_pipe(ialu_reg);
13655 %}
13656
13657
13658 // END This section of the file is automatically generated. Do not edit --------------
13659
13660
13661 // ============================================================================
13662 // Floating Point Arithmetic Instructions
13663
13664 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13665 match(Set dst (AddF src1 src2));
13666
13667 ins_cost(INSN_COST * 5);
13668 format %{ "fadds $dst, $src1, $src2" %}
13669
13670 ins_encode %{
13671 __ fadds(as_FloatRegister($dst$$reg),
13672 as_FloatRegister($src1$$reg),
13673 as_FloatRegister($src2$$reg));
13674 %}
13675
13676 ins_pipe(fp_dop_reg_reg_s);
13677 %}
13678
13679 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13680 match(Set dst (AddD src1 src2));
13681
13682 ins_cost(INSN_COST * 5);
13683 format %{ "faddd $dst, $src1, $src2" %}
13684
13685 ins_encode %{
13686 __ faddd(as_FloatRegister($dst$$reg),
13687 as_FloatRegister($src1$$reg),
13688 as_FloatRegister($src2$$reg));
13689 %}
13690
13691 ins_pipe(fp_dop_reg_reg_d);
13692 %}
13693
13694 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13695 match(Set dst (SubF src1 src2));
13696
13697 ins_cost(INSN_COST * 5);
13698 format %{ "fsubs $dst, $src1, $src2" %}
13699
13700 ins_encode %{
13701 __ fsubs(as_FloatRegister($dst$$reg),
13702 as_FloatRegister($src1$$reg),
13703 as_FloatRegister($src2$$reg));
13704 %}
13705
13706 ins_pipe(fp_dop_reg_reg_s);
13707 %}
13708
13709 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13710 match(Set dst (SubD src1 src2));
13711
13712 ins_cost(INSN_COST * 5);
13713 format %{ "fsubd $dst, $src1, $src2" %}
13714
13715 ins_encode %{
13716 __ fsubd(as_FloatRegister($dst$$reg),
13717 as_FloatRegister($src1$$reg),
13718 as_FloatRegister($src2$$reg));
13719 %}
13720
13721 ins_pipe(fp_dop_reg_reg_d);
13722 %}
13723
13724 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13725 match(Set dst (MulF src1 src2));
13726
13727 ins_cost(INSN_COST * 6);
13728 format %{ "fmuls $dst, $src1, $src2" %}
13729
13730 ins_encode %{
13731 __ fmuls(as_FloatRegister($dst$$reg),
13732 as_FloatRegister($src1$$reg),
13733 as_FloatRegister($src2$$reg));
13734 %}
13735
13736 ins_pipe(fp_dop_reg_reg_s);
13737 %}
13738
13739 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13740 match(Set dst (MulD src1 src2));
13741
13742 ins_cost(INSN_COST * 6);
13743 format %{ "fmuld $dst, $src1, $src2" %}
13744
13745 ins_encode %{
13746 __ fmuld(as_FloatRegister($dst$$reg),
13747 as_FloatRegister($src1$$reg),
13748 as_FloatRegister($src2$$reg));
13749 %}
13750
13751 ins_pipe(fp_dop_reg_reg_d);
13752 %}
13753
13754 // src1 * src2 + src3
13755 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13756 match(Set dst (FmaF src3 (Binary src1 src2)));
13757
13758 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13759
13760 ins_encode %{
13761 assert(UseFMA, "Needs FMA instructions support.");
13762 __ fmadds(as_FloatRegister($dst$$reg),
13763 as_FloatRegister($src1$$reg),
13764 as_FloatRegister($src2$$reg),
13765 as_FloatRegister($src3$$reg));
13766 %}
13767
13768 ins_pipe(pipe_class_default);
13769 %}
13770
13771 // src1 * src2 + src3
13772 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13773 match(Set dst (FmaD src3 (Binary src1 src2)));
13774
13775 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13776
13777 ins_encode %{
13778 assert(UseFMA, "Needs FMA instructions support.");
13779 __ fmaddd(as_FloatRegister($dst$$reg),
13780 as_FloatRegister($src1$$reg),
13781 as_FloatRegister($src2$$reg),
13782 as_FloatRegister($src3$$reg));
13783 %}
13784
13785 ins_pipe(pipe_class_default);
13786 %}
13787
13788 // src1 * (-src2) + src3
13789 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13790 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13791 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13792
13793 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13794
13795 ins_encode %{
13796 assert(UseFMA, "Needs FMA instructions support.");
13797 __ fmsubs(as_FloatRegister($dst$$reg),
13798 as_FloatRegister($src1$$reg),
13799 as_FloatRegister($src2$$reg),
13800 as_FloatRegister($src3$$reg));
13801 %}
13802
13803 ins_pipe(pipe_class_default);
13804 %}
13805
13806 // src1 * (-src2) + src3
13807 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13808 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13809 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13810
13811 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13812
13813 ins_encode %{
13814 assert(UseFMA, "Needs FMA instructions support.");
13815 __ fmsubd(as_FloatRegister($dst$$reg),
13816 as_FloatRegister($src1$$reg),
13817 as_FloatRegister($src2$$reg),
13818 as_FloatRegister($src3$$reg));
13819 %}
13820
13821 ins_pipe(pipe_class_default);
13822 %}
13823
13824 // src1 * (-src2) - src3
13825 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13826 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13827 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13828
13829 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13830
13831 ins_encode %{
13832 assert(UseFMA, "Needs FMA instructions support.");
13833 __ fnmadds(as_FloatRegister($dst$$reg),
13834 as_FloatRegister($src1$$reg),
13835 as_FloatRegister($src2$$reg),
13836 as_FloatRegister($src3$$reg));
13837 %}
13838
13839 ins_pipe(pipe_class_default);
13840 %}
13841
13842 // src1 * (-src2) - src3
13843 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13844 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13845 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13846
13847 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13848
13849 ins_encode %{
13850 assert(UseFMA, "Needs FMA instructions support.");
13851 __ fnmaddd(as_FloatRegister($dst$$reg),
13852 as_FloatRegister($src1$$reg),
13853 as_FloatRegister($src2$$reg),
13854 as_FloatRegister($src3$$reg));
13855 %}
13856
13857 ins_pipe(pipe_class_default);
13858 %}
13859
13860 // src1 * src2 - src3
13861 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13862 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13863
13864 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13865
13866 ins_encode %{
13867 assert(UseFMA, "Needs FMA instructions support.");
13868 __ fnmsubs(as_FloatRegister($dst$$reg),
13869 as_FloatRegister($src1$$reg),
13870 as_FloatRegister($src2$$reg),
13871 as_FloatRegister($src3$$reg));
13872 %}
13873
13874 ins_pipe(pipe_class_default);
13875 %}
13876
13877 // src1 * src2 - src3
13878 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13879 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13880
13881 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13882
13883 ins_encode %{
13884 assert(UseFMA, "Needs FMA instructions support.");
13885 // n.b. insn name should be fnmsubd
13886 __ fnmsub(as_FloatRegister($dst$$reg),
13887 as_FloatRegister($src1$$reg),
13888 as_FloatRegister($src2$$reg),
13889 as_FloatRegister($src3$$reg));
13890 %}
13891
13892 ins_pipe(pipe_class_default);
13893 %}
13894
13895
13896 // Math.max(FF)F
13897 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13898 match(Set dst (MaxF src1 src2));
13899
13900 format %{ "fmaxs $dst, $src1, $src2" %}
13901 ins_encode %{
13902 __ fmaxs(as_FloatRegister($dst$$reg),
13903 as_FloatRegister($src1$$reg),
13904 as_FloatRegister($src2$$reg));
13905 %}
13906
13907 ins_pipe(fp_dop_reg_reg_s);
13908 %}
13909
13910 // Math.min(FF)F
13911 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13912 match(Set dst (MinF src1 src2));
13913
13914 format %{ "fmins $dst, $src1, $src2" %}
13915 ins_encode %{
13916 __ fmins(as_FloatRegister($dst$$reg),
13917 as_FloatRegister($src1$$reg),
13918 as_FloatRegister($src2$$reg));
13919 %}
13920
13921 ins_pipe(fp_dop_reg_reg_s);
13922 %}
13923
13924 // Math.max(DD)D
13925 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13926 match(Set dst (MaxD src1 src2));
13927
13928 format %{ "fmaxd $dst, $src1, $src2" %}
13929 ins_encode %{
13930 __ fmaxd(as_FloatRegister($dst$$reg),
13931 as_FloatRegister($src1$$reg),
13932 as_FloatRegister($src2$$reg));
13933 %}
13934
13935 ins_pipe(fp_dop_reg_reg_d);
13936 %}
13937
13938 // Math.min(DD)D
13939 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13940 match(Set dst (MinD src1 src2));
13941
13942 format %{ "fmind $dst, $src1, $src2" %}
13943 ins_encode %{
13944 __ fmind(as_FloatRegister($dst$$reg),
13945 as_FloatRegister($src1$$reg),
13946 as_FloatRegister($src2$$reg));
13947 %}
13948
13949 ins_pipe(fp_dop_reg_reg_d);
13950 %}
13951
13952
13953 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13954 match(Set dst (DivF src1 src2));
13955
13956 ins_cost(INSN_COST * 18);
13957 format %{ "fdivs $dst, $src1, $src2" %}
13958
13959 ins_encode %{
13960 __ fdivs(as_FloatRegister($dst$$reg),
13961 as_FloatRegister($src1$$reg),
13962 as_FloatRegister($src2$$reg));
13963 %}
13964
13965 ins_pipe(fp_div_s);
13966 %}
13967
13968 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13969 match(Set dst (DivD src1 src2));
13970
13971 ins_cost(INSN_COST * 32);
13972 format %{ "fdivd $dst, $src1, $src2" %}
13973
13974 ins_encode %{
13975 __ fdivd(as_FloatRegister($dst$$reg),
13976 as_FloatRegister($src1$$reg),
13977 as_FloatRegister($src2$$reg));
13978 %}
13979
13980 ins_pipe(fp_div_d);
13981 %}
13982
13983 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13984 match(Set dst (NegF src));
13985
13986 ins_cost(INSN_COST * 3);
13987 format %{ "fneg $dst, $src" %}
13988
13989 ins_encode %{
13990 __ fnegs(as_FloatRegister($dst$$reg),
13991 as_FloatRegister($src$$reg));
13992 %}
13993
13994 ins_pipe(fp_uop_s);
13995 %}
13996
13997 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13998 match(Set dst (NegD src));
13999
14000 ins_cost(INSN_COST * 3);
14001 format %{ "fnegd $dst, $src" %}
14002
14003 ins_encode %{
14004 __ fnegd(as_FloatRegister($dst$$reg),
14005 as_FloatRegister($src$$reg));
14006 %}
14007
14008 ins_pipe(fp_uop_d);
14009 %}
14010
14011 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14012 %{
14013 match(Set dst (AbsI src));
14014
14015 effect(KILL cr);
14016 ins_cost(INSN_COST * 2);
14017 format %{ "cmpw $src, zr\n\t"
14018 "cnegw $dst, $src, Assembler::LT\t# int abs"
14019 %}
14020
14021 ins_encode %{
14022 __ cmpw(as_Register($src$$reg), zr);
14023 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14024 %}
14025 ins_pipe(pipe_class_default);
14026 %}
14027
14028 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14029 %{
14030 match(Set dst (AbsL src));
14031
14032 effect(KILL cr);
14033 ins_cost(INSN_COST * 2);
14034 format %{ "cmp $src, zr\n\t"
14035 "cneg $dst, $src, Assembler::LT\t# long abs"
14036 %}
14037
14038 ins_encode %{
14039 __ cmp(as_Register($src$$reg), zr);
14040 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14041 %}
14042 ins_pipe(pipe_class_default);
14043 %}
14044
14045 instruct absF_reg(vRegF dst, vRegF src) %{
14046 match(Set dst (AbsF src));
14047
14048 ins_cost(INSN_COST * 3);
14049 format %{ "fabss $dst, $src" %}
14050 ins_encode %{
14051 __ fabss(as_FloatRegister($dst$$reg),
14052 as_FloatRegister($src$$reg));
14053 %}
14054
14055 ins_pipe(fp_uop_s);
14056 %}
14057
14058 instruct absD_reg(vRegD dst, vRegD src) %{
14059 match(Set dst (AbsD src));
14060
14061 ins_cost(INSN_COST * 3);
14062 format %{ "fabsd $dst, $src" %}
14063 ins_encode %{
14064 __ fabsd(as_FloatRegister($dst$$reg),
14065 as_FloatRegister($src$$reg));
14066 %}
14067
14068 ins_pipe(fp_uop_d);
14069 %}
14070
14071 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14072 match(Set dst (AbsF (SubF src1 src2)));
14073
14074 ins_cost(INSN_COST * 3);
14075 format %{ "fabds $dst, $src1, $src2" %}
14076 ins_encode %{
14077 __ fabds(as_FloatRegister($dst$$reg),
14078 as_FloatRegister($src1$$reg),
14079 as_FloatRegister($src2$$reg));
14080 %}
14081
14082 ins_pipe(fp_uop_s);
14083 %}
14084
14085 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14086 match(Set dst (AbsD (SubD src1 src2)));
14087
14088 ins_cost(INSN_COST * 3);
14089 format %{ "fabdd $dst, $src1, $src2" %}
14090 ins_encode %{
14091 __ fabdd(as_FloatRegister($dst$$reg),
14092 as_FloatRegister($src1$$reg),
14093 as_FloatRegister($src2$$reg));
14094 %}
14095
14096 ins_pipe(fp_uop_d);
14097 %}
14098
14099 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14100 match(Set dst (SqrtD src));
14101
14102 ins_cost(INSN_COST * 50);
14103 format %{ "fsqrtd $dst, $src" %}
14104 ins_encode %{
14105 __ fsqrtd(as_FloatRegister($dst$$reg),
14106 as_FloatRegister($src$$reg));
14107 %}
14108
14109 ins_pipe(fp_div_s);
14110 %}
14111
14112 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14113 match(Set dst (SqrtF src));
14114
14115 ins_cost(INSN_COST * 50);
14116 format %{ "fsqrts $dst, $src" %}
14117 ins_encode %{
14118 __ fsqrts(as_FloatRegister($dst$$reg),
14119 as_FloatRegister($src$$reg));
14120 %}
14121
14122 ins_pipe(fp_div_d);
14123 %}
14124
14125 // Math.rint, floor, ceil
14126 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14127 match(Set dst (RoundDoubleMode src rmode));
14128 format %{ "frint $dst, $src, $rmode" %}
14129 ins_encode %{
14130 switch ($rmode$$constant) {
14131 case RoundDoubleModeNode::rmode_rint:
14132 __ frintnd(as_FloatRegister($dst$$reg),
14133 as_FloatRegister($src$$reg));
14134 break;
14135 case RoundDoubleModeNode::rmode_floor:
14136 __ frintmd(as_FloatRegister($dst$$reg),
14137 as_FloatRegister($src$$reg));
14138 break;
14139 case RoundDoubleModeNode::rmode_ceil:
14140 __ frintpd(as_FloatRegister($dst$$reg),
14141 as_FloatRegister($src$$reg));
14142 break;
14143 }
14144 %}
14145 ins_pipe(fp_uop_d);
14146 %}
14147
14148 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14149 match(Set dst (CopySignD src1 (Binary src2 zero)));
14150 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14151 format %{ "CopySignD $dst $src1 $src2" %}
14152 ins_encode %{
14153 FloatRegister dst = as_FloatRegister($dst$$reg),
14154 src1 = as_FloatRegister($src1$$reg),
14155 src2 = as_FloatRegister($src2$$reg),
14156 zero = as_FloatRegister($zero$$reg);
14157 __ fnegd(dst, zero);
14158 __ bsl(dst, __ T8B, src2, src1);
14159 %}
14160 ins_pipe(fp_uop_d);
14161 %}
14162
14163 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14164 match(Set dst (CopySignF src1 src2));
14165 effect(TEMP_DEF dst, USE src1, USE src2);
14166 format %{ "CopySignF $dst $src1 $src2" %}
14167 ins_encode %{
14168 FloatRegister dst = as_FloatRegister($dst$$reg),
14169 src1 = as_FloatRegister($src1$$reg),
14170 src2 = as_FloatRegister($src2$$reg);
14171 __ movi(dst, __ T2S, 0x80, 24);
14172 __ bsl(dst, __ T8B, src2, src1);
14173 %}
14174 ins_pipe(fp_uop_d);
14175 %}
14176
14177 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14178 match(Set dst (SignumD src (Binary zero one)));
14179 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14180 format %{ "signumD $dst, $src" %}
14181 ins_encode %{
14182 FloatRegister src = as_FloatRegister($src$$reg),
14183 dst = as_FloatRegister($dst$$reg),
14184 zero = as_FloatRegister($zero$$reg),
14185 one = as_FloatRegister($one$$reg);
14186 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14187 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14188 // Bit selection instruction gets bit from "one" for each enabled bit in
14189 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14190 // NaN the whole "src" will be copied because "dst" is zero. For all other
14191 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14192 // from "src", and all other bits are copied from 1.0.
14193 __ bsl(dst, __ T8B, one, src);
14194 %}
14195 ins_pipe(fp_uop_d);
14196 %}
14197
14198 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14199 match(Set dst (SignumF src (Binary zero one)));
14200 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14201 format %{ "signumF $dst, $src" %}
14202 ins_encode %{
14203 FloatRegister src = as_FloatRegister($src$$reg),
14204 dst = as_FloatRegister($dst$$reg),
14205 zero = as_FloatRegister($zero$$reg),
14206 one = as_FloatRegister($one$$reg);
14207 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14208 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14209 // Bit selection instruction gets bit from "one" for each enabled bit in
14210 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14211 // NaN the whole "src" will be copied because "dst" is zero. For all other
14212 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14213 // from "src", and all other bits are copied from 1.0.
14214 __ bsl(dst, __ T8B, one, src);
14215 %}
14216 ins_pipe(fp_uop_d);
14217 %}
14218
14219 instruct onspinwait() %{
14220 match(OnSpinWait);
14221 ins_cost(INSN_COST);
14222
14223 format %{ "onspinwait" %}
14224
14225 ins_encode %{
14226 __ spin_wait();
14227 %}
14228 ins_pipe(pipe_class_empty);
14229 %}
14230
14231 // ============================================================================
14232 // Logical Instructions
14233
14234 // Integer Logical Instructions
14235
14236 // And Instructions
14237
14238
14239 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14240 match(Set dst (AndI src1 src2));
14241
14242 format %{ "andw $dst, $src1, $src2\t# int" %}
14243
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ andw(as_Register($dst$$reg),
14247 as_Register($src1$$reg),
14248 as_Register($src2$$reg));
14249 %}
14250
14251 ins_pipe(ialu_reg_reg);
14252 %}
14253
14254 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14255 match(Set dst (AndI src1 src2));
14256
14257 format %{ "andsw $dst, $src1, $src2\t# int" %}
14258
14259 ins_cost(INSN_COST);
14260 ins_encode %{
14261 __ andw(as_Register($dst$$reg),
14262 as_Register($src1$$reg),
14263 (uint64_t)($src2$$constant));
14264 %}
14265
14266 ins_pipe(ialu_reg_imm);
14267 %}
14268
14269 // Or Instructions
14270
14271 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14272 match(Set dst (OrI src1 src2));
14273
14274 format %{ "orrw $dst, $src1, $src2\t# int" %}
14275
14276 ins_cost(INSN_COST);
14277 ins_encode %{
14278 __ orrw(as_Register($dst$$reg),
14279 as_Register($src1$$reg),
14280 as_Register($src2$$reg));
14281 %}
14282
14283 ins_pipe(ialu_reg_reg);
14284 %}
14285
14286 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14287 match(Set dst (OrI src1 src2));
14288
14289 format %{ "orrw $dst, $src1, $src2\t# int" %}
14290
14291 ins_cost(INSN_COST);
14292 ins_encode %{
14293 __ orrw(as_Register($dst$$reg),
14294 as_Register($src1$$reg),
14295 (uint64_t)($src2$$constant));
14296 %}
14297
14298 ins_pipe(ialu_reg_imm);
14299 %}
14300
14301 // Xor Instructions
14302
14303 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14304 match(Set dst (XorI src1 src2));
14305
14306 format %{ "eorw $dst, $src1, $src2\t# int" %}
14307
14308 ins_cost(INSN_COST);
14309 ins_encode %{
14310 __ eorw(as_Register($dst$$reg),
14311 as_Register($src1$$reg),
14312 as_Register($src2$$reg));
14313 %}
14314
14315 ins_pipe(ialu_reg_reg);
14316 %}
14317
14318 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14319 match(Set dst (XorI src1 src2));
14320
14321 format %{ "eorw $dst, $src1, $src2\t# int" %}
14322
14323 ins_cost(INSN_COST);
14324 ins_encode %{
14325 __ eorw(as_Register($dst$$reg),
14326 as_Register($src1$$reg),
14327 (uint64_t)($src2$$constant));
14328 %}
14329
14330 ins_pipe(ialu_reg_imm);
14331 %}
14332
14333 // Long Logical Instructions
14334 // TODO
14335
14336 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14337 match(Set dst (AndL src1 src2));
14338
14339 format %{ "and $dst, $src1, $src2\t# int" %}
14340
14341 ins_cost(INSN_COST);
14342 ins_encode %{
14343 __ andr(as_Register($dst$$reg),
14344 as_Register($src1$$reg),
14345 as_Register($src2$$reg));
14346 %}
14347
14348 ins_pipe(ialu_reg_reg);
14349 %}
14350
14351 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14352 match(Set dst (AndL src1 src2));
14353
14354 format %{ "and $dst, $src1, $src2\t# int" %}
14355
14356 ins_cost(INSN_COST);
14357 ins_encode %{
14358 __ andr(as_Register($dst$$reg),
14359 as_Register($src1$$reg),
14360 (uint64_t)($src2$$constant));
14361 %}
14362
14363 ins_pipe(ialu_reg_imm);
14364 %}
14365
14366 // Or Instructions
14367
14368 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14369 match(Set dst (OrL src1 src2));
14370
14371 format %{ "orr $dst, $src1, $src2\t# int" %}
14372
14373 ins_cost(INSN_COST);
14374 ins_encode %{
14375 __ orr(as_Register($dst$$reg),
14376 as_Register($src1$$reg),
14377 as_Register($src2$$reg));
14378 %}
14379
14380 ins_pipe(ialu_reg_reg);
14381 %}
14382
14383 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14384 match(Set dst (OrL src1 src2));
14385
14386 format %{ "orr $dst, $src1, $src2\t# int" %}
14387
14388 ins_cost(INSN_COST);
14389 ins_encode %{
14390 __ orr(as_Register($dst$$reg),
14391 as_Register($src1$$reg),
14392 (uint64_t)($src2$$constant));
14393 %}
14394
14395 ins_pipe(ialu_reg_imm);
14396 %}
14397
14398 // Xor Instructions
14399
14400 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14401 match(Set dst (XorL src1 src2));
14402
14403 format %{ "eor $dst, $src1, $src2\t# int" %}
14404
14405 ins_cost(INSN_COST);
14406 ins_encode %{
14407 __ eor(as_Register($dst$$reg),
14408 as_Register($src1$$reg),
14409 as_Register($src2$$reg));
14410 %}
14411
14412 ins_pipe(ialu_reg_reg);
14413 %}
14414
14415 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14416 match(Set dst (XorL src1 src2));
14417
14418 ins_cost(INSN_COST);
14419 format %{ "eor $dst, $src1, $src2\t# int" %}
14420
14421 ins_encode %{
14422 __ eor(as_Register($dst$$reg),
14423 as_Register($src1$$reg),
14424 (uint64_t)($src2$$constant));
14425 %}
14426
14427 ins_pipe(ialu_reg_imm);
14428 %}
14429
14430 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14431 %{
14432 match(Set dst (ConvI2L src));
14433
14434 ins_cost(INSN_COST);
14435 format %{ "sxtw $dst, $src\t# i2l" %}
14436 ins_encode %{
14437 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14438 %}
14439 ins_pipe(ialu_reg_shift);
14440 %}
14441
14442 // this pattern occurs in bigmath arithmetic
14443 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14444 %{
14445 match(Set dst (AndL (ConvI2L src) mask));
14446
14447 ins_cost(INSN_COST);
14448 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14449 ins_encode %{
14450 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14451 %}
14452
14453 ins_pipe(ialu_reg_shift);
14454 %}
14455
14456 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14457 match(Set dst (ConvL2I src));
14458
14459 ins_cost(INSN_COST);
14460 format %{ "movw $dst, $src \t// l2i" %}
14461
14462 ins_encode %{
14463 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14464 %}
14465
14466 ins_pipe(ialu_reg);
14467 %}
14468
14469 instruct convD2F_reg(vRegF dst, vRegD src) %{
14470 match(Set dst (ConvD2F src));
14471
14472 ins_cost(INSN_COST * 5);
14473 format %{ "fcvtd $dst, $src \t// d2f" %}
14474
14475 ins_encode %{
14476 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14477 %}
14478
14479 ins_pipe(fp_d2f);
14480 %}
14481
14482 instruct convF2D_reg(vRegD dst, vRegF src) %{
14483 match(Set dst (ConvF2D src));
14484
14485 ins_cost(INSN_COST * 5);
14486 format %{ "fcvts $dst, $src \t// f2d" %}
14487
14488 ins_encode %{
14489 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14490 %}
14491
14492 ins_pipe(fp_f2d);
14493 %}
14494
14495 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14496 match(Set dst (ConvF2I src));
14497
14498 ins_cost(INSN_COST * 5);
14499 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14500
14501 ins_encode %{
14502 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14503 %}
14504
14505 ins_pipe(fp_f2i);
14506 %}
14507
14508 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14509 match(Set dst (ConvF2L src));
14510
14511 ins_cost(INSN_COST * 5);
14512 format %{ "fcvtzs $dst, $src \t// f2l" %}
14513
14514 ins_encode %{
14515 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14516 %}
14517
14518 ins_pipe(fp_f2l);
14519 %}
14520
14521 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14522 match(Set dst (ConvF2HF src));
14523 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14524 "smov $dst, $tmp\t# move result from $tmp to $dst"
14525 %}
14526 effect(TEMP tmp);
14527 ins_encode %{
14528 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14529 %}
14530 ins_pipe(pipe_slow);
14531 %}
14532
14533 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14534 match(Set dst (ConvHF2F src));
14535 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14536 "fcvt $dst, $tmp\t# convert half to single precision"
14537 %}
14538 effect(TEMP tmp);
14539 ins_encode %{
14540 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14541 %}
14542 ins_pipe(pipe_slow);
14543 %}
14544
14545 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14546 match(Set dst (ConvI2F src));
14547
14548 ins_cost(INSN_COST * 5);
14549 format %{ "scvtfws $dst, $src \t// i2f" %}
14550
14551 ins_encode %{
14552 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14553 %}
14554
14555 ins_pipe(fp_i2f);
14556 %}
14557
14558 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14559 match(Set dst (ConvL2F src));
14560
14561 ins_cost(INSN_COST * 5);
14562 format %{ "scvtfs $dst, $src \t// l2f" %}
14563
14564 ins_encode %{
14565 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14566 %}
14567
14568 ins_pipe(fp_l2f);
14569 %}
14570
14571 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14572 match(Set dst (ConvD2I src));
14573
14574 ins_cost(INSN_COST * 5);
14575 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14576
14577 ins_encode %{
14578 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14579 %}
14580
14581 ins_pipe(fp_d2i);
14582 %}
14583
14584 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14585 match(Set dst (ConvD2L src));
14586
14587 ins_cost(INSN_COST * 5);
14588 format %{ "fcvtzd $dst, $src \t// d2l" %}
14589
14590 ins_encode %{
14591 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14592 %}
14593
14594 ins_pipe(fp_d2l);
14595 %}
14596
14597 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14598 match(Set dst (ConvI2D src));
14599
14600 ins_cost(INSN_COST * 5);
14601 format %{ "scvtfwd $dst, $src \t// i2d" %}
14602
14603 ins_encode %{
14604 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14605 %}
14606
14607 ins_pipe(fp_i2d);
14608 %}
14609
14610 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14611 match(Set dst (ConvL2D src));
14612
14613 ins_cost(INSN_COST * 5);
14614 format %{ "scvtfd $dst, $src \t// l2d" %}
14615
14616 ins_encode %{
14617 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14618 %}
14619
14620 ins_pipe(fp_l2d);
14621 %}
14622
14623 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14624 %{
14625 match(Set dst (RoundD src));
14626 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14627 format %{ "java_round_double $dst,$src"%}
14628 ins_encode %{
14629 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14630 as_FloatRegister($ftmp$$reg));
14631 %}
14632 ins_pipe(pipe_slow);
14633 %}
14634
14635 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14636 %{
14637 match(Set dst (RoundF src));
14638 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14639 format %{ "java_round_float $dst,$src"%}
14640 ins_encode %{
14641 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14642 as_FloatRegister($ftmp$$reg));
14643 %}
14644 ins_pipe(pipe_slow);
14645 %}
14646
14647 // stack <-> reg and reg <-> reg shuffles with no conversion
14648
14649 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14650
14651 match(Set dst (MoveF2I src));
14652
14653 effect(DEF dst, USE src);
14654
14655 ins_cost(4 * INSN_COST);
14656
14657 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14658
14659 ins_encode %{
14660 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14661 %}
14662
14663 ins_pipe(iload_reg_reg);
14664
14665 %}
14666
14667 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14668
14669 match(Set dst (MoveI2F src));
14670
14671 effect(DEF dst, USE src);
14672
14673 ins_cost(4 * INSN_COST);
14674
14675 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14676
14677 ins_encode %{
14678 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14679 %}
14680
14681 ins_pipe(pipe_class_memory);
14682
14683 %}
14684
14685 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14686
14687 match(Set dst (MoveD2L src));
14688
14689 effect(DEF dst, USE src);
14690
14691 ins_cost(4 * INSN_COST);
14692
14693 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14694
14695 ins_encode %{
14696 __ ldr($dst$$Register, Address(sp, $src$$disp));
14697 %}
14698
14699 ins_pipe(iload_reg_reg);
14700
14701 %}
14702
14703 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14704
14705 match(Set dst (MoveL2D src));
14706
14707 effect(DEF dst, USE src);
14708
14709 ins_cost(4 * INSN_COST);
14710
14711 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14712
14713 ins_encode %{
14714 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14715 %}
14716
14717 ins_pipe(pipe_class_memory);
14718
14719 %}
14720
14721 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14722
14723 match(Set dst (MoveF2I src));
14724
14725 effect(DEF dst, USE src);
14726
14727 ins_cost(INSN_COST);
14728
14729 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14730
14731 ins_encode %{
14732 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14733 %}
14734
14735 ins_pipe(pipe_class_memory);
14736
14737 %}
14738
14739 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14740
14741 match(Set dst (MoveI2F src));
14742
14743 effect(DEF dst, USE src);
14744
14745 ins_cost(INSN_COST);
14746
14747 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14748
14749 ins_encode %{
14750 __ strw($src$$Register, Address(sp, $dst$$disp));
14751 %}
14752
14753 ins_pipe(istore_reg_reg);
14754
14755 %}
14756
14757 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14758
14759 match(Set dst (MoveD2L src));
14760
14761 effect(DEF dst, USE src);
14762
14763 ins_cost(INSN_COST);
14764
14765 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14766
14767 ins_encode %{
14768 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14769 %}
14770
14771 ins_pipe(pipe_class_memory);
14772
14773 %}
14774
14775 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14776
14777 match(Set dst (MoveL2D src));
14778
14779 effect(DEF dst, USE src);
14780
14781 ins_cost(INSN_COST);
14782
14783 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14784
14785 ins_encode %{
14786 __ str($src$$Register, Address(sp, $dst$$disp));
14787 %}
14788
14789 ins_pipe(istore_reg_reg);
14790
14791 %}
14792
14793 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14794
14795 match(Set dst (MoveF2I src));
14796
14797 effect(DEF dst, USE src);
14798
14799 ins_cost(INSN_COST);
14800
14801 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14802
14803 ins_encode %{
14804 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14805 %}
14806
14807 ins_pipe(fp_f2i);
14808
14809 %}
14810
14811 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14812
14813 match(Set dst (MoveI2F src));
14814
14815 effect(DEF dst, USE src);
14816
14817 ins_cost(INSN_COST);
14818
14819 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14820
14821 ins_encode %{
14822 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14823 %}
14824
14825 ins_pipe(fp_i2f);
14826
14827 %}
14828
14829 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14830
14831 match(Set dst (MoveD2L src));
14832
14833 effect(DEF dst, USE src);
14834
14835 ins_cost(INSN_COST);
14836
14837 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14838
14839 ins_encode %{
14840 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14841 %}
14842
14843 ins_pipe(fp_d2l);
14844
14845 %}
14846
14847 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14848
14849 match(Set dst (MoveL2D src));
14850
14851 effect(DEF dst, USE src);
14852
14853 ins_cost(INSN_COST);
14854
14855 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14856
14857 ins_encode %{
14858 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14859 %}
14860
14861 ins_pipe(fp_l2d);
14862
14863 %}
14864
14865 // ============================================================================
14866 // clearing of an array
14867
14868 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14869 %{
14870 match(Set dummy (ClearArray (Binary cnt base) zero));
14871 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14872
14873 ins_cost(4 * INSN_COST);
14874 format %{ "ClearArray $cnt, $base" %}
14875
14876 ins_encode %{
14877 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14878 if (tpc == nullptr) {
14879 ciEnv::current()->record_failure("CodeCache is full");
14880 return;
14881 }
14882 %}
14883
14884 ins_pipe(pipe_class_memory);
14885 %}
14886
14887 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14888 %{
14889 predicate(((ClearArrayNode*)n)->word_copy_only());
14890 match(Set dummy (ClearArray (Binary cnt base) val));
14891 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14892
14893 ins_cost(4 * INSN_COST);
14894 format %{ "ClearArray $cnt, $base, $val" %}
14895
14896 ins_encode %{
14897 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14898 %}
14899
14900 ins_pipe(pipe_class_memory);
14901 %}
14902
14903 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14904 %{
14905 predicate((uint64_t)n->in(2)->get_long()
14906 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14907 && !((ClearArrayNode*)n)->word_copy_only());
14908 match(Set dummy (ClearArray cnt base));
14909 effect(TEMP temp, USE_KILL base, KILL cr);
14910
14911 ins_cost(4 * INSN_COST);
14912 format %{ "ClearArray $cnt, $base" %}
14913
14914 ins_encode %{
14915 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14916 if (tpc == nullptr) {
14917 ciEnv::current()->record_failure("CodeCache is full");
14918 return;
14919 }
14920 %}
14921
14922 ins_pipe(pipe_class_memory);
14923 %}
14924
14925 // ============================================================================
14926 // Overflow Math Instructions
14927
14928 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14929 %{
14930 match(Set cr (OverflowAddI op1 op2));
14931
14932 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14933 ins_cost(INSN_COST);
14934 ins_encode %{
14935 __ cmnw($op1$$Register, $op2$$Register);
14936 %}
14937
14938 ins_pipe(icmp_reg_reg);
14939 %}
14940
14941 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14942 %{
14943 match(Set cr (OverflowAddI op1 op2));
14944
14945 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14946 ins_cost(INSN_COST);
14947 ins_encode %{
14948 __ cmnw($op1$$Register, $op2$$constant);
14949 %}
14950
14951 ins_pipe(icmp_reg_imm);
14952 %}
14953
14954 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14955 %{
14956 match(Set cr (OverflowAddL op1 op2));
14957
14958 format %{ "cmn $op1, $op2\t# overflow check long" %}
14959 ins_cost(INSN_COST);
14960 ins_encode %{
14961 __ cmn($op1$$Register, $op2$$Register);
14962 %}
14963
14964 ins_pipe(icmp_reg_reg);
14965 %}
14966
14967 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14968 %{
14969 match(Set cr (OverflowAddL op1 op2));
14970
14971 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14972 ins_cost(INSN_COST);
14973 ins_encode %{
14974 __ adds(zr, $op1$$Register, $op2$$constant);
14975 %}
14976
14977 ins_pipe(icmp_reg_imm);
14978 %}
14979
14980 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14981 %{
14982 match(Set cr (OverflowSubI op1 op2));
14983
14984 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14985 ins_cost(INSN_COST);
14986 ins_encode %{
14987 __ cmpw($op1$$Register, $op2$$Register);
14988 %}
14989
14990 ins_pipe(icmp_reg_reg);
14991 %}
14992
14993 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14994 %{
14995 match(Set cr (OverflowSubI op1 op2));
14996
14997 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14998 ins_cost(INSN_COST);
14999 ins_encode %{
15000 __ cmpw($op1$$Register, $op2$$constant);
15001 %}
15002
15003 ins_pipe(icmp_reg_imm);
15004 %}
15005
15006 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15007 %{
15008 match(Set cr (OverflowSubL op1 op2));
15009
15010 format %{ "cmp $op1, $op2\t# overflow check long" %}
15011 ins_cost(INSN_COST);
15012 ins_encode %{
15013 __ cmp($op1$$Register, $op2$$Register);
15014 %}
15015
15016 ins_pipe(icmp_reg_reg);
15017 %}
15018
15019 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15020 %{
15021 match(Set cr (OverflowSubL op1 op2));
15022
15023 format %{ "cmp $op1, $op2\t# overflow check long" %}
15024 ins_cost(INSN_COST);
15025 ins_encode %{
15026 __ subs(zr, $op1$$Register, $op2$$constant);
15027 %}
15028
15029 ins_pipe(icmp_reg_imm);
15030 %}
15031
15032 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15033 %{
15034 match(Set cr (OverflowSubI zero op1));
15035
15036 format %{ "cmpw zr, $op1\t# overflow check int" %}
15037 ins_cost(INSN_COST);
15038 ins_encode %{
15039 __ cmpw(zr, $op1$$Register);
15040 %}
15041
15042 ins_pipe(icmp_reg_imm);
15043 %}
15044
15045 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15046 %{
15047 match(Set cr (OverflowSubL zero op1));
15048
15049 format %{ "cmp zr, $op1\t# overflow check long" %}
15050 ins_cost(INSN_COST);
15051 ins_encode %{
15052 __ cmp(zr, $op1$$Register);
15053 %}
15054
15055 ins_pipe(icmp_reg_imm);
15056 %}
15057
15058 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15059 %{
15060 match(Set cr (OverflowMulI op1 op2));
15061
15062 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15063 "cmp rscratch1, rscratch1, sxtw\n\t"
15064 "movw rscratch1, #0x80000000\n\t"
15065 "cselw rscratch1, rscratch1, zr, NE\n\t"
15066 "cmpw rscratch1, #1" %}
15067 ins_cost(5 * INSN_COST);
15068 ins_encode %{
15069 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15070 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15071 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15072 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15073 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15074 %}
15075
15076 ins_pipe(pipe_slow);
15077 %}
15078
15079 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15080 %{
15081 match(If cmp (OverflowMulI op1 op2));
15082 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15083 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15084 effect(USE labl, KILL cr);
15085
15086 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15087 "cmp rscratch1, rscratch1, sxtw\n\t"
15088 "b$cmp $labl" %}
15089 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15090 ins_encode %{
15091 Label* L = $labl$$label;
15092 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15093 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15094 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15095 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15096 %}
15097
15098 ins_pipe(pipe_serial);
15099 %}
15100
15101 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15102 %{
15103 match(Set cr (OverflowMulL op1 op2));
15104
15105 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15106 "smulh rscratch2, $op1, $op2\n\t"
15107 "cmp rscratch2, rscratch1, ASR #63\n\t"
15108 "movw rscratch1, #0x80000000\n\t"
15109 "cselw rscratch1, rscratch1, zr, NE\n\t"
15110 "cmpw rscratch1, #1" %}
15111 ins_cost(6 * INSN_COST);
15112 ins_encode %{
15113 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15114 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15115 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15116 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15117 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15118 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15119 %}
15120
15121 ins_pipe(pipe_slow);
15122 %}
15123
15124 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15125 %{
15126 match(If cmp (OverflowMulL op1 op2));
15127 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15128 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15129 effect(USE labl, KILL cr);
15130
15131 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15132 "smulh rscratch2, $op1, $op2\n\t"
15133 "cmp rscratch2, rscratch1, ASR #63\n\t"
15134 "b$cmp $labl" %}
15135 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15136 ins_encode %{
15137 Label* L = $labl$$label;
15138 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15139 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15140 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15141 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15142 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15143 %}
15144
15145 ins_pipe(pipe_serial);
15146 %}
15147
15148 // ============================================================================
15149 // Compare Instructions
15150
15151 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15152 %{
15153 match(Set cr (CmpI op1 op2));
15154
15155 effect(DEF cr, USE op1, USE op2);
15156
15157 ins_cost(INSN_COST);
15158 format %{ "cmpw $op1, $op2" %}
15159
15160 ins_encode(aarch64_enc_cmpw(op1, op2));
15161
15162 ins_pipe(icmp_reg_reg);
15163 %}
15164
15165 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15166 %{
15167 match(Set cr (CmpI op1 zero));
15168
15169 effect(DEF cr, USE op1);
15170
15171 ins_cost(INSN_COST);
15172 format %{ "cmpw $op1, 0" %}
15173
15174 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15175
15176 ins_pipe(icmp_reg_imm);
15177 %}
15178
15179 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15180 %{
15181 match(Set cr (CmpI op1 op2));
15182
15183 effect(DEF cr, USE op1);
15184
15185 ins_cost(INSN_COST);
15186 format %{ "cmpw $op1, $op2" %}
15187
15188 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15189
15190 ins_pipe(icmp_reg_imm);
15191 %}
15192
15193 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15194 %{
15195 match(Set cr (CmpI op1 op2));
15196
15197 effect(DEF cr, USE op1);
15198
15199 ins_cost(INSN_COST * 2);
15200 format %{ "cmpw $op1, $op2" %}
15201
15202 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15203
15204 ins_pipe(icmp_reg_imm);
15205 %}
15206
15207 // Unsigned compare Instructions; really, same as signed compare
15208 // except it should only be used to feed an If or a CMovI which takes a
15209 // cmpOpU.
15210
15211 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15212 %{
15213 match(Set cr (CmpU op1 op2));
15214
15215 effect(DEF cr, USE op1, USE op2);
15216
15217 ins_cost(INSN_COST);
15218 format %{ "cmpw $op1, $op2\t# unsigned" %}
15219
15220 ins_encode(aarch64_enc_cmpw(op1, op2));
15221
15222 ins_pipe(icmp_reg_reg);
15223 %}
15224
15225 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15226 %{
15227 match(Set cr (CmpU op1 zero));
15228
15229 effect(DEF cr, USE op1);
15230
15231 ins_cost(INSN_COST);
15232 format %{ "cmpw $op1, #0\t# unsigned" %}
15233
15234 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15235
15236 ins_pipe(icmp_reg_imm);
15237 %}
15238
15239 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15240 %{
15241 match(Set cr (CmpU op1 op2));
15242
15243 effect(DEF cr, USE op1);
15244
15245 ins_cost(INSN_COST);
15246 format %{ "cmpw $op1, $op2\t# unsigned" %}
15247
15248 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15249
15250 ins_pipe(icmp_reg_imm);
15251 %}
15252
15253 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15254 %{
15255 match(Set cr (CmpU op1 op2));
15256
15257 effect(DEF cr, USE op1);
15258
15259 ins_cost(INSN_COST * 2);
15260 format %{ "cmpw $op1, $op2\t# unsigned" %}
15261
15262 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15263
15264 ins_pipe(icmp_reg_imm);
15265 %}
15266
15267 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15268 %{
15269 match(Set cr (CmpL op1 op2));
15270
15271 effect(DEF cr, USE op1, USE op2);
15272
15273 ins_cost(INSN_COST);
15274 format %{ "cmp $op1, $op2" %}
15275
15276 ins_encode(aarch64_enc_cmp(op1, op2));
15277
15278 ins_pipe(icmp_reg_reg);
15279 %}
15280
15281 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15282 %{
15283 match(Set cr (CmpL op1 zero));
15284
15285 effect(DEF cr, USE op1);
15286
15287 ins_cost(INSN_COST);
15288 format %{ "tst $op1" %}
15289
15290 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15291
15292 ins_pipe(icmp_reg_imm);
15293 %}
15294
15295 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15296 %{
15297 match(Set cr (CmpL op1 op2));
15298
15299 effect(DEF cr, USE op1);
15300
15301 ins_cost(INSN_COST);
15302 format %{ "cmp $op1, $op2" %}
15303
15304 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15305
15306 ins_pipe(icmp_reg_imm);
15307 %}
15308
15309 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15310 %{
15311 match(Set cr (CmpL op1 op2));
15312
15313 effect(DEF cr, USE op1);
15314
15315 ins_cost(INSN_COST * 2);
15316 format %{ "cmp $op1, $op2" %}
15317
15318 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15319
15320 ins_pipe(icmp_reg_imm);
15321 %}
15322
15323 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15324 %{
15325 match(Set cr (CmpUL op1 op2));
15326
15327 effect(DEF cr, USE op1, USE op2);
15328
15329 ins_cost(INSN_COST);
15330 format %{ "cmp $op1, $op2" %}
15331
15332 ins_encode(aarch64_enc_cmp(op1, op2));
15333
15334 ins_pipe(icmp_reg_reg);
15335 %}
15336
15337 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15338 %{
15339 match(Set cr (CmpUL op1 zero));
15340
15341 effect(DEF cr, USE op1);
15342
15343 ins_cost(INSN_COST);
15344 format %{ "tst $op1" %}
15345
15346 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15347
15348 ins_pipe(icmp_reg_imm);
15349 %}
15350
15351 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15352 %{
15353 match(Set cr (CmpUL op1 op2));
15354
15355 effect(DEF cr, USE op1);
15356
15357 ins_cost(INSN_COST);
15358 format %{ "cmp $op1, $op2" %}
15359
15360 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15361
15362 ins_pipe(icmp_reg_imm);
15363 %}
15364
15365 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15366 %{
15367 match(Set cr (CmpUL op1 op2));
15368
15369 effect(DEF cr, USE op1);
15370
15371 ins_cost(INSN_COST * 2);
15372 format %{ "cmp $op1, $op2" %}
15373
15374 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15375
15376 ins_pipe(icmp_reg_imm);
15377 %}
15378
15379 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15380 %{
15381 match(Set cr (CmpP op1 op2));
15382
15383 effect(DEF cr, USE op1, USE op2);
15384
15385 ins_cost(INSN_COST);
15386 format %{ "cmp $op1, $op2\t // ptr" %}
15387
15388 ins_encode(aarch64_enc_cmpp(op1, op2));
15389
15390 ins_pipe(icmp_reg_reg);
15391 %}
15392
15393 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15394 %{
15395 match(Set cr (CmpN op1 op2));
15396
15397 effect(DEF cr, USE op1, USE op2);
15398
15399 ins_cost(INSN_COST);
15400 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15401
15402 ins_encode(aarch64_enc_cmpn(op1, op2));
15403
15404 ins_pipe(icmp_reg_reg);
15405 %}
15406
15407 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15408 %{
15409 match(Set cr (CmpP op1 zero));
15410
15411 effect(DEF cr, USE op1, USE zero);
15412
15413 ins_cost(INSN_COST);
15414 format %{ "cmp $op1, 0\t // ptr" %}
15415
15416 ins_encode(aarch64_enc_testp(op1));
15417
15418 ins_pipe(icmp_reg_imm);
15419 %}
15420
15421 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15422 %{
15423 match(Set cr (CmpN op1 zero));
15424
15425 effect(DEF cr, USE op1, USE zero);
15426
15427 ins_cost(INSN_COST);
15428 format %{ "cmp $op1, 0\t // compressed ptr" %}
15429
15430 ins_encode(aarch64_enc_testn(op1));
15431
15432 ins_pipe(icmp_reg_imm);
15433 %}
15434
15435 // FP comparisons
15436 //
15437 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15438 // using normal cmpOp. See declaration of rFlagsReg for details.
15439
15440 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15441 %{
15442 match(Set cr (CmpF src1 src2));
15443
15444 ins_cost(3 * INSN_COST);
15445 format %{ "fcmps $src1, $src2" %}
15446
15447 ins_encode %{
15448 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15449 %}
15450
15451 ins_pipe(pipe_class_compare);
15452 %}
15453
15454 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15455 %{
15456 match(Set cr (CmpF src1 src2));
15457
15458 ins_cost(3 * INSN_COST);
15459 format %{ "fcmps $src1, 0.0" %}
15460
15461 ins_encode %{
15462 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15463 %}
15464
15465 ins_pipe(pipe_class_compare);
15466 %}
15467 // FROM HERE
15468
15469 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15470 %{
15471 match(Set cr (CmpD src1 src2));
15472
15473 ins_cost(3 * INSN_COST);
15474 format %{ "fcmpd $src1, $src2" %}
15475
15476 ins_encode %{
15477 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15478 %}
15479
15480 ins_pipe(pipe_class_compare);
15481 %}
15482
15483 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15484 %{
15485 match(Set cr (CmpD src1 src2));
15486
15487 ins_cost(3 * INSN_COST);
15488 format %{ "fcmpd $src1, 0.0" %}
15489
15490 ins_encode %{
15491 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15492 %}
15493
15494 ins_pipe(pipe_class_compare);
15495 %}
15496
15497 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15498 %{
15499 match(Set dst (CmpF3 src1 src2));
15500 effect(KILL cr);
15501
15502 ins_cost(5 * INSN_COST);
15503 format %{ "fcmps $src1, $src2\n\t"
15504 "csinvw($dst, zr, zr, eq\n\t"
15505 "csnegw($dst, $dst, $dst, lt)"
15506 %}
15507
15508 ins_encode %{
15509 Label done;
15510 FloatRegister s1 = as_FloatRegister($src1$$reg);
15511 FloatRegister s2 = as_FloatRegister($src2$$reg);
15512 Register d = as_Register($dst$$reg);
15513 __ fcmps(s1, s2);
15514 // installs 0 if EQ else -1
15515 __ csinvw(d, zr, zr, Assembler::EQ);
15516 // keeps -1 if less or unordered else installs 1
15517 __ csnegw(d, d, d, Assembler::LT);
15518 __ bind(done);
15519 %}
15520
15521 ins_pipe(pipe_class_default);
15522
15523 %}
15524
15525 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15526 %{
15527 match(Set dst (CmpD3 src1 src2));
15528 effect(KILL cr);
15529
15530 ins_cost(5 * INSN_COST);
15531 format %{ "fcmpd $src1, $src2\n\t"
15532 "csinvw($dst, zr, zr, eq\n\t"
15533 "csnegw($dst, $dst, $dst, lt)"
15534 %}
15535
15536 ins_encode %{
15537 Label done;
15538 FloatRegister s1 = as_FloatRegister($src1$$reg);
15539 FloatRegister s2 = as_FloatRegister($src2$$reg);
15540 Register d = as_Register($dst$$reg);
15541 __ fcmpd(s1, s2);
15542 // installs 0 if EQ else -1
15543 __ csinvw(d, zr, zr, Assembler::EQ);
15544 // keeps -1 if less or unordered else installs 1
15545 __ csnegw(d, d, d, Assembler::LT);
15546 __ bind(done);
15547 %}
15548 ins_pipe(pipe_class_default);
15549
15550 %}
15551
15552 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15553 %{
15554 match(Set dst (CmpF3 src1 zero));
15555 effect(KILL cr);
15556
15557 ins_cost(5 * INSN_COST);
15558 format %{ "fcmps $src1, 0.0\n\t"
15559 "csinvw($dst, zr, zr, eq\n\t"
15560 "csnegw($dst, $dst, $dst, lt)"
15561 %}
15562
15563 ins_encode %{
15564 Label done;
15565 FloatRegister s1 = as_FloatRegister($src1$$reg);
15566 Register d = as_Register($dst$$reg);
15567 __ fcmps(s1, 0.0);
15568 // installs 0 if EQ else -1
15569 __ csinvw(d, zr, zr, Assembler::EQ);
15570 // keeps -1 if less or unordered else installs 1
15571 __ csnegw(d, d, d, Assembler::LT);
15572 __ bind(done);
15573 %}
15574
15575 ins_pipe(pipe_class_default);
15576
15577 %}
15578
15579 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15580 %{
15581 match(Set dst (CmpD3 src1 zero));
15582 effect(KILL cr);
15583
15584 ins_cost(5 * INSN_COST);
15585 format %{ "fcmpd $src1, 0.0\n\t"
15586 "csinvw($dst, zr, zr, eq\n\t"
15587 "csnegw($dst, $dst, $dst, lt)"
15588 %}
15589
15590 ins_encode %{
15591 Label done;
15592 FloatRegister s1 = as_FloatRegister($src1$$reg);
15593 Register d = as_Register($dst$$reg);
15594 __ fcmpd(s1, 0.0);
15595 // installs 0 if EQ else -1
15596 __ csinvw(d, zr, zr, Assembler::EQ);
15597 // keeps -1 if less or unordered else installs 1
15598 __ csnegw(d, d, d, Assembler::LT);
15599 __ bind(done);
15600 %}
15601 ins_pipe(pipe_class_default);
15602
15603 %}
15604
15605 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15606 %{
15607 match(Set dst (CmpLTMask p q));
15608 effect(KILL cr);
15609
15610 ins_cost(3 * INSN_COST);
15611
15612 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15613 "csetw $dst, lt\n\t"
15614 "subw $dst, zr, $dst"
15615 %}
15616
15617 ins_encode %{
15618 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15619 __ csetw(as_Register($dst$$reg), Assembler::LT);
15620 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15621 %}
15622
15623 ins_pipe(ialu_reg_reg);
15624 %}
15625
15626 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15627 %{
15628 match(Set dst (CmpLTMask src zero));
15629 effect(KILL cr);
15630
15631 ins_cost(INSN_COST);
15632
15633 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15634
15635 ins_encode %{
15636 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15637 %}
15638
15639 ins_pipe(ialu_reg_shift);
15640 %}
15641
15642 // ============================================================================
15643 // Max and Min
15644
15645 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15646
15647 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15648 %{
15649 effect(DEF cr, USE src);
15650 ins_cost(INSN_COST);
15651 format %{ "cmpw $src, 0" %}
15652
15653 ins_encode %{
15654 __ cmpw($src$$Register, 0);
15655 %}
15656 ins_pipe(icmp_reg_imm);
15657 %}
15658
15659 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15660 %{
15661 match(Set dst (MinI src1 src2));
15662 ins_cost(INSN_COST * 3);
15663
15664 expand %{
15665 rFlagsReg cr;
15666 compI_reg_reg(cr, src1, src2);
15667 cmovI_reg_reg_lt(dst, src1, src2, cr);
15668 %}
15669 %}
15670
15671 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15672 %{
15673 match(Set dst (MaxI src1 src2));
15674 ins_cost(INSN_COST * 3);
15675
15676 expand %{
15677 rFlagsReg cr;
15678 compI_reg_reg(cr, src1, src2);
15679 cmovI_reg_reg_gt(dst, src1, src2, cr);
15680 %}
15681 %}
15682
15683
15684 // ============================================================================
15685 // Branch Instructions
15686
15687 // Direct Branch.
15688 instruct branch(label lbl)
15689 %{
15690 match(Goto);
15691
15692 effect(USE lbl);
15693
15694 ins_cost(BRANCH_COST);
15695 format %{ "b $lbl" %}
15696
15697 ins_encode(aarch64_enc_b(lbl));
15698
15699 ins_pipe(pipe_branch);
15700 %}
15701
15702 // Conditional Near Branch
15703 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15704 %{
15705 // Same match rule as `branchConFar'.
15706 match(If cmp cr);
15707
15708 effect(USE lbl);
15709
15710 ins_cost(BRANCH_COST);
15711 // If set to 1 this indicates that the current instruction is a
15712 // short variant of a long branch. This avoids using this
15713 // instruction in first-pass matching. It will then only be used in
15714 // the `Shorten_branches' pass.
15715 // ins_short_branch(1);
15716 format %{ "b$cmp $lbl" %}
15717
15718 ins_encode(aarch64_enc_br_con(cmp, lbl));
15719
15720 ins_pipe(pipe_branch_cond);
15721 %}
15722
15723 // Conditional Near Branch Unsigned
15724 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15725 %{
15726 // Same match rule as `branchConFar'.
15727 match(If cmp cr);
15728
15729 effect(USE lbl);
15730
15731 ins_cost(BRANCH_COST);
15732 // If set to 1 this indicates that the current instruction is a
15733 // short variant of a long branch. This avoids using this
15734 // instruction in first-pass matching. It will then only be used in
15735 // the `Shorten_branches' pass.
15736 // ins_short_branch(1);
15737 format %{ "b$cmp $lbl\t# unsigned" %}
15738
15739 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15740
15741 ins_pipe(pipe_branch_cond);
15742 %}
15743
15744 // Make use of CBZ and CBNZ. These instructions, as well as being
15745 // shorter than (cmp; branch), have the additional benefit of not
15746 // killing the flags.
15747
15748 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15749 match(If cmp (CmpI op1 op2));
15750 effect(USE labl);
15751
15752 ins_cost(BRANCH_COST);
15753 format %{ "cbw$cmp $op1, $labl" %}
15754 ins_encode %{
15755 Label* L = $labl$$label;
15756 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15757 if (cond == Assembler::EQ)
15758 __ cbzw($op1$$Register, *L);
15759 else
15760 __ cbnzw($op1$$Register, *L);
15761 %}
15762 ins_pipe(pipe_cmp_branch);
15763 %}
15764
15765 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15766 match(If cmp (CmpL op1 op2));
15767 effect(USE labl);
15768
15769 ins_cost(BRANCH_COST);
15770 format %{ "cb$cmp $op1, $labl" %}
15771 ins_encode %{
15772 Label* L = $labl$$label;
15773 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15774 if (cond == Assembler::EQ)
15775 __ cbz($op1$$Register, *L);
15776 else
15777 __ cbnz($op1$$Register, *L);
15778 %}
15779 ins_pipe(pipe_cmp_branch);
15780 %}
15781
15782 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15783 match(If cmp (CmpP op1 op2));
15784 effect(USE labl);
15785
15786 ins_cost(BRANCH_COST);
15787 format %{ "cb$cmp $op1, $labl" %}
15788 ins_encode %{
15789 Label* L = $labl$$label;
15790 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15791 if (cond == Assembler::EQ)
15792 __ cbz($op1$$Register, *L);
15793 else
15794 __ cbnz($op1$$Register, *L);
15795 %}
15796 ins_pipe(pipe_cmp_branch);
15797 %}
15798
15799 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15800 match(If cmp (CmpN op1 op2));
15801 effect(USE labl);
15802
15803 ins_cost(BRANCH_COST);
15804 format %{ "cbw$cmp $op1, $labl" %}
15805 ins_encode %{
15806 Label* L = $labl$$label;
15807 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15808 if (cond == Assembler::EQ)
15809 __ cbzw($op1$$Register, *L);
15810 else
15811 __ cbnzw($op1$$Register, *L);
15812 %}
15813 ins_pipe(pipe_cmp_branch);
15814 %}
15815
15816 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15817 match(If cmp (CmpP (DecodeN oop) zero));
15818 effect(USE labl);
15819
15820 ins_cost(BRANCH_COST);
15821 format %{ "cb$cmp $oop, $labl" %}
15822 ins_encode %{
15823 Label* L = $labl$$label;
15824 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15825 if (cond == Assembler::EQ)
15826 __ cbzw($oop$$Register, *L);
15827 else
15828 __ cbnzw($oop$$Register, *L);
15829 %}
15830 ins_pipe(pipe_cmp_branch);
15831 %}
15832
15833 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15834 match(If cmp (CmpU op1 op2));
15835 effect(USE labl);
15836
15837 ins_cost(BRANCH_COST);
15838 format %{ "cbw$cmp $op1, $labl" %}
15839 ins_encode %{
15840 Label* L = $labl$$label;
15841 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15842 if (cond == Assembler::EQ || cond == Assembler::LS) {
15843 __ cbzw($op1$$Register, *L);
15844 } else {
15845 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15846 __ cbnzw($op1$$Register, *L);
15847 }
15848 %}
15849 ins_pipe(pipe_cmp_branch);
15850 %}
15851
15852 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15853 match(If cmp (CmpUL op1 op2));
15854 effect(USE labl);
15855
15856 ins_cost(BRANCH_COST);
15857 format %{ "cb$cmp $op1, $labl" %}
15858 ins_encode %{
15859 Label* L = $labl$$label;
15860 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15861 if (cond == Assembler::EQ || cond == Assembler::LS) {
15862 __ cbz($op1$$Register, *L);
15863 } else {
15864 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15865 __ cbnz($op1$$Register, *L);
15866 }
15867 %}
15868 ins_pipe(pipe_cmp_branch);
15869 %}
15870
15871 // Test bit and Branch
15872
15873 // Patterns for short (< 32KiB) variants
15874 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15875 match(If cmp (CmpL op1 op2));
15876 effect(USE labl);
15877
15878 ins_cost(BRANCH_COST);
15879 format %{ "cb$cmp $op1, $labl # long" %}
15880 ins_encode %{
15881 Label* L = $labl$$label;
15882 Assembler::Condition cond =
15883 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15884 __ tbr(cond, $op1$$Register, 63, *L);
15885 %}
15886 ins_pipe(pipe_cmp_branch);
15887 ins_short_branch(1);
15888 %}
15889
15890 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15891 match(If cmp (CmpI op1 op2));
15892 effect(USE labl);
15893
15894 ins_cost(BRANCH_COST);
15895 format %{ "cb$cmp $op1, $labl # int" %}
15896 ins_encode %{
15897 Label* L = $labl$$label;
15898 Assembler::Condition cond =
15899 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15900 __ tbr(cond, $op1$$Register, 31, *L);
15901 %}
15902 ins_pipe(pipe_cmp_branch);
15903 ins_short_branch(1);
15904 %}
15905
15906 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15907 match(If cmp (CmpL (AndL op1 op2) op3));
15908 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15909 effect(USE labl);
15910
15911 ins_cost(BRANCH_COST);
15912 format %{ "tb$cmp $op1, $op2, $labl" %}
15913 ins_encode %{
15914 Label* L = $labl$$label;
15915 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15916 int bit = exact_log2_long($op2$$constant);
15917 __ tbr(cond, $op1$$Register, bit, *L);
15918 %}
15919 ins_pipe(pipe_cmp_branch);
15920 ins_short_branch(1);
15921 %}
15922
15923 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15924 match(If cmp (CmpI (AndI op1 op2) op3));
15925 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15926 effect(USE labl);
15927
15928 ins_cost(BRANCH_COST);
15929 format %{ "tb$cmp $op1, $op2, $labl" %}
15930 ins_encode %{
15931 Label* L = $labl$$label;
15932 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15933 int bit = exact_log2((juint)$op2$$constant);
15934 __ tbr(cond, $op1$$Register, bit, *L);
15935 %}
15936 ins_pipe(pipe_cmp_branch);
15937 ins_short_branch(1);
15938 %}
15939
15940 // And far variants
15941 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15942 match(If cmp (CmpL op1 op2));
15943 effect(USE labl);
15944
15945 ins_cost(BRANCH_COST);
15946 format %{ "cb$cmp $op1, $labl # long" %}
15947 ins_encode %{
15948 Label* L = $labl$$label;
15949 Assembler::Condition cond =
15950 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15951 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15952 %}
15953 ins_pipe(pipe_cmp_branch);
15954 %}
15955
15956 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15957 match(If cmp (CmpI op1 op2));
15958 effect(USE labl);
15959
15960 ins_cost(BRANCH_COST);
15961 format %{ "cb$cmp $op1, $labl # int" %}
15962 ins_encode %{
15963 Label* L = $labl$$label;
15964 Assembler::Condition cond =
15965 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15966 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15967 %}
15968 ins_pipe(pipe_cmp_branch);
15969 %}
15970
15971 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15972 match(If cmp (CmpL (AndL op1 op2) op3));
15973 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15974 effect(USE labl);
15975
15976 ins_cost(BRANCH_COST);
15977 format %{ "tb$cmp $op1, $op2, $labl" %}
15978 ins_encode %{
15979 Label* L = $labl$$label;
15980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15981 int bit = exact_log2_long($op2$$constant);
15982 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15983 %}
15984 ins_pipe(pipe_cmp_branch);
15985 %}
15986
15987 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15988 match(If cmp (CmpI (AndI op1 op2) op3));
15989 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15990 effect(USE labl);
15991
15992 ins_cost(BRANCH_COST);
15993 format %{ "tb$cmp $op1, $op2, $labl" %}
15994 ins_encode %{
15995 Label* L = $labl$$label;
15996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15997 int bit = exact_log2((juint)$op2$$constant);
15998 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15999 %}
16000 ins_pipe(pipe_cmp_branch);
16001 %}
16002
16003 // Test bits
16004
16005 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16006 match(Set cr (CmpL (AndL op1 op2) op3));
16007 predicate(Assembler::operand_valid_for_logical_immediate
16008 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16009
16010 ins_cost(INSN_COST);
16011 format %{ "tst $op1, $op2 # long" %}
16012 ins_encode %{
16013 __ tst($op1$$Register, $op2$$constant);
16014 %}
16015 ins_pipe(ialu_reg_reg);
16016 %}
16017
16018 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16019 match(Set cr (CmpI (AndI op1 op2) op3));
16020 predicate(Assembler::operand_valid_for_logical_immediate
16021 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16022
16023 ins_cost(INSN_COST);
16024 format %{ "tst $op1, $op2 # int" %}
16025 ins_encode %{
16026 __ tstw($op1$$Register, $op2$$constant);
16027 %}
16028 ins_pipe(ialu_reg_reg);
16029 %}
16030
16031 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16032 match(Set cr (CmpL (AndL op1 op2) op3));
16033
16034 ins_cost(INSN_COST);
16035 format %{ "tst $op1, $op2 # long" %}
16036 ins_encode %{
16037 __ tst($op1$$Register, $op2$$Register);
16038 %}
16039 ins_pipe(ialu_reg_reg);
16040 %}
16041
16042 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16043 match(Set cr (CmpI (AndI op1 op2) op3));
16044
16045 ins_cost(INSN_COST);
16046 format %{ "tstw $op1, $op2 # int" %}
16047 ins_encode %{
16048 __ tstw($op1$$Register, $op2$$Register);
16049 %}
16050 ins_pipe(ialu_reg_reg);
16051 %}
16052
16053
16054 // Conditional Far Branch
16055 // Conditional Far Branch Unsigned
16056 // TODO: fixme
16057
16058 // counted loop end branch near
16059 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16060 %{
16061 match(CountedLoopEnd cmp cr);
16062
16063 effect(USE lbl);
16064
16065 ins_cost(BRANCH_COST);
16066 // short variant.
16067 // ins_short_branch(1);
16068 format %{ "b$cmp $lbl \t// counted loop end" %}
16069
16070 ins_encode(aarch64_enc_br_con(cmp, lbl));
16071
16072 ins_pipe(pipe_branch);
16073 %}
16074
16075 // counted loop end branch far
16076 // TODO: fixme
16077
16078 // ============================================================================
16079 // inlined locking and unlocking
16080
16081 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16082 %{
16083 predicate(LockingMode != LM_LIGHTWEIGHT);
16084 match(Set cr (FastLock object box));
16085 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16086
16087 ins_cost(5 * INSN_COST);
16088 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16089
16090 ins_encode %{
16091 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16092 %}
16093
16094 ins_pipe(pipe_serial);
16095 %}
16096
16097 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16098 %{
16099 predicate(LockingMode != LM_LIGHTWEIGHT);
16100 match(Set cr (FastUnlock object box));
16101 effect(TEMP tmp, TEMP tmp2);
16102
16103 ins_cost(5 * INSN_COST);
16104 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16105
16106 ins_encode %{
16107 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16108 %}
16109
16110 ins_pipe(pipe_serial);
16111 %}
16112
16113 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16114 %{
16115 predicate(LockingMode == LM_LIGHTWEIGHT);
16116 match(Set cr (FastLock object box));
16117 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16118
16119 ins_cost(5 * INSN_COST);
16120 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16121
16122 ins_encode %{
16123 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16124 %}
16125
16126 ins_pipe(pipe_serial);
16127 %}
16128
16129 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16130 %{
16131 predicate(LockingMode == LM_LIGHTWEIGHT);
16132 match(Set cr (FastUnlock object box));
16133 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16134
16135 ins_cost(5 * INSN_COST);
16136 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16137
16138 ins_encode %{
16139 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16140 %}
16141
16142 ins_pipe(pipe_serial);
16143 %}
16144
16145 // ============================================================================
16146 // Safepoint Instructions
16147
16148 // TODO
16149 // provide a near and far version of this code
16150
16151 instruct safePoint(rFlagsReg cr, iRegP poll)
16152 %{
16153 match(SafePoint poll);
16154 effect(KILL cr);
16155
16156 format %{
16157 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16158 %}
16159 ins_encode %{
16160 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16161 %}
16162 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16163 %}
16164
16165
16166 // ============================================================================
16167 // Procedure Call/Return Instructions
16168
16169 // Call Java Static Instruction
16170
16171 instruct CallStaticJavaDirect(method meth)
16172 %{
16173 match(CallStaticJava);
16174
16175 effect(USE meth);
16176
16177 ins_cost(CALL_COST);
16178
16179 format %{ "call,static $meth \t// ==> " %}
16180
16181 ins_encode(aarch64_enc_java_static_call(meth),
16182 aarch64_enc_call_epilog);
16183
16184 ins_pipe(pipe_class_call);
16185 %}
16186
16187 // TO HERE
16188
16189 // Call Java Dynamic Instruction
16190 instruct CallDynamicJavaDirect(method meth)
16191 %{
16192 match(CallDynamicJava);
16193
16194 effect(USE meth);
16195
16196 ins_cost(CALL_COST);
16197
16198 format %{ "CALL,dynamic $meth \t// ==> " %}
16199
16200 ins_encode(aarch64_enc_java_dynamic_call(meth),
16201 aarch64_enc_call_epilog);
16202
16203 ins_pipe(pipe_class_call);
16204 %}
16205
16206 // Call Runtime Instruction
16207
16208 instruct CallRuntimeDirect(method meth)
16209 %{
16210 match(CallRuntime);
16211
16212 effect(USE meth);
16213
16214 ins_cost(CALL_COST);
16215
16216 format %{ "CALL, runtime $meth" %}
16217
16218 ins_encode( aarch64_enc_java_to_runtime(meth) );
16219
16220 ins_pipe(pipe_class_call);
16221 %}
16222
16223 // Call Runtime Instruction
16224
16225 instruct CallLeafDirect(method meth)
16226 %{
16227 match(CallLeaf);
16228
16229 effect(USE meth);
16230
16231 ins_cost(CALL_COST);
16232
16233 format %{ "CALL, runtime leaf $meth" %}
16234
16235 ins_encode( aarch64_enc_java_to_runtime(meth) );
16236
16237 ins_pipe(pipe_class_call);
16238 %}
16239
16240 // Call Runtime Instruction without safepoint and with vector arguments
16241 instruct CallLeafDirectVector(method meth)
16242 %{
16243 match(CallLeafVector);
16244
16245 effect(USE meth);
16246
16247 ins_cost(CALL_COST);
16248
16249 format %{ "CALL, runtime leaf vector $meth" %}
16250
16251 ins_encode(aarch64_enc_java_to_runtime(meth));
16252
16253 ins_pipe(pipe_class_call);
16254 %}
16255
16256 // Call Runtime Instruction
16257
16258 // entry point is null, target holds the address to call
16259 instruct CallLeafNoFPIndirect(iRegP target)
16260 %{
16261 predicate(n->as_Call()->entry_point() == nullptr);
16262
16263 match(CallLeafNoFP target);
16264
16265 ins_cost(CALL_COST);
16266
16267 format %{ "CALL, runtime leaf nofp indirect $target" %}
16268
16269 ins_encode %{
16270 __ blr($target$$Register);
16271 %}
16272
16273 ins_pipe(pipe_class_call);
16274 %}
16275
16276 instruct CallLeafNoFPDirect(method meth)
16277 %{
16278 predicate(n->as_Call()->entry_point() != nullptr);
16279
16280 match(CallLeafNoFP);
16281
16282 effect(USE meth);
16283
16284 ins_cost(CALL_COST);
16285
16286 format %{ "CALL, runtime leaf nofp $meth" %}
16287
16288 ins_encode( aarch64_enc_java_to_runtime(meth) );
16289
16290 ins_pipe(pipe_class_call);
16291 %}
16292
16293 // Tail Call; Jump from runtime stub to Java code.
16294 // Also known as an 'interprocedural jump'.
16295 // Target of jump will eventually return to caller.
16296 // TailJump below removes the return address.
16297 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16298 // emitted just above the TailCall which has reset rfp to the caller state.
16299 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16300 %{
16301 match(TailCall jump_target method_ptr);
16302
16303 ins_cost(CALL_COST);
16304
16305 format %{ "br $jump_target\t# $method_ptr holds method" %}
16306
16307 ins_encode(aarch64_enc_tail_call(jump_target));
16308
16309 ins_pipe(pipe_class_call);
16310 %}
16311
16312 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16313 %{
16314 match(TailJump jump_target ex_oop);
16315
16316 ins_cost(CALL_COST);
16317
16318 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16319
16320 ins_encode(aarch64_enc_tail_jmp(jump_target));
16321
16322 ins_pipe(pipe_class_call);
16323 %}
16324
16325 // Forward exception.
16326 instruct ForwardExceptionjmp()
16327 %{
16328 match(ForwardException);
16329 ins_cost(CALL_COST);
16330
16331 format %{ "b forward_exception_stub" %}
16332 ins_encode %{
16333 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16334 %}
16335 ins_pipe(pipe_class_call);
16336 %}
16337
16338 // Create exception oop: created by stack-crawling runtime code.
16339 // Created exception is now available to this handler, and is setup
16340 // just prior to jumping to this handler. No code emitted.
16341 // TODO check
16342 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16343 instruct CreateException(iRegP_R0 ex_oop)
16344 %{
16345 match(Set ex_oop (CreateEx));
16346
16347 format %{ " -- \t// exception oop; no code emitted" %}
16348
16349 size(0);
16350
16351 ins_encode( /*empty*/ );
16352
16353 ins_pipe(pipe_class_empty);
16354 %}
16355
16356 // Rethrow exception: The exception oop will come in the first
16357 // argument position. Then JUMP (not call) to the rethrow stub code.
16358 instruct RethrowException() %{
16359 match(Rethrow);
16360 ins_cost(CALL_COST);
16361
16362 format %{ "b rethrow_stub" %}
16363
16364 ins_encode( aarch64_enc_rethrow() );
16365
16366 ins_pipe(pipe_class_call);
16367 %}
16368
16369
16370 // Return Instruction
16371 // epilog node loads ret address into lr as part of frame pop
16372 instruct Ret()
16373 %{
16374 match(Return);
16375
16376 format %{ "ret\t// return register" %}
16377
16378 ins_encode( aarch64_enc_ret() );
16379
16380 ins_pipe(pipe_branch);
16381 %}
16382
16383 // Die now.
16384 instruct ShouldNotReachHere() %{
16385 match(Halt);
16386
16387 ins_cost(CALL_COST);
16388 format %{ "ShouldNotReachHere" %}
16389
16390 ins_encode %{
16391 if (is_reachable()) {
16392 const char* str = __ code_string(_halt_reason);
16393 __ stop(str);
16394 }
16395 %}
16396
16397 ins_pipe(pipe_class_default);
16398 %}
16399
16400 // ============================================================================
16401 // Partial Subtype Check
16402 //
16403 // superklass array for an instance of the superklass. Set a hidden
16404 // internal cache on a hit (cache is checked with exposed code in
16405 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16406 // encoding ALSO sets flags.
16407
16408 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16409 %{
16410 match(Set result (PartialSubtypeCheck sub super));
16411 predicate(!UseSecondarySupersTable);
16412 effect(KILL cr, KILL temp);
16413
16414 ins_cost(20 * INSN_COST); // slightly larger than the next version
16415 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16416
16417 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16418
16419 opcode(0x1); // Force zero of result reg on hit
16420
16421 ins_pipe(pipe_class_memory);
16422 %}
16423
16424 // Two versions of partialSubtypeCheck, both used when we need to
16425 // search for a super class in the secondary supers array. The first
16426 // is used when we don't know _a priori_ the class being searched
16427 // for. The second, far more common, is used when we do know: this is
16428 // used for instanceof, checkcast, and any case where C2 can determine
16429 // it by constant propagation.
16430
16431 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16432 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16433 rFlagsReg cr)
16434 %{
16435 match(Set result (PartialSubtypeCheck sub super));
16436 predicate(UseSecondarySupersTable);
16437 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16438
16439 ins_cost(10 * INSN_COST); // slightly larger than the next version
16440 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16441
16442 ins_encode %{
16443 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16444 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16445 $vtemp$$FloatRegister,
16446 $result$$Register, /*L_success*/nullptr);
16447 %}
16448
16449 ins_pipe(pipe_class_memory);
16450 %}
16451
16452 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16453 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16454 rFlagsReg cr)
16455 %{
16456 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16457 predicate(UseSecondarySupersTable);
16458 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16459
16460 ins_cost(5 * INSN_COST); // smaller than the next version
16461 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16462
16463 ins_encode %{
16464 bool success = false;
16465 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16466 if (InlineSecondarySupersTest) {
16467 success =
16468 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16469 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16470 $vtemp$$FloatRegister,
16471 $result$$Register,
16472 super_klass_slot);
16473 } else {
16474 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16475 success = (call != nullptr);
16476 }
16477 if (!success) {
16478 ciEnv::current()->record_failure("CodeCache is full");
16479 return;
16480 }
16481 %}
16482
16483 ins_pipe(pipe_class_memory);
16484 %}
16485
16486 // Intrisics for String.compareTo()
16487
16488 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16489 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16490 %{
16491 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16492 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16493 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16494
16495 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16496 ins_encode %{
16497 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16498 __ string_compare($str1$$Register, $str2$$Register,
16499 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16500 $tmp1$$Register, $tmp2$$Register,
16501 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16502 %}
16503 ins_pipe(pipe_class_memory);
16504 %}
16505
16506 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16507 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16508 %{
16509 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16510 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16511 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16512
16513 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16514 ins_encode %{
16515 __ string_compare($str1$$Register, $str2$$Register,
16516 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16517 $tmp1$$Register, $tmp2$$Register,
16518 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16519 %}
16520 ins_pipe(pipe_class_memory);
16521 %}
16522
16523 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16524 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16525 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16526 %{
16527 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16528 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16529 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16530 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16531
16532 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16533 ins_encode %{
16534 __ string_compare($str1$$Register, $str2$$Register,
16535 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16536 $tmp1$$Register, $tmp2$$Register,
16537 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16538 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16539 %}
16540 ins_pipe(pipe_class_memory);
16541 %}
16542
16543 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16544 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16545 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16546 %{
16547 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16548 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16549 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16550 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16551
16552 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16553 ins_encode %{
16554 __ string_compare($str1$$Register, $str2$$Register,
16555 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16556 $tmp1$$Register, $tmp2$$Register,
16557 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16558 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16559 %}
16560 ins_pipe(pipe_class_memory);
16561 %}
16562
16563 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16564 // these string_compare variants as NEON register type for convenience so that the prototype of
16565 // string_compare can be shared with all variants.
16566
16567 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16568 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16569 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16570 pRegGov_P1 pgtmp2, rFlagsReg cr)
16571 %{
16572 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16573 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16574 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16575 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16576
16577 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16578 ins_encode %{
16579 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16580 __ string_compare($str1$$Register, $str2$$Register,
16581 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16582 $tmp1$$Register, $tmp2$$Register,
16583 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16584 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16585 StrIntrinsicNode::LL);
16586 %}
16587 ins_pipe(pipe_class_memory);
16588 %}
16589
16590 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16591 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16592 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16593 pRegGov_P1 pgtmp2, rFlagsReg cr)
16594 %{
16595 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16596 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16597 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16598 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16599
16600 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16601 ins_encode %{
16602 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16603 __ string_compare($str1$$Register, $str2$$Register,
16604 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16605 $tmp1$$Register, $tmp2$$Register,
16606 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16607 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16608 StrIntrinsicNode::LU);
16609 %}
16610 ins_pipe(pipe_class_memory);
16611 %}
16612
16613 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16614 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16615 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16616 pRegGov_P1 pgtmp2, rFlagsReg cr)
16617 %{
16618 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16619 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16620 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16621 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16622
16623 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16624 ins_encode %{
16625 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16626 __ string_compare($str1$$Register, $str2$$Register,
16627 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16628 $tmp1$$Register, $tmp2$$Register,
16629 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16630 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16631 StrIntrinsicNode::UL);
16632 %}
16633 ins_pipe(pipe_class_memory);
16634 %}
16635
16636 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16637 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16638 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16639 pRegGov_P1 pgtmp2, rFlagsReg cr)
16640 %{
16641 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16642 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16643 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16644 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16645
16646 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16647 ins_encode %{
16648 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16649 __ string_compare($str1$$Register, $str2$$Register,
16650 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16651 $tmp1$$Register, $tmp2$$Register,
16652 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16653 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16654 StrIntrinsicNode::UU);
16655 %}
16656 ins_pipe(pipe_class_memory);
16657 %}
16658
16659 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16660 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16661 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16662 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16663 %{
16664 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16665 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16666 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16667 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16668 TEMP vtmp0, TEMP vtmp1, KILL cr);
16669 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16670 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16671
16672 ins_encode %{
16673 __ string_indexof($str1$$Register, $str2$$Register,
16674 $cnt1$$Register, $cnt2$$Register,
16675 $tmp1$$Register, $tmp2$$Register,
16676 $tmp3$$Register, $tmp4$$Register,
16677 $tmp5$$Register, $tmp6$$Register,
16678 -1, $result$$Register, StrIntrinsicNode::UU);
16679 %}
16680 ins_pipe(pipe_class_memory);
16681 %}
16682
16683 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16684 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16685 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16686 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16687 %{
16688 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16689 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16690 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16691 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16692 TEMP vtmp0, TEMP vtmp1, KILL cr);
16693 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16694 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16695
16696 ins_encode %{
16697 __ string_indexof($str1$$Register, $str2$$Register,
16698 $cnt1$$Register, $cnt2$$Register,
16699 $tmp1$$Register, $tmp2$$Register,
16700 $tmp3$$Register, $tmp4$$Register,
16701 $tmp5$$Register, $tmp6$$Register,
16702 -1, $result$$Register, StrIntrinsicNode::LL);
16703 %}
16704 ins_pipe(pipe_class_memory);
16705 %}
16706
16707 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16708 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16709 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16710 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16711 %{
16712 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16713 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16714 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16715 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16716 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16717 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16718 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16719
16720 ins_encode %{
16721 __ string_indexof($str1$$Register, $str2$$Register,
16722 $cnt1$$Register, $cnt2$$Register,
16723 $tmp1$$Register, $tmp2$$Register,
16724 $tmp3$$Register, $tmp4$$Register,
16725 $tmp5$$Register, $tmp6$$Register,
16726 -1, $result$$Register, StrIntrinsicNode::UL);
16727 %}
16728 ins_pipe(pipe_class_memory);
16729 %}
16730
16731 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16732 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16733 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16734 %{
16735 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16736 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16737 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16738 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16739 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16740 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16741
16742 ins_encode %{
16743 int icnt2 = (int)$int_cnt2$$constant;
16744 __ string_indexof($str1$$Register, $str2$$Register,
16745 $cnt1$$Register, zr,
16746 $tmp1$$Register, $tmp2$$Register,
16747 $tmp3$$Register, $tmp4$$Register, zr, zr,
16748 icnt2, $result$$Register, StrIntrinsicNode::UU);
16749 %}
16750 ins_pipe(pipe_class_memory);
16751 %}
16752
16753 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16754 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16755 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16756 %{
16757 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16758 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16759 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16760 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16761 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16762 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16763
16764 ins_encode %{
16765 int icnt2 = (int)$int_cnt2$$constant;
16766 __ string_indexof($str1$$Register, $str2$$Register,
16767 $cnt1$$Register, zr,
16768 $tmp1$$Register, $tmp2$$Register,
16769 $tmp3$$Register, $tmp4$$Register, zr, zr,
16770 icnt2, $result$$Register, StrIntrinsicNode::LL);
16771 %}
16772 ins_pipe(pipe_class_memory);
16773 %}
16774
16775 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16776 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16777 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16778 %{
16779 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16780 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16781 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16782 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16783 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16784 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16785
16786 ins_encode %{
16787 int icnt2 = (int)$int_cnt2$$constant;
16788 __ string_indexof($str1$$Register, $str2$$Register,
16789 $cnt1$$Register, zr,
16790 $tmp1$$Register, $tmp2$$Register,
16791 $tmp3$$Register, $tmp4$$Register, zr, zr,
16792 icnt2, $result$$Register, StrIntrinsicNode::UL);
16793 %}
16794 ins_pipe(pipe_class_memory);
16795 %}
16796
16797 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16798 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16799 iRegINoSp tmp3, rFlagsReg cr)
16800 %{
16801 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16802 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16803 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16804 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16805
16806 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16807
16808 ins_encode %{
16809 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16810 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16811 $tmp3$$Register);
16812 %}
16813 ins_pipe(pipe_class_memory);
16814 %}
16815
16816 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16817 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16818 iRegINoSp tmp3, rFlagsReg cr)
16819 %{
16820 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16821 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16822 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16823 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16824
16825 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16826
16827 ins_encode %{
16828 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16829 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16830 $tmp3$$Register);
16831 %}
16832 ins_pipe(pipe_class_memory);
16833 %}
16834
16835 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16836 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16837 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16838 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16839 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16840 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16841 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16842 ins_encode %{
16843 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16844 $result$$Register, $ztmp1$$FloatRegister,
16845 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16846 $ptmp$$PRegister, true /* isL */);
16847 %}
16848 ins_pipe(pipe_class_memory);
16849 %}
16850
16851 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16852 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16853 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16854 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16855 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16856 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16857 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16858 ins_encode %{
16859 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16860 $result$$Register, $ztmp1$$FloatRegister,
16861 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16862 $ptmp$$PRegister, false /* isL */);
16863 %}
16864 ins_pipe(pipe_class_memory);
16865 %}
16866
16867 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16868 iRegI_R0 result, rFlagsReg cr)
16869 %{
16870 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16871 match(Set result (StrEquals (Binary str1 str2) cnt));
16872 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16873
16874 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16875 ins_encode %{
16876 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16877 __ string_equals($str1$$Register, $str2$$Register,
16878 $result$$Register, $cnt$$Register);
16879 %}
16880 ins_pipe(pipe_class_memory);
16881 %}
16882
16883 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16884 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16885 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16886 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16887 iRegP_R10 tmp, rFlagsReg cr)
16888 %{
16889 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16890 match(Set result (AryEq ary1 ary2));
16891 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16892 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16893 TEMP vtmp6, TEMP vtmp7, KILL cr);
16894
16895 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16896 ins_encode %{
16897 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16898 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16899 $result$$Register, $tmp$$Register, 1);
16900 if (tpc == nullptr) {
16901 ciEnv::current()->record_failure("CodeCache is full");
16902 return;
16903 }
16904 %}
16905 ins_pipe(pipe_class_memory);
16906 %}
16907
16908 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16909 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16910 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16911 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16912 iRegP_R10 tmp, rFlagsReg cr)
16913 %{
16914 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16915 match(Set result (AryEq ary1 ary2));
16916 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16917 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16918 TEMP vtmp6, TEMP vtmp7, KILL cr);
16919
16920 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16921 ins_encode %{
16922 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16923 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16924 $result$$Register, $tmp$$Register, 2);
16925 if (tpc == nullptr) {
16926 ciEnv::current()->record_failure("CodeCache is full");
16927 return;
16928 }
16929 %}
16930 ins_pipe(pipe_class_memory);
16931 %}
16932
16933 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16934 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16935 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16936 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16937 %{
16938 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16939 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16940 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16941
16942 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16943 ins_encode %{
16944 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16945 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16946 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16947 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16948 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16949 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16950 (BasicType)$basic_type$$constant);
16951 if (tpc == nullptr) {
16952 ciEnv::current()->record_failure("CodeCache is full");
16953 return;
16954 }
16955 %}
16956 ins_pipe(pipe_class_memory);
16957 %}
16958
16959 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16960 %{
16961 match(Set result (CountPositives ary1 len));
16962 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16963 format %{ "count positives byte[] $ary1,$len -> $result" %}
16964 ins_encode %{
16965 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16966 if (tpc == nullptr) {
16967 ciEnv::current()->record_failure("CodeCache is full");
16968 return;
16969 }
16970 %}
16971 ins_pipe( pipe_slow );
16972 %}
16973
16974 // fast char[] to byte[] compression
16975 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16976 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16977 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16978 iRegI_R0 result, rFlagsReg cr)
16979 %{
16980 match(Set result (StrCompressedCopy src (Binary dst len)));
16981 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16982 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16983
16984 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16985 ins_encode %{
16986 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16987 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16988 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16989 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16990 %}
16991 ins_pipe(pipe_slow);
16992 %}
16993
16994 // fast byte[] to char[] inflation
16995 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16996 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16997 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16998 %{
16999 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17000 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17001 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17002 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17003
17004 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17005 ins_encode %{
17006 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17007 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17008 $vtmp2$$FloatRegister, $tmp$$Register);
17009 if (tpc == nullptr) {
17010 ciEnv::current()->record_failure("CodeCache is full");
17011 return;
17012 }
17013 %}
17014 ins_pipe(pipe_class_memory);
17015 %}
17016
17017 // encode char[] to byte[] in ISO_8859_1
17018 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17019 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17020 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17021 iRegI_R0 result, rFlagsReg cr)
17022 %{
17023 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17024 match(Set result (EncodeISOArray src (Binary dst len)));
17025 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17026 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17027
17028 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17029 ins_encode %{
17030 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17031 $result$$Register, false,
17032 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17033 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17034 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17035 %}
17036 ins_pipe(pipe_class_memory);
17037 %}
17038
17039 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17040 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17041 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17042 iRegI_R0 result, rFlagsReg cr)
17043 %{
17044 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17045 match(Set result (EncodeISOArray src (Binary dst len)));
17046 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17047 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17048
17049 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17050 ins_encode %{
17051 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17052 $result$$Register, true,
17053 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17054 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17055 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17056 %}
17057 ins_pipe(pipe_class_memory);
17058 %}
17059
17060 //----------------------------- CompressBits/ExpandBits ------------------------
17061
17062 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17063 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17064 match(Set dst (CompressBits src mask));
17065 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17066 format %{ "mov $tsrc, $src\n\t"
17067 "mov $tmask, $mask\n\t"
17068 "bext $tdst, $tsrc, $tmask\n\t"
17069 "mov $dst, $tdst"
17070 %}
17071 ins_encode %{
17072 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17073 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17074 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17075 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17076 %}
17077 ins_pipe(pipe_slow);
17078 %}
17079
17080 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17081 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17082 match(Set dst (CompressBits (LoadI mem) mask));
17083 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17084 format %{ "ldrs $tsrc, $mem\n\t"
17085 "ldrs $tmask, $mask\n\t"
17086 "bext $tdst, $tsrc, $tmask\n\t"
17087 "mov $dst, $tdst"
17088 %}
17089 ins_encode %{
17090 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17091 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17092 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17093 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17094 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17095 %}
17096 ins_pipe(pipe_slow);
17097 %}
17098
17099 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17100 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17101 match(Set dst (CompressBits src mask));
17102 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17103 format %{ "mov $tsrc, $src\n\t"
17104 "mov $tmask, $mask\n\t"
17105 "bext $tdst, $tsrc, $tmask\n\t"
17106 "mov $dst, $tdst"
17107 %}
17108 ins_encode %{
17109 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17110 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17111 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17112 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17113 %}
17114 ins_pipe(pipe_slow);
17115 %}
17116
17117 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17118 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17119 match(Set dst (CompressBits (LoadL mem) mask));
17120 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17121 format %{ "ldrd $tsrc, $mem\n\t"
17122 "ldrd $tmask, $mask\n\t"
17123 "bext $tdst, $tsrc, $tmask\n\t"
17124 "mov $dst, $tdst"
17125 %}
17126 ins_encode %{
17127 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17128 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17129 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17130 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17131 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17132 %}
17133 ins_pipe(pipe_slow);
17134 %}
17135
17136 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17137 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17138 match(Set dst (ExpandBits src mask));
17139 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17140 format %{ "mov $tsrc, $src\n\t"
17141 "mov $tmask, $mask\n\t"
17142 "bdep $tdst, $tsrc, $tmask\n\t"
17143 "mov $dst, $tdst"
17144 %}
17145 ins_encode %{
17146 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17147 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17148 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17149 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17150 %}
17151 ins_pipe(pipe_slow);
17152 %}
17153
17154 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17155 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17156 match(Set dst (ExpandBits (LoadI mem) mask));
17157 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17158 format %{ "ldrs $tsrc, $mem\n\t"
17159 "ldrs $tmask, $mask\n\t"
17160 "bdep $tdst, $tsrc, $tmask\n\t"
17161 "mov $dst, $tdst"
17162 %}
17163 ins_encode %{
17164 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17165 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17166 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17167 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17168 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17169 %}
17170 ins_pipe(pipe_slow);
17171 %}
17172
17173 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17174 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17175 match(Set dst (ExpandBits src mask));
17176 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17177 format %{ "mov $tsrc, $src\n\t"
17178 "mov $tmask, $mask\n\t"
17179 "bdep $tdst, $tsrc, $tmask\n\t"
17180 "mov $dst, $tdst"
17181 %}
17182 ins_encode %{
17183 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17184 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17185 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17186 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17187 %}
17188 ins_pipe(pipe_slow);
17189 %}
17190
17191
17192 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17193 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17194 match(Set dst (ExpandBits (LoadL mem) mask));
17195 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17196 format %{ "ldrd $tsrc, $mem\n\t"
17197 "ldrd $tmask, $mask\n\t"
17198 "bdep $tdst, $tsrc, $tmask\n\t"
17199 "mov $dst, $tdst"
17200 %}
17201 ins_encode %{
17202 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17203 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17204 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17205 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17206 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17207 %}
17208 ins_pipe(pipe_slow);
17209 %}
17210
17211 // ============================================================================
17212 // This name is KNOWN by the ADLC and cannot be changed.
17213 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17214 // for this guy.
17215 instruct tlsLoadP(thread_RegP dst)
17216 %{
17217 match(Set dst (ThreadLocal));
17218
17219 ins_cost(0);
17220
17221 format %{ " -- \t// $dst=Thread::current(), empty" %}
17222
17223 size(0);
17224
17225 ins_encode( /*empty*/ );
17226
17227 ins_pipe(pipe_class_empty);
17228 %}
17229
17230 //----------PEEPHOLE RULES-----------------------------------------------------
17231 // These must follow all instruction definitions as they use the names
17232 // defined in the instructions definitions.
17233 //
17234 // peepmatch ( root_instr_name [preceding_instruction]* );
17235 //
17236 // peepconstraint %{
17237 // (instruction_number.operand_name relational_op instruction_number.operand_name
17238 // [, ...] );
17239 // // instruction numbers are zero-based using left to right order in peepmatch
17240 //
17241 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17242 // // provide an instruction_number.operand_name for each operand that appears
17243 // // in the replacement instruction's match rule
17244 //
17245 // ---------VM FLAGS---------------------------------------------------------
17246 //
17247 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17248 //
17249 // Each peephole rule is given an identifying number starting with zero and
17250 // increasing by one in the order seen by the parser. An individual peephole
17251 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17252 // on the command-line.
17253 //
17254 // ---------CURRENT LIMITATIONS----------------------------------------------
17255 //
17256 // Only match adjacent instructions in same basic block
17257 // Only equality constraints
17258 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17259 // Only one replacement instruction
17260 //
17261 // ---------EXAMPLE----------------------------------------------------------
17262 //
17263 // // pertinent parts of existing instructions in architecture description
17264 // instruct movI(iRegINoSp dst, iRegI src)
17265 // %{
17266 // match(Set dst (CopyI src));
17267 // %}
17268 //
17269 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17270 // %{
17271 // match(Set dst (AddI dst src));
17272 // effect(KILL cr);
17273 // %}
17274 //
17275 // // Change (inc mov) to lea
17276 // peephole %{
17277 // // increment preceded by register-register move
17278 // peepmatch ( incI_iReg movI );
17279 // // require that the destination register of the increment
17280 // // match the destination register of the move
17281 // peepconstraint ( 0.dst == 1.dst );
17282 // // construct a replacement instruction that sets
17283 // // the destination to ( move's source register + one )
17284 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17285 // %}
17286 //
17287
17288 // Implementation no longer uses movX instructions since
17289 // machine-independent system no longer uses CopyX nodes.
17290 //
17291 // peephole
17292 // %{
17293 // peepmatch (incI_iReg movI);
17294 // peepconstraint (0.dst == 1.dst);
17295 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17296 // %}
17297
17298 // peephole
17299 // %{
17300 // peepmatch (decI_iReg movI);
17301 // peepconstraint (0.dst == 1.dst);
17302 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17303 // %}
17304
17305 // peephole
17306 // %{
17307 // peepmatch (addI_iReg_imm movI);
17308 // peepconstraint (0.dst == 1.dst);
17309 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17310 // %}
17311
17312 // peephole
17313 // %{
17314 // peepmatch (incL_iReg movL);
17315 // peepconstraint (0.dst == 1.dst);
17316 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17317 // %}
17318
17319 // peephole
17320 // %{
17321 // peepmatch (decL_iReg movL);
17322 // peepconstraint (0.dst == 1.dst);
17323 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17324 // %}
17325
17326 // peephole
17327 // %{
17328 // peepmatch (addL_iReg_imm movL);
17329 // peepconstraint (0.dst == 1.dst);
17330 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17331 // %}
17332
17333 // peephole
17334 // %{
17335 // peepmatch (addP_iReg_imm movP);
17336 // peepconstraint (0.dst == 1.dst);
17337 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17338 // %}
17339
17340 // // Change load of spilled value to only a spill
17341 // instruct storeI(memory mem, iRegI src)
17342 // %{
17343 // match(Set mem (StoreI mem src));
17344 // %}
17345 //
17346 // instruct loadI(iRegINoSp dst, memory mem)
17347 // %{
17348 // match(Set dst (LoadI mem));
17349 // %}
17350 //
17351
17352 //----------SMARTSPILL RULES---------------------------------------------------
17353 // These must follow all instruction definitions as they use the names
17354 // defined in the instructions definitions.
17355
17356 // Local Variables:
17357 // mode: c++
17358 // End: