1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1652 // lea(rscratch1, RuntimeAddress(addr)
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else if (_entry_point == nullptr) {
1658 // See CallLeafNoFPIndirect
1659 return 1 * NativeInstruction::instruction_size;
1660 } else {
1661 return 6 * NativeInstruction::instruction_size;
1662 }
1663 }
1664
1665 //=============================================================================
1666
1667 #ifndef PRODUCT
1668 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1669 st->print("BREAKPOINT");
1670 }
1671 #endif
1672
1673 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1674 __ brk(0);
1675 }
1676
1677 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1678 return MachNode::size(ra_);
1679 }
1680
1681 //=============================================================================
1682
1683 #ifndef PRODUCT
1684 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1685 st->print("nop \t# %d bytes pad for loops and calls", _count);
1686 }
1687 #endif
1688
1689 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1690 for (int i = 0; i < _count; i++) {
1691 __ nop();
1692 }
1693 }
1694
1695 uint MachNopNode::size(PhaseRegAlloc*) const {
1696 return _count * NativeInstruction::instruction_size;
1697 }
1698
1699 //=============================================================================
1700 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1701
1702 int ConstantTable::calculate_table_base_offset() const {
1703 return 0; // absolute addressing, no offset
1704 }
1705
1706 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1707 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1708 ShouldNotReachHere();
1709 }
1710
1711 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1712 // Empty encoding
1713 }
1714
1715 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1716 return 0;
1717 }
1718
1719 #ifndef PRODUCT
1720 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1721 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1722 }
1723 #endif
1724
1725 #ifndef PRODUCT
1726 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1727 Compile* C = ra_->C;
1728
1729 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1730
1731 if (C->output()->need_stack_bang(framesize))
1732 st->print("# stack bang size=%d\n\t", framesize);
1733
1734 if (VM_Version::use_rop_protection()) {
1735 st->print("ldr zr, [lr]\n\t");
1736 st->print("paciaz\n\t");
1737 }
1738 if (framesize < ((1 << 9) + 2 * wordSize)) {
1739 st->print("sub sp, sp, #%d\n\t", framesize);
1740 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1741 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1742 } else {
1743 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1744 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1745 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1746 st->print("sub sp, sp, rscratch1");
1747 }
1748 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1749 st->print("\n\t");
1750 st->print("ldr rscratch1, [guard]\n\t");
1751 st->print("dmb ishld\n\t");
1752 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1753 st->print("cmp rscratch1, rscratch2\n\t");
1754 st->print("b.eq skip");
1755 st->print("\n\t");
1756 st->print("blr #nmethod_entry_barrier_stub\n\t");
1757 st->print("b skip\n\t");
1758 st->print("guard: int\n\t");
1759 st->print("\n\t");
1760 st->print("skip:\n\t");
1761 }
1762 }
1763 #endif
1764
1765 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1766 Compile* C = ra_->C;
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 __ verified_entry(C, 0);
1773
1774 if (C->stub_function() == nullptr) {
1775 __ entry_barrier();
1776 }
1777
1778 if (!Compile::current()->output()->in_scratch_emit_size()) {
1779 __ bind(*_verified_entry);
1780 }
1781
1782 if (VerifyStackAtCalls) {
1783 Unimplemented();
1784 }
1785
1786 C->output()->set_frame_complete(__ offset());
1787
1788 if (C->has_mach_constant_base_node()) {
1789 // NOTE: We set the table base offset here because users might be
1790 // emitted before MachConstantBaseNode.
1791 ConstantTable& constant_table = C->output()->constant_table();
1792 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1793 }
1794 }
1795
1796 int MachPrologNode::reloc() const
1797 {
1798 return 0;
1799 }
1800
1801 //=============================================================================
1802
1803 #ifndef PRODUCT
1804 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1805 Compile* C = ra_->C;
1806 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1807
1808 st->print("# pop frame %d\n\t",framesize);
1809
1810 if (framesize == 0) {
1811 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1812 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1813 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1814 st->print("add sp, sp, #%d\n\t", framesize);
1815 } else {
1816 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1817 st->print("add sp, sp, rscratch1\n\t");
1818 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1819 }
1820 if (VM_Version::use_rop_protection()) {
1821 st->print("autiaz\n\t");
1822 st->print("ldr zr, [lr]\n\t");
1823 }
1824
1825 if (do_polling() && C->is_method_compilation()) {
1826 st->print("# test polling word\n\t");
1827 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1828 st->print("cmp sp, rscratch1\n\t");
1829 st->print("bhi #slow_path");
1830 }
1831 }
1832 #endif
1833
1834 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1835 Compile* C = ra_->C;
1836 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1837
1838 __ remove_frame(framesize, C->needs_stack_repair());
1839
1840 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1841 __ reserved_stack_check();
1842 }
1843
1844 if (do_polling() && C->is_method_compilation()) {
1845 Label dummy_label;
1846 Label* code_stub = &dummy_label;
1847 if (!C->output()->in_scratch_emit_size()) {
1848 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1849 C->output()->add_stub(stub);
1850 code_stub = &stub->entry();
1851 }
1852 __ relocate(relocInfo::poll_return_type);
1853 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1854 }
1855 }
1856
1857 int MachEpilogNode::reloc() const {
1858 // Return number of relocatable values contained in this instruction.
1859 return 1; // 1 for polling page.
1860 }
1861
1862 const Pipeline * MachEpilogNode::pipeline() const {
1863 return MachNode::pipeline_class();
1864 }
1865
1866 //=============================================================================
1867
1868 static enum RC rc_class(OptoReg::Name reg) {
1869
1870 if (reg == OptoReg::Bad) {
1871 return rc_bad;
1872 }
1873
1874 // we have 32 int registers * 2 halves
1875 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1876
1877 if (reg < slots_of_int_registers) {
1878 return rc_int;
1879 }
1880
1881 // we have 32 float register * 8 halves
1882 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1883 if (reg < slots_of_int_registers + slots_of_float_registers) {
1884 return rc_float;
1885 }
1886
1887 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1888 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1889 return rc_predicate;
1890 }
1891
1892 // Between predicate regs & stack is the flags.
1893 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1894
1895 return rc_stack;
1896 }
1897
1898 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1899 Compile* C = ra_->C;
1900
1901 // Get registers to move.
1902 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1903 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1904 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1905 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1906
1907 enum RC src_hi_rc = rc_class(src_hi);
1908 enum RC src_lo_rc = rc_class(src_lo);
1909 enum RC dst_hi_rc = rc_class(dst_hi);
1910 enum RC dst_lo_rc = rc_class(dst_lo);
1911
1912 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1913
1914 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1915 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1916 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1917 "expected aligned-adjacent pairs");
1918 }
1919
1920 if (src_lo == dst_lo && src_hi == dst_hi) {
1921 return 0; // Self copy, no move.
1922 }
1923
1924 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1925 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1926 int src_offset = ra_->reg2offset(src_lo);
1927 int dst_offset = ra_->reg2offset(dst_lo);
1928
1929 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1930 uint ireg = ideal_reg();
1931 if (ireg == Op_VecA && masm) {
1932 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1933 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1934 // stack->stack
1935 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1938 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1939 sve_vector_reg_size_in_bytes);
1940 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1941 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1942 sve_vector_reg_size_in_bytes);
1943 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1944 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1945 as_FloatRegister(Matcher::_regEncode[src_lo]),
1946 as_FloatRegister(Matcher::_regEncode[src_lo]));
1947 } else {
1948 ShouldNotReachHere();
1949 }
1950 } else if (masm) {
1951 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1952 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1953 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1954 // stack->stack
1955 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1956 if (ireg == Op_VecD) {
1957 __ unspill(rscratch1, true, src_offset);
1958 __ spill(rscratch1, true, dst_offset);
1959 } else {
1960 __ spill_copy128(src_offset, dst_offset);
1961 }
1962 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1963 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1964 ireg == Op_VecD ? __ T8B : __ T16B,
1965 as_FloatRegister(Matcher::_regEncode[src_lo]));
1966 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1967 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1968 ireg == Op_VecD ? __ D : __ Q,
1969 ra_->reg2offset(dst_lo));
1970 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1971 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1972 ireg == Op_VecD ? __ D : __ Q,
1973 ra_->reg2offset(src_lo));
1974 } else {
1975 ShouldNotReachHere();
1976 }
1977 }
1978 } else if (masm) {
1979 switch (src_lo_rc) {
1980 case rc_int:
1981 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1982 if (is64) {
1983 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 } else {
1986 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1987 as_Register(Matcher::_regEncode[src_lo]));
1988 }
1989 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1990 if (is64) {
1991 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 } else {
1994 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1995 as_Register(Matcher::_regEncode[src_lo]));
1996 }
1997 } else { // gpr --> stack spill
1998 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1999 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2000 }
2001 break;
2002 case rc_float:
2003 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2004 if (is64) {
2005 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 } else {
2008 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2009 as_FloatRegister(Matcher::_regEncode[src_lo]));
2010 }
2011 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2012 if (is64) {
2013 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 } else {
2016 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]));
2018 }
2019 } else { // fpr --> stack spill
2020 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2021 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2022 is64 ? __ D : __ S, dst_offset);
2023 }
2024 break;
2025 case rc_stack:
2026 if (dst_lo_rc == rc_int) { // stack --> gpr load
2027 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2028 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 is64 ? __ D : __ S, src_offset);
2031 } else if (dst_lo_rc == rc_predicate) {
2032 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2033 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2034 } else { // stack --> stack copy
2035 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2036 if (ideal_reg() == Op_RegVectMask) {
2037 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2038 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2039 } else {
2040 __ unspill(rscratch1, is64, src_offset);
2041 __ spill(rscratch1, is64, dst_offset);
2042 }
2043 }
2044 break;
2045 case rc_predicate:
2046 if (dst_lo_rc == rc_predicate) {
2047 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2048 } else if (dst_lo_rc == rc_stack) {
2049 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2050 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2051 } else {
2052 assert(false, "bad src and dst rc_class combination.");
2053 ShouldNotReachHere();
2054 }
2055 break;
2056 default:
2057 assert(false, "bad rc_class for spill");
2058 ShouldNotReachHere();
2059 }
2060 }
2061
2062 if (st) {
2063 st->print("spill ");
2064 if (src_lo_rc == rc_stack) {
2065 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2066 } else {
2067 st->print("%s -> ", Matcher::regName[src_lo]);
2068 }
2069 if (dst_lo_rc == rc_stack) {
2070 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2071 } else {
2072 st->print("%s", Matcher::regName[dst_lo]);
2073 }
2074 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2075 int vsize = 0;
2076 switch (ideal_reg()) {
2077 case Op_VecD:
2078 vsize = 64;
2079 break;
2080 case Op_VecX:
2081 vsize = 128;
2082 break;
2083 case Op_VecA:
2084 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2085 break;
2086 default:
2087 assert(false, "bad register type for spill");
2088 ShouldNotReachHere();
2089 }
2090 st->print("\t# vector spill size = %d", vsize);
2091 } else if (ideal_reg() == Op_RegVectMask) {
2092 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2093 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2094 st->print("\t# predicate spill size = %d", vsize);
2095 } else {
2096 st->print("\t# spill size = %d", is64 ? 64 : 32);
2097 }
2098 }
2099
2100 return 0;
2101
2102 }
2103
2104 #ifndef PRODUCT
2105 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2106 if (!ra_)
2107 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2108 else
2109 implementation(nullptr, ra_, false, st);
2110 }
2111 #endif
2112
2113 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2114 implementation(masm, ra_, false, nullptr);
2115 }
2116
2117 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2118 return MachNode::size(ra_);
2119 }
2120
2121 //=============================================================================
2122
2123 #ifndef PRODUCT
2124 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2125 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2126 int reg = ra_->get_reg_first(this);
2127 st->print("add %s, rsp, #%d]\t# box lock",
2128 Matcher::regName[reg], offset);
2129 }
2130 #endif
2131
2132 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2133 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2134 int reg = ra_->get_encode(this);
2135
2136 // This add will handle any 24-bit signed offset. 24 bits allows an
2137 // 8 megabyte stack frame.
2138 __ add(as_Register(reg), sp, offset);
2139 }
2140
2141 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2142 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2143 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2144
2145 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2146 return NativeInstruction::instruction_size;
2147 } else {
2148 return 2 * NativeInstruction::instruction_size;
2149 }
2150 }
2151
2152 ///=============================================================================
2153 #ifndef PRODUCT
2154 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2155 {
2156 st->print_cr("# MachVEPNode");
2157 if (!_verified) {
2158 st->print_cr("\t load_class");
2159 } else {
2160 st->print_cr("\t unpack_inline_arg");
2161 }
2162 }
2163 #endif
2164
2165 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2166 {
2167 if (!_verified) {
2168 __ ic_check(1);
2169 } else {
2170 // insert a nop at the start of the prolog so we can patch in a
2171 // branch if we need to invalidate the method later
2172 __ nop();
2173
2174 // TODO 8284443 Avoid creation of temporary frame
2175 if (ra_->C->stub_function() == nullptr) {
2176 __ verified_entry(ra_->C, 0);
2177 __ entry_barrier();
2178 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2179 __ remove_frame(framesize, false);
2180 }
2181 // Unpack inline type args passed as oop and then jump to
2182 // the verified entry point (skipping the unverified entry).
2183 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2184 // Emit code for verified entry and save increment for stack repair on return
2185 __ verified_entry(ra_->C, sp_inc);
2186 if (Compile::current()->output()->in_scratch_emit_size()) {
2187 Label dummy_verified_entry;
2188 __ b(dummy_verified_entry);
2189 } else {
2190 __ b(*_verified_entry);
2191 }
2192 }
2193 }
2194
2195 //=============================================================================
2196 #ifndef PRODUCT
2197 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2198 {
2199 st->print_cr("# MachUEPNode");
2200 if (UseCompressedClassPointers) {
2201 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2202 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2203 st->print_cr("\tcmpw rscratch1, r10");
2204 } else {
2205 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2206 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2207 st->print_cr("\tcmp rscratch1, r10");
2208 }
2209 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2210 }
2211 #endif
2212
2213 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2214 {
2215 __ ic_check(InteriorEntryAlignment);
2216 }
2217
2218 // REQUIRED EMIT CODE
2219
2220 //=============================================================================
2221
2222 // Emit exception handler code.
2223 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2224 {
2225 // mov rscratch1 #exception_blob_entry_point
2226 // br rscratch1
2227 // Note that the code buffer's insts_mark is always relative to insts.
2228 // That's why we must use the macroassembler to generate a handler.
2229 address base = __ start_a_stub(size_exception_handler());
2230 if (base == nullptr) {
2231 ciEnv::current()->record_failure("CodeCache is full");
2232 return 0; // CodeBuffer::expand failed
2233 }
2234 int offset = __ offset();
2235 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2236 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2237 __ end_a_stub();
2238 return offset;
2239 }
2240
2241 // Emit deopt handler code.
2242 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2243 {
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 address base = __ start_a_stub(size_deopt_handler());
2247 if (base == nullptr) {
2248 ciEnv::current()->record_failure("CodeCache is full");
2249 return 0; // CodeBuffer::expand failed
2250 }
2251 int offset = __ offset();
2252
2253 __ adr(lr, __ pc());
2254 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2255
2256 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2257 __ end_a_stub();
2258 return offset;
2259 }
2260
2261 // REQUIRED MATCHER CODE
2262
2263 //=============================================================================
2264
2265 bool Matcher::match_rule_supported(int opcode) {
2266 if (!has_match_rule(opcode))
2267 return false;
2268
2269 switch (opcode) {
2270 case Op_OnSpinWait:
2271 return VM_Version::supports_on_spin_wait();
2272 case Op_CacheWB:
2273 case Op_CacheWBPreSync:
2274 case Op_CacheWBPostSync:
2275 if (!VM_Version::supports_data_cache_line_flush()) {
2276 return false;
2277 }
2278 break;
2279 case Op_ExpandBits:
2280 case Op_CompressBits:
2281 if (!VM_Version::supports_svebitperm()) {
2282 return false;
2283 }
2284 break;
2285 case Op_FmaF:
2286 case Op_FmaD:
2287 case Op_FmaVF:
2288 case Op_FmaVD:
2289 if (!UseFMA) {
2290 return false;
2291 }
2292 break;
2293 }
2294
2295 return true; // Per default match rules are supported.
2296 }
2297
2298 const RegMask* Matcher::predicate_reg_mask(void) {
2299 return &_PR_REG_mask;
2300 }
2301
2302 bool Matcher::supports_vector_calling_convention(void) {
2303 return EnableVectorSupport && UseVectorStubs;
2304 }
2305
2306 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2307 assert(EnableVectorSupport && UseVectorStubs, "sanity");
2308 int lo = V0_num;
2309 int hi = V0_H_num;
2310 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2311 hi = V0_K_num;
2312 }
2313 return OptoRegPair(hi, lo);
2314 }
2315
2316 // Is this branch offset short enough that a short branch can be used?
2317 //
2318 // NOTE: If the platform does not provide any short branch variants, then
2319 // this method should return false for offset 0.
2320 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2321 // The passed offset is relative to address of the branch.
2322
2323 return (-32768 <= offset && offset < 32768);
2324 }
2325
2326 // Vector width in bytes.
2327 int Matcher::vector_width_in_bytes(BasicType bt) {
2328 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2329 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2330 // Minimum 2 values in vector
2331 if (size < 2*type2aelembytes(bt)) size = 0;
2332 // But never < 4
2333 if (size < 4) size = 0;
2334 return size;
2335 }
2336
2337 // Limits on vector size (number of elements) loaded into vector.
2338 int Matcher::max_vector_size(const BasicType bt) {
2339 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2340 }
2341
2342 int Matcher::min_vector_size(const BasicType bt) {
2343 int max_size = max_vector_size(bt);
2344 // Limit the min vector size to 8 bytes.
2345 int size = 8 / type2aelembytes(bt);
2346 if (bt == T_BYTE) {
2347 // To support vector api shuffle/rearrange.
2348 size = 4;
2349 } else if (bt == T_BOOLEAN) {
2350 // To support vector api load/store mask.
2351 size = 2;
2352 }
2353 if (size < 2) size = 2;
2354 return MIN2(size, max_size);
2355 }
2356
2357 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2358 return Matcher::max_vector_size(bt);
2359 }
2360
2361 // Actual max scalable vector register length.
2362 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2363 return Matcher::max_vector_size(bt);
2364 }
2365
2366 // Vector ideal reg.
2367 uint Matcher::vector_ideal_reg(int len) {
2368 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2369 return Op_VecA;
2370 }
2371 switch(len) {
2372 // For 16-bit/32-bit mask vector, reuse VecD.
2373 case 2:
2374 case 4:
2375 case 8: return Op_VecD;
2376 case 16: return Op_VecX;
2377 }
2378 ShouldNotReachHere();
2379 return 0;
2380 }
2381
2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2383 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2384 switch (ideal_reg) {
2385 case Op_VecA: return new vecAOper();
2386 case Op_VecD: return new vecDOper();
2387 case Op_VecX: return new vecXOper();
2388 }
2389 ShouldNotReachHere();
2390 return nullptr;
2391 }
2392
2393 bool Matcher::is_reg2reg_move(MachNode* m) {
2394 return false;
2395 }
2396
2397 bool Matcher::is_generic_vector(MachOper* opnd) {
2398 return opnd->opcode() == VREG;
2399 }
2400
2401 // Return whether or not this register is ever used as an argument.
2402 // This function is used on startup to build the trampoline stubs in
2403 // generateOptoStub. Registers not mentioned will be killed by the VM
2404 // call in the trampoline, and arguments in those registers not be
2405 // available to the callee.
2406 bool Matcher::can_be_java_arg(int reg)
2407 {
2408 return
2409 reg == R0_num || reg == R0_H_num ||
2410 reg == R1_num || reg == R1_H_num ||
2411 reg == R2_num || reg == R2_H_num ||
2412 reg == R3_num || reg == R3_H_num ||
2413 reg == R4_num || reg == R4_H_num ||
2414 reg == R5_num || reg == R5_H_num ||
2415 reg == R6_num || reg == R6_H_num ||
2416 reg == R7_num || reg == R7_H_num ||
2417 reg == V0_num || reg == V0_H_num ||
2418 reg == V1_num || reg == V1_H_num ||
2419 reg == V2_num || reg == V2_H_num ||
2420 reg == V3_num || reg == V3_H_num ||
2421 reg == V4_num || reg == V4_H_num ||
2422 reg == V5_num || reg == V5_H_num ||
2423 reg == V6_num || reg == V6_H_num ||
2424 reg == V7_num || reg == V7_H_num;
2425 }
2426
2427 bool Matcher::is_spillable_arg(int reg)
2428 {
2429 return can_be_java_arg(reg);
2430 }
2431
2432 uint Matcher::int_pressure_limit()
2433 {
2434 // JDK-8183543: When taking the number of available registers as int
2435 // register pressure threshold, the jtreg test:
2436 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2437 // failed due to C2 compilation failure with
2438 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2439 //
2440 // A derived pointer is live at CallNode and then is flagged by RA
2441 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2442 // derived pointers and lastly fail to spill after reaching maximum
2443 // number of iterations. Lowering the default pressure threshold to
2444 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2445 // a high register pressure area of the code so that split_DEF can
2446 // generate DefinitionSpillCopy for the derived pointer.
2447 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2448 if (!PreserveFramePointer) {
2449 // When PreserveFramePointer is off, frame pointer is allocatable,
2450 // but different from other SOC registers, it is excluded from
2451 // fatproj's mask because its save type is No-Save. Decrease 1 to
2452 // ensure high pressure at fatproj when PreserveFramePointer is off.
2453 // See check_pressure_at_fatproj().
2454 default_int_pressure_threshold--;
2455 }
2456 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2457 }
2458
2459 uint Matcher::float_pressure_limit()
2460 {
2461 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2462 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2463 }
2464
2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2466 return false;
2467 }
2468
2469 RegMask Matcher::divI_proj_mask() {
2470 ShouldNotReachHere();
2471 return RegMask();
2472 }
2473
2474 // Register for MODI projection of divmodI.
2475 RegMask Matcher::modI_proj_mask() {
2476 ShouldNotReachHere();
2477 return RegMask();
2478 }
2479
2480 // Register for DIVL projection of divmodL.
2481 RegMask Matcher::divL_proj_mask() {
2482 ShouldNotReachHere();
2483 return RegMask();
2484 }
2485
2486 // Register for MODL projection of divmodL.
2487 RegMask Matcher::modL_proj_mask() {
2488 ShouldNotReachHere();
2489 return RegMask();
2490 }
2491
2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2493 return FP_REG_mask();
2494 }
2495
2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2497 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2498 Node* u = addp->fast_out(i);
2499 if (u->is_LoadStore()) {
2500 // On AArch64, LoadStoreNodes (i.e. compare and swap
2501 // instructions) only take register indirect as an operand, so
2502 // any attempt to use an AddPNode as an input to a LoadStoreNode
2503 // must fail.
2504 return false;
2505 }
2506 if (u->is_Mem()) {
2507 int opsize = u->as_Mem()->memory_size();
2508 assert(opsize > 0, "unexpected memory operand size");
2509 if (u->as_Mem()->memory_size() != (1<<shift)) {
2510 return false;
2511 }
2512 }
2513 }
2514 return true;
2515 }
2516
2517 // Convert BootTest condition to Assembler condition.
2518 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2519 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2520 Assembler::Condition result;
2521 switch(cond) {
2522 case BoolTest::eq:
2523 result = Assembler::EQ; break;
2524 case BoolTest::ne:
2525 result = Assembler::NE; break;
2526 case BoolTest::le:
2527 result = Assembler::LE; break;
2528 case BoolTest::ge:
2529 result = Assembler::GE; break;
2530 case BoolTest::lt:
2531 result = Assembler::LT; break;
2532 case BoolTest::gt:
2533 result = Assembler::GT; break;
2534 case BoolTest::ule:
2535 result = Assembler::LS; break;
2536 case BoolTest::uge:
2537 result = Assembler::HS; break;
2538 case BoolTest::ult:
2539 result = Assembler::LO; break;
2540 case BoolTest::ugt:
2541 result = Assembler::HI; break;
2542 case BoolTest::overflow:
2543 result = Assembler::VS; break;
2544 case BoolTest::no_overflow:
2545 result = Assembler::VC; break;
2546 default:
2547 ShouldNotReachHere();
2548 return Assembler::Condition(-1);
2549 }
2550
2551 // Check conversion
2552 if (cond & BoolTest::unsigned_compare) {
2553 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2554 } else {
2555 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2556 }
2557
2558 return result;
2559 }
2560
2561 // Binary src (Replicate con)
2562 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2563 if (n == nullptr || m == nullptr) {
2564 return false;
2565 }
2566
2567 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2568 return false;
2569 }
2570
2571 Node* imm_node = m->in(1);
2572 if (!imm_node->is_Con()) {
2573 return false;
2574 }
2575
2576 const Type* t = imm_node->bottom_type();
2577 if (!(t->isa_int() || t->isa_long())) {
2578 return false;
2579 }
2580
2581 switch (n->Opcode()) {
2582 case Op_AndV:
2583 case Op_OrV:
2584 case Op_XorV: {
2585 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2586 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2587 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2588 }
2589 case Op_AddVB:
2590 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2591 case Op_AddVS:
2592 case Op_AddVI:
2593 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2594 case Op_AddVL:
2595 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2596 default:
2597 return false;
2598 }
2599 }
2600
2601 // (XorV src (Replicate m1))
2602 // (XorVMask src (MaskAll m1))
2603 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2604 if (n != nullptr && m != nullptr) {
2605 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2606 VectorNode::is_all_ones_vector(m);
2607 }
2608 return false;
2609 }
2610
2611 // Should the matcher clone input 'm' of node 'n'?
2612 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2613 if (is_vshift_con_pattern(n, m) ||
2614 is_vector_bitwise_not_pattern(n, m) ||
2615 is_valid_sve_arith_imm_pattern(n, m) ||
2616 is_encode_and_store_pattern(n, m)) {
2617 mstack.push(m, Visit);
2618 return true;
2619 }
2620 return false;
2621 }
2622
2623 // Should the Matcher clone shifts on addressing modes, expecting them
2624 // to be subsumed into complex addressing expressions or compute them
2625 // into registers?
2626 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2627
2628 // Loads and stores with indirect memory input (e.g., volatile loads and
2629 // stores) do not subsume the input into complex addressing expressions. If
2630 // the addressing expression is input to at least one such load or store, do
2631 // not clone the addressing expression. Query needs_acquiring_load and
2632 // needs_releasing_store as a proxy for indirect memory input, as it is not
2633 // possible to directly query for indirect memory input at this stage.
2634 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2635 Node* n = m->fast_out(i);
2636 if (n->is_Load() && needs_acquiring_load(n)) {
2637 return false;
2638 }
2639 if (n->is_Store() && needs_releasing_store(n)) {
2640 return false;
2641 }
2642 }
2643
2644 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2645 return true;
2646 }
2647
2648 Node *off = m->in(AddPNode::Offset);
2649 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2650 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2651 // Are there other uses besides address expressions?
2652 !is_visited(off)) {
2653 address_visited.set(off->_idx); // Flag as address_visited
2654 mstack.push(off->in(2), Visit);
2655 Node *conv = off->in(1);
2656 if (conv->Opcode() == Op_ConvI2L &&
2657 // Are there other uses besides address expressions?
2658 !is_visited(conv)) {
2659 address_visited.set(conv->_idx); // Flag as address_visited
2660 mstack.push(conv->in(1), Pre_Visit);
2661 } else {
2662 mstack.push(conv, Pre_Visit);
2663 }
2664 address_visited.test_set(m->_idx); // Flag as address_visited
2665 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2666 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2667 return true;
2668 } else if (off->Opcode() == Op_ConvI2L &&
2669 // Are there other uses besides address expressions?
2670 !is_visited(off)) {
2671 address_visited.test_set(m->_idx); // Flag as address_visited
2672 address_visited.set(off->_idx); // Flag as address_visited
2673 mstack.push(off->in(1), Pre_Visit);
2674 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2675 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2676 return true;
2677 }
2678 return false;
2679 }
2680
2681 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2682 { \
2683 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2684 guarantee(DISP == 0, "mode not permitted for volatile"); \
2685 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2686 __ INSN(REG, as_Register(BASE)); \
2687 }
2688
2689
2690 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2691 {
2692 Address::extend scale;
2693
2694 // Hooboy, this is fugly. We need a way to communicate to the
2695 // encoder that the index needs to be sign extended, so we have to
2696 // enumerate all the cases.
2697 switch (opcode) {
2698 case INDINDEXSCALEDI2L:
2699 case INDINDEXSCALEDI2LN:
2700 case INDINDEXI2L:
2701 case INDINDEXI2LN:
2702 scale = Address::sxtw(size);
2703 break;
2704 default:
2705 scale = Address::lsl(size);
2706 }
2707
2708 if (index == -1) {
2709 return Address(base, disp);
2710 } else {
2711 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2712 return Address(base, as_Register(index), scale);
2713 }
2714 }
2715
2716
2717 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2718 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2719 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2720 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2721 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2722
2723 // Used for all non-volatile memory accesses. The use of
2724 // $mem->opcode() to discover whether this pattern uses sign-extended
2725 // offsets is something of a kludge.
2726 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2727 Register reg, int opcode,
2728 Register base, int index, int scale, int disp,
2729 int size_in_memory)
2730 {
2731 Address addr = mem2address(opcode, base, index, scale, disp);
2732 if (addr.getMode() == Address::base_plus_offset) {
2733 /* Fix up any out-of-range offsets. */
2734 assert_different_registers(rscratch1, base);
2735 assert_different_registers(rscratch1, reg);
2736 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2737 }
2738 (masm->*insn)(reg, addr);
2739 }
2740
2741 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2742 FloatRegister reg, int opcode,
2743 Register base, int index, int size, int disp,
2744 int size_in_memory)
2745 {
2746 Address::extend scale;
2747
2748 switch (opcode) {
2749 case INDINDEXSCALEDI2L:
2750 case INDINDEXSCALEDI2LN:
2751 scale = Address::sxtw(size);
2752 break;
2753 default:
2754 scale = Address::lsl(size);
2755 }
2756
2757 if (index == -1) {
2758 // Fix up any out-of-range offsets.
2759 assert_different_registers(rscratch1, base);
2760 Address addr = Address(base, disp);
2761 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2762 (masm->*insn)(reg, addr);
2763 } else {
2764 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2765 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2766 }
2767 }
2768
2769 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2770 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2771 int opcode, Register base, int index, int size, int disp)
2772 {
2773 if (index == -1) {
2774 (masm->*insn)(reg, T, Address(base, disp));
2775 } else {
2776 assert(disp == 0, "unsupported address mode");
2777 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2778 }
2779 }
2780
2781 %}
2782
2783
2784
2785 //----------ENCODING BLOCK-----------------------------------------------------
2786 // This block specifies the encoding classes used by the compiler to
2787 // output byte streams. Encoding classes are parameterized macros
2788 // used by Machine Instruction Nodes in order to generate the bit
2789 // encoding of the instruction. Operands specify their base encoding
2790 // interface with the interface keyword. There are currently
2791 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2792 // COND_INTER. REG_INTER causes an operand to generate a function
2793 // which returns its register number when queried. CONST_INTER causes
2794 // an operand to generate a function which returns the value of the
2795 // constant when queried. MEMORY_INTER causes an operand to generate
2796 // four functions which return the Base Register, the Index Register,
2797 // the Scale Value, and the Offset Value of the operand when queried.
2798 // COND_INTER causes an operand to generate six functions which return
2799 // the encoding code (ie - encoding bits for the instruction)
2800 // associated with each basic boolean condition for a conditional
2801 // instruction.
2802 //
2803 // Instructions specify two basic values for encoding. Again, a
2804 // function is available to check if the constant displacement is an
2805 // oop. They use the ins_encode keyword to specify their encoding
2806 // classes (which must be a sequence of enc_class names, and their
2807 // parameters, specified in the encoding block), and they use the
2808 // opcode keyword to specify, in order, their primary, secondary, and
2809 // tertiary opcode. Only the opcode sections which a particular
2810 // instruction needs for encoding need to be specified.
2811 encode %{
2812 // Build emit functions for each basic byte or larger field in the
2813 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2814 // from C++ code in the enc_class source block. Emit functions will
2815 // live in the main source block for now. In future, we can
2816 // generalize this by adding a syntax that specifies the sizes of
2817 // fields in an order, so that the adlc can build the emit functions
2818 // automagically
2819
2820 // catch all for unimplemented encodings
2821 enc_class enc_unimplemented %{
2822 __ unimplemented("C2 catch all");
2823 %}
2824
2825 // BEGIN Non-volatile memory access
2826
2827 // This encoding class is generated automatically from ad_encode.m4.
2828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2829 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2830 Register dst_reg = as_Register($dst$$reg);
2831 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2832 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2833 %}
2834
2835 // This encoding class is generated automatically from ad_encode.m4.
2836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2837 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2838 Register dst_reg = as_Register($dst$$reg);
2839 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2841 %}
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2926 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2934 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2942 Register src_reg = as_Register($src$$reg);
2943 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strb0(memory1 mem) %{
2950 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2957 Register src_reg = as_Register($src$$reg);
2958 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh0(memory2 mem) %{
2965 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2966 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2967 %}
2968
2969 // This encoding class is generated automatically from ad_encode.m4.
2970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2971 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2972 Register src_reg = as_Register($src$$reg);
2973 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strw0(memory4 mem) %{
2980 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2981 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2982 %}
2983
2984 // This encoding class is generated automatically from ad_encode.m4.
2985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2986 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2987 Register src_reg = as_Register($src$$reg);
2988 // we sometimes get asked to store the stack pointer into the
2989 // current thread -- we cannot do that directly on AArch64
2990 if (src_reg == r31_sp) {
2991 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2992 __ mov(rscratch2, sp);
2993 src_reg = rscratch2;
2994 }
2995 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_str0(memory8 mem) %{
3002 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3004 %}
3005
3006 // This encoding class is generated automatically from ad_encode.m4.
3007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3008 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3009 FloatRegister src_reg = as_FloatRegister($src$$reg);
3010 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3017 FloatRegister src_reg = as_FloatRegister($src$$reg);
3018 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3025 __ membar(Assembler::StoreStore);
3026 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3028 %}
3029
3030 // END Non-volatile memory access
3031
3032 // Vector loads and stores
3033 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3034 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3035 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3036 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3037 %}
3038
3039 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3040 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3041 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3042 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3043 %}
3044
3045 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3046 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3047 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3048 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3049 %}
3050
3051 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3052 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3053 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3054 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3055 %}
3056
3057 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3058 FloatRegister src_reg = as_FloatRegister($src$$reg);
3059 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3060 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3061 %}
3062
3063 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3064 FloatRegister src_reg = as_FloatRegister($src$$reg);
3065 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3066 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3067 %}
3068
3069 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3070 FloatRegister src_reg = as_FloatRegister($src$$reg);
3071 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3072 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3073 %}
3074
3075 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3076 FloatRegister src_reg = as_FloatRegister($src$$reg);
3077 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3078 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3079 %}
3080
3081 // volatile loads and stores
3082
3083 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3084 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3085 rscratch1, stlrb);
3086 %}
3087
3088 enc_class aarch64_enc_stlrb0(memory mem) %{
3089 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3090 rscratch1, stlrb);
3091 %}
3092
3093 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3094 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3095 rscratch1, stlrh);
3096 %}
3097
3098 enc_class aarch64_enc_stlrh0(memory mem) %{
3099 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3100 rscratch1, stlrh);
3101 %}
3102
3103 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3104 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3105 rscratch1, stlrw);
3106 %}
3107
3108 enc_class aarch64_enc_stlrw0(memory mem) %{
3109 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, stlrw);
3111 %}
3112
3113 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3114 Register dst_reg = as_Register($dst$$reg);
3115 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, ldarb);
3117 __ sxtbw(dst_reg, dst_reg);
3118 %}
3119
3120 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3121 Register dst_reg = as_Register($dst$$reg);
3122 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, ldarb);
3124 __ sxtb(dst_reg, dst_reg);
3125 %}
3126
3127 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3128 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 %}
3131
3132 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3133 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3134 rscratch1, ldarb);
3135 %}
3136
3137 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3138 Register dst_reg = as_Register($dst$$reg);
3139 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, ldarh);
3141 __ sxthw(dst_reg, dst_reg);
3142 %}
3143
3144 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3145 Register dst_reg = as_Register($dst$$reg);
3146 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarh);
3148 __ sxth(dst_reg, dst_reg);
3149 %}
3150
3151 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3152 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 %}
3155
3156 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3157 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, ldarh);
3159 %}
3160
3161 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3162 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, ldarw);
3164 %}
3165
3166 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3167 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, ldarw);
3169 %}
3170
3171 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldar);
3174 %}
3175
3176 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3177 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarw);
3179 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3180 %}
3181
3182 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3183 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldar);
3185 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3186 %}
3187
3188 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3189 Register src_reg = as_Register($src$$reg);
3190 // we sometimes get asked to store the stack pointer into the
3191 // current thread -- we cannot do that directly on AArch64
3192 if (src_reg == r31_sp) {
3193 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3194 __ mov(rscratch2, sp);
3195 src_reg = rscratch2;
3196 }
3197 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, stlr);
3199 %}
3200
3201 enc_class aarch64_enc_stlr0(memory mem) %{
3202 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, stlr);
3204 %}
3205
3206 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3207 {
3208 FloatRegister src_reg = as_FloatRegister($src$$reg);
3209 __ fmovs(rscratch2, src_reg);
3210 }
3211 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlrw);
3213 %}
3214
3215 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3216 {
3217 FloatRegister src_reg = as_FloatRegister($src$$reg);
3218 __ fmovd(rscratch2, src_reg);
3219 }
3220 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, stlr);
3222 %}
3223
3224 // synchronized read/update encodings
3225
3226 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3227 Register dst_reg = as_Register($dst$$reg);
3228 Register base = as_Register($mem$$base);
3229 int index = $mem$$index;
3230 int scale = $mem$$scale;
3231 int disp = $mem$$disp;
3232 if (index == -1) {
3233 if (disp != 0) {
3234 __ lea(rscratch1, Address(base, disp));
3235 __ ldaxr(dst_reg, rscratch1);
3236 } else {
3237 // TODO
3238 // should we ever get anything other than this case?
3239 __ ldaxr(dst_reg, base);
3240 }
3241 } else {
3242 Register index_reg = as_Register(index);
3243 if (disp == 0) {
3244 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3245 __ ldaxr(dst_reg, rscratch1);
3246 } else {
3247 __ lea(rscratch1, Address(base, disp));
3248 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3249 __ ldaxr(dst_reg, rscratch1);
3250 }
3251 }
3252 %}
3253
3254 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3255 Register src_reg = as_Register($src$$reg);
3256 Register base = as_Register($mem$$base);
3257 int index = $mem$$index;
3258 int scale = $mem$$scale;
3259 int disp = $mem$$disp;
3260 if (index == -1) {
3261 if (disp != 0) {
3262 __ lea(rscratch2, Address(base, disp));
3263 __ stlxr(rscratch1, src_reg, rscratch2);
3264 } else {
3265 // TODO
3266 // should we ever get anything other than this case?
3267 __ stlxr(rscratch1, src_reg, base);
3268 }
3269 } else {
3270 Register index_reg = as_Register(index);
3271 if (disp == 0) {
3272 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3273 __ stlxr(rscratch1, src_reg, rscratch2);
3274 } else {
3275 __ lea(rscratch2, Address(base, disp));
3276 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3277 __ stlxr(rscratch1, src_reg, rscratch2);
3278 }
3279 }
3280 __ cmpw(rscratch1, zr);
3281 %}
3282
3283 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3284 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3285 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3286 Assembler::xword, /*acquire*/ false, /*release*/ true,
3287 /*weak*/ false, noreg);
3288 %}
3289
3290 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3291 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3292 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3293 Assembler::word, /*acquire*/ false, /*release*/ true,
3294 /*weak*/ false, noreg);
3295 %}
3296
3297 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3298 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3299 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3300 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3301 /*weak*/ false, noreg);
3302 %}
3303
3304 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3305 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3306 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3307 Assembler::byte, /*acquire*/ false, /*release*/ true,
3308 /*weak*/ false, noreg);
3309 %}
3310
3311
3312 // The only difference between aarch64_enc_cmpxchg and
3313 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3314 // CompareAndSwap sequence to serve as a barrier on acquiring a
3315 // lock.
3316 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::xword, /*acquire*/ true, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::word, /*acquire*/ true, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3338 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3339 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3340 Assembler::byte, /*acquire*/ true, /*release*/ true,
3341 /*weak*/ false, noreg);
3342 %}
3343
3344 // auxiliary used for CompareAndSwapX to set result register
3345 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3346 Register res_reg = as_Register($res$$reg);
3347 __ cset(res_reg, Assembler::EQ);
3348 %}
3349
3350 // prefetch encodings
3351
3352 enc_class aarch64_enc_prefetchw(memory mem) %{
3353 Register base = as_Register($mem$$base);
3354 int index = $mem$$index;
3355 int scale = $mem$$scale;
3356 int disp = $mem$$disp;
3357 if (index == -1) {
3358 // Fix up any out-of-range offsets.
3359 assert_different_registers(rscratch1, base);
3360 Address addr = Address(base, disp);
3361 addr = __ legitimize_address(addr, 8, rscratch1);
3362 __ prfm(addr, PSTL1KEEP);
3363 } else {
3364 Register index_reg = as_Register(index);
3365 if (disp == 0) {
3366 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3367 } else {
3368 __ lea(rscratch1, Address(base, disp));
3369 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3370 }
3371 }
3372 %}
3373
3374 // mov encodings
3375
3376 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3377 uint32_t con = (uint32_t)$src$$constant;
3378 Register dst_reg = as_Register($dst$$reg);
3379 if (con == 0) {
3380 __ movw(dst_reg, zr);
3381 } else {
3382 __ movw(dst_reg, con);
3383 }
3384 %}
3385
3386 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3387 Register dst_reg = as_Register($dst$$reg);
3388 uint64_t con = (uint64_t)$src$$constant;
3389 if (con == 0) {
3390 __ mov(dst_reg, zr);
3391 } else {
3392 __ mov(dst_reg, con);
3393 }
3394 %}
3395
3396 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3397 Register dst_reg = as_Register($dst$$reg);
3398 address con = (address)$src$$constant;
3399 if (con == nullptr || con == (address)1) {
3400 ShouldNotReachHere();
3401 } else {
3402 relocInfo::relocType rtype = $src->constant_reloc();
3403 if (rtype == relocInfo::oop_type) {
3404 __ movoop(dst_reg, (jobject)con);
3405 } else if (rtype == relocInfo::metadata_type) {
3406 __ mov_metadata(dst_reg, (Metadata*)con);
3407 } else {
3408 assert(rtype == relocInfo::none, "unexpected reloc type");
3409 if (! __ is_valid_AArch64_address(con) ||
3410 con < (address)(uintptr_t)os::vm_page_size()) {
3411 __ mov(dst_reg, con);
3412 } else {
3413 uint64_t offset;
3414 __ adrp(dst_reg, con, offset);
3415 __ add(dst_reg, dst_reg, offset);
3416 }
3417 }
3418 }
3419 %}
3420
3421 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3422 Register dst_reg = as_Register($dst$$reg);
3423 __ mov(dst_reg, zr);
3424 %}
3425
3426 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, (uint64_t)1);
3429 %}
3430
3431 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3432 __ load_byte_map_base($dst$$Register);
3433 %}
3434
3435 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 address con = (address)$src$$constant;
3438 if (con == nullptr) {
3439 ShouldNotReachHere();
3440 } else {
3441 relocInfo::relocType rtype = $src->constant_reloc();
3442 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3443 __ set_narrow_oop(dst_reg, (jobject)con);
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 address con = (address)$src$$constant;
3455 if (con == nullptr) {
3456 ShouldNotReachHere();
3457 } else {
3458 relocInfo::relocType rtype = $src->constant_reloc();
3459 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3460 __ set_narrow_klass(dst_reg, (Klass *)con);
3461 }
3462 %}
3463
3464 // arithmetic encodings
3465
3466 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ subw(dst_reg, src_reg, -con);
3474 } else {
3475 __ addw(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src_reg = as_Register($src1$$reg);
3482 int32_t con = (int32_t)$src2$$constant;
3483 // add has primary == 0, subtract has primary == 1
3484 if ($primary) { con = -con; }
3485 if (con < 0) {
3486 __ sub(dst_reg, src_reg, -con);
3487 } else {
3488 __ add(dst_reg, src_reg, con);
3489 }
3490 %}
3491
3492 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3497 %}
3498
3499 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src1_reg = as_Register($src1$$reg);
3502 Register src2_reg = as_Register($src2$$reg);
3503 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3504 %}
3505
3506 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 Register src1_reg = as_Register($src1$$reg);
3509 Register src2_reg = as_Register($src2$$reg);
3510 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3511 %}
3512
3513 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3514 Register dst_reg = as_Register($dst$$reg);
3515 Register src1_reg = as_Register($src1$$reg);
3516 Register src2_reg = as_Register($src2$$reg);
3517 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3518 %}
3519
3520 // compare instruction encodings
3521
3522 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3523 Register reg1 = as_Register($src1$$reg);
3524 Register reg2 = as_Register($src2$$reg);
3525 __ cmpw(reg1, reg2);
3526 %}
3527
3528 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3529 Register reg = as_Register($src1$$reg);
3530 int32_t val = $src2$$constant;
3531 if (val >= 0) {
3532 __ subsw(zr, reg, val);
3533 } else {
3534 __ addsw(zr, reg, -val);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3539 Register reg1 = as_Register($src1$$reg);
3540 uint32_t val = (uint32_t)$src2$$constant;
3541 __ movw(rscratch1, val);
3542 __ cmpw(reg1, rscratch1);
3543 %}
3544
3545 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 Register reg2 = as_Register($src2$$reg);
3548 __ cmp(reg1, reg2);
3549 %}
3550
3551 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3552 Register reg = as_Register($src1$$reg);
3553 int64_t val = $src2$$constant;
3554 if (val >= 0) {
3555 __ subs(zr, reg, val);
3556 } else if (val != -val) {
3557 __ adds(zr, reg, -val);
3558 } else {
3559 // aargh, Long.MIN_VALUE is a special case
3560 __ orr(rscratch1, zr, (uint64_t)val);
3561 __ subs(zr, reg, rscratch1);
3562 }
3563 %}
3564
3565 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 uint64_t val = (uint64_t)$src2$$constant;
3568 __ mov(rscratch1, val);
3569 __ cmp(reg1, rscratch1);
3570 %}
3571
3572 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3573 Register reg1 = as_Register($src1$$reg);
3574 Register reg2 = as_Register($src2$$reg);
3575 __ cmp(reg1, reg2);
3576 %}
3577
3578 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3579 Register reg1 = as_Register($src1$$reg);
3580 Register reg2 = as_Register($src2$$reg);
3581 __ cmpw(reg1, reg2);
3582 %}
3583
3584 enc_class aarch64_enc_testp(iRegP src) %{
3585 Register reg = as_Register($src$$reg);
3586 __ cmp(reg, zr);
3587 %}
3588
3589 enc_class aarch64_enc_testn(iRegN src) %{
3590 Register reg = as_Register($src$$reg);
3591 __ cmpw(reg, zr);
3592 %}
3593
3594 enc_class aarch64_enc_b(label lbl) %{
3595 Label *L = $lbl$$label;
3596 __ b(*L);
3597 %}
3598
3599 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3600 Label *L = $lbl$$label;
3601 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3602 %}
3603
3604 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3605 Label *L = $lbl$$label;
3606 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3607 %}
3608
3609 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3610 %{
3611 Register sub_reg = as_Register($sub$$reg);
3612 Register super_reg = as_Register($super$$reg);
3613 Register temp_reg = as_Register($temp$$reg);
3614 Register result_reg = as_Register($result$$reg);
3615
3616 Label miss;
3617 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3618 nullptr, &miss,
3619 /*set_cond_codes:*/ true);
3620 if ($primary) {
3621 __ mov(result_reg, zr);
3622 }
3623 __ bind(miss);
3624 %}
3625
3626 enc_class aarch64_enc_java_static_call(method meth) %{
3627 address addr = (address)$meth$$method;
3628 address call;
3629 if (!_method) {
3630 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3631 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3632 if (call == nullptr) {
3633 ciEnv::current()->record_failure("CodeCache is full");
3634 return;
3635 }
3636 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3637 // The NOP here is purely to ensure that eliding a call to
3638 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3639 __ nop();
3640 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3641 } else {
3642 int method_index = resolved_method_index(masm);
3643 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3644 : static_call_Relocation::spec(method_index);
3645 call = __ trampoline_call(Address(addr, rspec));
3646 if (call == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3651 // Calls of the same statically bound method can share
3652 // a stub to the interpreter.
3653 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3654 } else {
3655 // Emit stub for static call
3656 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3657 if (stub == nullptr) {
3658 ciEnv::current()->record_failure("CodeCache is full");
3659 return;
3660 }
3661 }
3662 }
3663
3664 __ post_call_nop();
3665
3666 // Only non uncommon_trap calls need to reinitialize ptrue.
3667 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3673 int method_index = resolved_method_index(masm);
3674 address call = __ ic_call((address)$meth$$method, method_index);
3675 if (call == nullptr) {
3676 ciEnv::current()->record_failure("CodeCache is full");
3677 return;
3678 }
3679 __ post_call_nop();
3680 if (Compile::current()->max_vector_size() > 0) {
3681 __ reinitialize_ptrue();
3682 }
3683 %}
3684
3685 enc_class aarch64_enc_call_epilog() %{
3686 if (VerifyStackAtCalls) {
3687 // Check that stack depth is unchanged: find majik cookie on stack
3688 __ call_Unimplemented();
3689 }
3690 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3691 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3692 // Search for the corresponding projection, get the register and emit code that initialized it.
3693 uint con = (tf()->range_cc()->cnt() - 1);
3694 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3695 ProjNode* proj = fast_out(i)->as_Proj();
3696 if (proj->_con == con) {
3697 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3698 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3699 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3700 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3701 __ cmp(r0, zr);
3702 __ cset(toReg, Assembler::NE);
3703 if (reg->is_stack()) {
3704 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3705 __ str(toReg, Address(sp, st_off));
3706 }
3707 break;
3708 }
3709 }
3710 if (return_value_is_used()) {
3711 // An inline type is returned as fields in multiple registers.
3712 // R0 either contains an oop if the inline type is buffered or a pointer
3713 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3714 // if the lowest bit is set to allow C2 to use the oop after null checking.
3715 // r0 &= (r0 & 1) - 1
3716 __ andr(rscratch1, r0, 0x1);
3717 __ sub(rscratch1, rscratch1, 0x1);
3718 __ andr(r0, r0, rscratch1);
3719 }
3720 }
3721 %}
3722
3723 enc_class aarch64_enc_java_to_runtime(method meth) %{
3724 // some calls to generated routines (arraycopy code) are scheduled
3725 // by C2 as runtime calls. if so we can call them using a br (they
3726 // will be in a reachable segment) otherwise we have to use a blr
3727 // which loads the absolute address into a register.
3728 address entry = (address)$meth$$method;
3729 CodeBlob *cb = CodeCache::find_blob(entry);
3730 if (cb) {
3731 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3732 if (call == nullptr) {
3733 ciEnv::current()->record_failure("CodeCache is full");
3734 return;
3735 }
3736 __ post_call_nop();
3737 } else {
3738 Label retaddr;
3739 // Make the anchor frame walkable
3740 __ adr(rscratch2, retaddr);
3741 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3742 __ lea(rscratch1, RuntimeAddress(entry));
3743 __ blr(rscratch1);
3744 __ bind(retaddr);
3745 __ post_call_nop();
3746 }
3747 if (Compile::current()->max_vector_size() > 0) {
3748 __ reinitialize_ptrue();
3749 }
3750 %}
3751
3752 enc_class aarch64_enc_rethrow() %{
3753 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3754 %}
3755
3756 enc_class aarch64_enc_ret() %{
3757 #ifdef ASSERT
3758 if (Compile::current()->max_vector_size() > 0) {
3759 __ verify_ptrue();
3760 }
3761 #endif
3762 __ ret(lr);
3763 %}
3764
3765 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3766 Register target_reg = as_Register($jump_target$$reg);
3767 __ br(target_reg);
3768 %}
3769
3770 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3771 Register target_reg = as_Register($jump_target$$reg);
3772 // exception oop should be in r0
3773 // ret addr has been popped into lr
3774 // callee expects it in r3
3775 __ mov(r3, lr);
3776 __ br(target_reg);
3777 %}
3778
3779 %}
3780
3781 //----------FRAME--------------------------------------------------------------
3782 // Definition of frame structure and management information.
3783 //
3784 // S T A C K L A Y O U T Allocators stack-slot number
3785 // | (to get allocators register number
3786 // G Owned by | | v add OptoReg::stack0())
3787 // r CALLER | |
3788 // o | +--------+ pad to even-align allocators stack-slot
3789 // w V | pad0 | numbers; owned by CALLER
3790 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3791 // h ^ | in | 5
3792 // | | args | 4 Holes in incoming args owned by SELF
3793 // | | | | 3
3794 // | | +--------+
3795 // V | | old out| Empty on Intel, window on Sparc
3796 // | old |preserve| Must be even aligned.
3797 // | SP-+--------+----> Matcher::_old_SP, even aligned
3798 // | | in | 3 area for Intel ret address
3799 // Owned by |preserve| Empty on Sparc.
3800 // SELF +--------+
3801 // | | pad2 | 2 pad to align old SP
3802 // | +--------+ 1
3803 // | | locks | 0
3804 // | +--------+----> OptoReg::stack0(), even aligned
3805 // | | pad1 | 11 pad to align new SP
3806 // | +--------+
3807 // | | | 10
3808 // | | spills | 9 spills
3809 // V | | 8 (pad0 slot for callee)
3810 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3811 // ^ | out | 7
3812 // | | args | 6 Holes in outgoing args owned by CALLEE
3813 // Owned by +--------+
3814 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3815 // | new |preserve| Must be even-aligned.
3816 // | SP-+--------+----> Matcher::_new_SP, even aligned
3817 // | | |
3818 //
3819 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3820 // known from SELF's arguments and the Java calling convention.
3821 // Region 6-7 is determined per call site.
3822 // Note 2: If the calling convention leaves holes in the incoming argument
3823 // area, those holes are owned by SELF. Holes in the outgoing area
3824 // are owned by the CALLEE. Holes should not be necessary in the
3825 // incoming area, as the Java calling convention is completely under
3826 // the control of the AD file. Doubles can be sorted and packed to
3827 // avoid holes. Holes in the outgoing arguments may be necessary for
3828 // varargs C calling conventions.
3829 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3830 // even aligned with pad0 as needed.
3831 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3832 // (the latter is true on Intel but is it false on AArch64?)
3833 // region 6-11 is even aligned; it may be padded out more so that
3834 // the region from SP to FP meets the minimum stack alignment.
3835 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3836 // alignment. Region 11, pad1, may be dynamically extended so that
3837 // SP meets the minimum alignment.
3838
3839 frame %{
3840 // These three registers define part of the calling convention
3841 // between compiled code and the interpreter.
3842
3843 // Inline Cache Register or Method for I2C.
3844 inline_cache_reg(R12);
3845
3846 // Number of stack slots consumed by locking an object
3847 sync_stack_slots(2);
3848
3849 // Compiled code's Frame Pointer
3850 frame_pointer(R31);
3851
3852 // Interpreter stores its frame pointer in a register which is
3853 // stored to the stack by I2CAdaptors.
3854 // I2CAdaptors convert from interpreted java to compiled java.
3855 interpreter_frame_pointer(R29);
3856
3857 // Stack alignment requirement
3858 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3859
3860 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3861 // for calls to C. Supports the var-args backing area for register parms.
3862 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3863
3864 // The after-PROLOG location of the return address. Location of
3865 // return address specifies a type (REG or STACK) and a number
3866 // representing the register number (i.e. - use a register name) or
3867 // stack slot.
3868 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3869 // Otherwise, it is above the locks and verification slot and alignment word
3870 // TODO this may well be correct but need to check why that - 2 is there
3871 // ppc port uses 0 but we definitely need to allow for fixed_slots
3872 // which folds in the space used for monitors
3873 return_addr(STACK - 2 +
3874 align_up((Compile::current()->in_preserve_stack_slots() +
3875 Compile::current()->fixed_slots()),
3876 stack_alignment_in_slots()));
3877
3878 // Location of compiled Java return values. Same as C for now.
3879 return_value
3880 %{
3881 // TODO do we allow ideal_reg == Op_RegN???
3882 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3883 "only return normal values");
3884
3885 static const int lo[Op_RegL + 1] = { // enum name
3886 0, // Op_Node
3887 0, // Op_Set
3888 R0_num, // Op_RegN
3889 R0_num, // Op_RegI
3890 R0_num, // Op_RegP
3891 V0_num, // Op_RegF
3892 V0_num, // Op_RegD
3893 R0_num // Op_RegL
3894 };
3895
3896 static const int hi[Op_RegL + 1] = { // enum name
3897 0, // Op_Node
3898 0, // Op_Set
3899 OptoReg::Bad, // Op_RegN
3900 OptoReg::Bad, // Op_RegI
3901 R0_H_num, // Op_RegP
3902 OptoReg::Bad, // Op_RegF
3903 V0_H_num, // Op_RegD
3904 R0_H_num // Op_RegL
3905 };
3906
3907 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3908 %}
3909 %}
3910
3911 //----------ATTRIBUTES---------------------------------------------------------
3912 //----------Operand Attributes-------------------------------------------------
3913 op_attrib op_cost(1); // Required cost attribute
3914
3915 //----------Instruction Attributes---------------------------------------------
3916 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3917 ins_attrib ins_size(32); // Required size attribute (in bits)
3918 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3919 // a non-matching short branch variant
3920 // of some long branch?
3921 ins_attrib ins_alignment(4); // Required alignment attribute (must
3922 // be a power of 2) specifies the
3923 // alignment that some part of the
3924 // instruction (not necessarily the
3925 // start) requires. If > 1, a
3926 // compute_padding() function must be
3927 // provided for the instruction
3928
3929 //----------OPERANDS-----------------------------------------------------------
3930 // Operand definitions must precede instruction definitions for correct parsing
3931 // in the ADLC because operands constitute user defined types which are used in
3932 // instruction definitions.
3933
3934 //----------Simple Operands----------------------------------------------------
3935
3936 // Integer operands 32 bit
3937 // 32 bit immediate
3938 operand immI()
3939 %{
3940 match(ConI);
3941
3942 op_cost(0);
3943 format %{ %}
3944 interface(CONST_INTER);
3945 %}
3946
3947 // 32 bit zero
3948 operand immI0()
3949 %{
3950 predicate(n->get_int() == 0);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 // 32 bit unit increment
3959 operand immI_1()
3960 %{
3961 predicate(n->get_int() == 1);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 // 32 bit unit decrement
3970 operand immI_M1()
3971 %{
3972 predicate(n->get_int() == -1);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 // Shift values for add/sub extension shift
3981 operand immIExt()
3982 %{
3983 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 operand immI_gt_1()
3992 %{
3993 predicate(n->get_int() > 1);
3994 match(ConI);
3995
3996 op_cost(0);
3997 format %{ %}
3998 interface(CONST_INTER);
3999 %}
4000
4001 operand immI_le_4()
4002 %{
4003 predicate(n->get_int() <= 4);
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immI_16()
4012 %{
4013 predicate(n->get_int() == 16);
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immI_24()
4022 %{
4023 predicate(n->get_int() == 24);
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immI_32()
4032 %{
4033 predicate(n->get_int() == 32);
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immI_48()
4042 %{
4043 predicate(n->get_int() == 48);
4044 match(ConI);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immI_56()
4052 %{
4053 predicate(n->get_int() == 56);
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immI_255()
4062 %{
4063 predicate(n->get_int() == 255);
4064 match(ConI);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 operand immI_65535()
4072 %{
4073 predicate(n->get_int() == 65535);
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immI_positive()
4082 %{
4083 predicate(n->get_int() > 0);
4084 match(ConI);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 // BoolTest condition for signed compare
4092 operand immI_cmp_cond()
4093 %{
4094 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4095 match(ConI);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 // BoolTest condition for unsigned compare
4103 operand immI_cmpU_cond()
4104 %{
4105 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4106 match(ConI);
4107
4108 op_cost(0);
4109 format %{ %}
4110 interface(CONST_INTER);
4111 %}
4112
4113 operand immL_255()
4114 %{
4115 predicate(n->get_long() == 255L);
4116 match(ConL);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immL_65535()
4124 %{
4125 predicate(n->get_long() == 65535L);
4126 match(ConL);
4127
4128 op_cost(0);
4129 format %{ %}
4130 interface(CONST_INTER);
4131 %}
4132
4133 operand immL_4294967295()
4134 %{
4135 predicate(n->get_long() == 4294967295L);
4136 match(ConL);
4137
4138 op_cost(0);
4139 format %{ %}
4140 interface(CONST_INTER);
4141 %}
4142
4143 operand immL_bitmask()
4144 %{
4145 predicate((n->get_long() != 0)
4146 && ((n->get_long() & 0xc000000000000000l) == 0)
4147 && is_power_of_2(n->get_long() + 1));
4148 match(ConL);
4149
4150 op_cost(0);
4151 format %{ %}
4152 interface(CONST_INTER);
4153 %}
4154
4155 operand immI_bitmask()
4156 %{
4157 predicate((n->get_int() != 0)
4158 && ((n->get_int() & 0xc0000000) == 0)
4159 && is_power_of_2(n->get_int() + 1));
4160 match(ConI);
4161
4162 op_cost(0);
4163 format %{ %}
4164 interface(CONST_INTER);
4165 %}
4166
4167 operand immL_positive_bitmaskI()
4168 %{
4169 predicate((n->get_long() != 0)
4170 && ((julong)n->get_long() < 0x80000000ULL)
4171 && is_power_of_2(n->get_long() + 1));
4172 match(ConL);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 // Scale values for scaled offset addressing modes (up to long but not quad)
4180 operand immIScale()
4181 %{
4182 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 // 5 bit signed integer
4191 operand immI5()
4192 %{
4193 predicate(Assembler::is_simm(n->get_int(), 5));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 // 7 bit unsigned integer
4202 operand immIU7()
4203 %{
4204 predicate(Assembler::is_uimm(n->get_int(), 7));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 // Offset for scaled or unscaled immediate loads and stores
4213 operand immIOffset()
4214 %{
4215 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4216 match(ConI);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 operand immIOffset1()
4224 %{
4225 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4226 match(ConI);
4227
4228 op_cost(0);
4229 format %{ %}
4230 interface(CONST_INTER);
4231 %}
4232
4233 operand immIOffset2()
4234 %{
4235 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 operand immIOffset4()
4244 %{
4245 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4246 match(ConI);
4247
4248 op_cost(0);
4249 format %{ %}
4250 interface(CONST_INTER);
4251 %}
4252
4253 operand immIOffset8()
4254 %{
4255 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4256 match(ConI);
4257
4258 op_cost(0);
4259 format %{ %}
4260 interface(CONST_INTER);
4261 %}
4262
4263 operand immIOffset16()
4264 %{
4265 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4266 match(ConI);
4267
4268 op_cost(0);
4269 format %{ %}
4270 interface(CONST_INTER);
4271 %}
4272
4273 operand immLOffset()
4274 %{
4275 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4276 match(ConL);
4277
4278 op_cost(0);
4279 format %{ %}
4280 interface(CONST_INTER);
4281 %}
4282
4283 operand immLoffset1()
4284 %{
4285 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4286 match(ConL);
4287
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 operand immLoffset2()
4294 %{
4295 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 operand immLoffset4()
4304 %{
4305 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4306 match(ConL);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 operand immLoffset8()
4314 %{
4315 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 operand immLoffset16()
4324 %{
4325 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4326 match(ConL);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 // 5 bit signed long integer
4334 operand immL5()
4335 %{
4336 predicate(Assembler::is_simm(n->get_long(), 5));
4337 match(ConL);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 // 7 bit unsigned long integer
4345 operand immLU7()
4346 %{
4347 predicate(Assembler::is_uimm(n->get_long(), 7));
4348 match(ConL);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value.
4356 operand immI8()
4357 %{
4358 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4359 match(ConI);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 // 8 bit signed value (simm8), or #simm8 LSL 8.
4367 operand immI8_shift8()
4368 %{
4369 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4370 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 8 bit signed value (simm8), or #simm8 LSL 8.
4379 operand immL8_shift8()
4380 %{
4381 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4382 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4383 match(ConL);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 // 8 bit integer valid for vector add sub immediate
4391 operand immBAddSubV()
4392 %{
4393 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 // 32 bit integer valid for add sub immediate
4402 operand immIAddSub()
4403 %{
4404 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4405 match(ConI);
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // 32 bit integer valid for vector add sub immediate
4412 operand immIAddSubV()
4413 %{
4414 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4415 match(ConI);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 // 32 bit unsigned integer valid for logical immediate
4423
4424 operand immBLog()
4425 %{
4426 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4427 match(ConI);
4428
4429 op_cost(0);
4430 format %{ %}
4431 interface(CONST_INTER);
4432 %}
4433
4434 operand immSLog()
4435 %{
4436 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4437 match(ConI);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 operand immILog()
4445 %{
4446 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // Integer operands 64 bit
4455 // 64 bit immediate
4456 operand immL()
4457 %{
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit zero
4466 operand immL0()
4467 %{
4468 predicate(n->get_long() == 0);
4469 match(ConL);
4470
4471 op_cost(0);
4472 format %{ %}
4473 interface(CONST_INTER);
4474 %}
4475
4476 // 64 bit unit decrement
4477 operand immL_M1()
4478 %{
4479 predicate(n->get_long() == -1);
4480 match(ConL);
4481
4482 op_cost(0);
4483 format %{ %}
4484 interface(CONST_INTER);
4485 %}
4486
4487 // 64 bit integer valid for add sub immediate
4488 operand immLAddSub()
4489 %{
4490 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4491 match(ConL);
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 // 64 bit integer valid for addv subv immediate
4498 operand immLAddSubV()
4499 %{
4500 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4501 match(ConL);
4502
4503 op_cost(0);
4504 format %{ %}
4505 interface(CONST_INTER);
4506 %}
4507
4508 // 64 bit integer valid for logical immediate
4509 operand immLLog()
4510 %{
4511 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4512 match(ConL);
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // Long Immediate: low 32-bit mask
4519 operand immL_32bits()
4520 %{
4521 predicate(n->get_long() == 0xFFFFFFFFL);
4522 match(ConL);
4523 op_cost(0);
4524 format %{ %}
4525 interface(CONST_INTER);
4526 %}
4527
4528 // Pointer operands
4529 // Pointer Immediate
4530 operand immP()
4531 %{
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // nullptr Pointer Immediate
4540 operand immP0()
4541 %{
4542 predicate(n->get_ptr() == 0);
4543 match(ConP);
4544
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // Pointer Immediate One
4551 // this is used in object initialization (initial object header)
4552 operand immP_1()
4553 %{
4554 predicate(n->get_ptr() == 1);
4555 match(ConP);
4556
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Card Table Byte Map Base
4563 operand immByteMapBase()
4564 %{
4565 // Get base of card map
4566 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4567 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4568 match(ConP);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // Float and Double operands
4576 // Double Immediate
4577 operand immD()
4578 %{
4579 match(ConD);
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 // Double Immediate: +0.0d
4586 operand immD0()
4587 %{
4588 predicate(jlong_cast(n->getd()) == 0);
4589 match(ConD);
4590
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // constant 'double +0.0'.
4597 operand immDPacked()
4598 %{
4599 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4600 match(ConD);
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Float Immediate
4607 operand immF()
4608 %{
4609 match(ConF);
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 // Float Immediate: +0.0f.
4616 operand immF0()
4617 %{
4618 predicate(jint_cast(n->getf()) == 0);
4619 match(ConF);
4620
4621 op_cost(0);
4622 format %{ %}
4623 interface(CONST_INTER);
4624 %}
4625
4626 //
4627 operand immFPacked()
4628 %{
4629 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4630 match(ConF);
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // Narrow pointer operands
4637 // Narrow Pointer Immediate
4638 operand immN()
4639 %{
4640 match(ConN);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // Narrow nullptr Pointer Immediate
4648 operand immN0()
4649 %{
4650 predicate(n->get_narrowcon() == 0);
4651 match(ConN);
4652
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 operand immNKlass()
4659 %{
4660 match(ConNKlass);
4661
4662 op_cost(0);
4663 format %{ %}
4664 interface(CONST_INTER);
4665 %}
4666
4667 // Integer 32 bit Register Operands
4668 // Integer 32 bitRegister (excludes SP)
4669 operand iRegI()
4670 %{
4671 constraint(ALLOC_IN_RC(any_reg32));
4672 match(RegI);
4673 match(iRegINoSp);
4674 op_cost(0);
4675 format %{ %}
4676 interface(REG_INTER);
4677 %}
4678
4679 // Integer 32 bit Register not Special
4680 operand iRegINoSp()
4681 %{
4682 constraint(ALLOC_IN_RC(no_special_reg32));
4683 match(RegI);
4684 op_cost(0);
4685 format %{ %}
4686 interface(REG_INTER);
4687 %}
4688
4689 // Integer 64 bit Register Operands
4690 // Integer 64 bit Register (includes SP)
4691 operand iRegL()
4692 %{
4693 constraint(ALLOC_IN_RC(any_reg));
4694 match(RegL);
4695 match(iRegLNoSp);
4696 op_cost(0);
4697 format %{ %}
4698 interface(REG_INTER);
4699 %}
4700
4701 // Integer 64 bit Register not Special
4702 operand iRegLNoSp()
4703 %{
4704 constraint(ALLOC_IN_RC(no_special_reg));
4705 match(RegL);
4706 match(iRegL_R0);
4707 format %{ %}
4708 interface(REG_INTER);
4709 %}
4710
4711 // Pointer Register Operands
4712 // Pointer Register
4713 operand iRegP()
4714 %{
4715 constraint(ALLOC_IN_RC(ptr_reg));
4716 match(RegP);
4717 match(iRegPNoSp);
4718 match(iRegP_R0);
4719 //match(iRegP_R2);
4720 //match(iRegP_R4);
4721 match(iRegP_R5);
4722 match(thread_RegP);
4723 op_cost(0);
4724 format %{ %}
4725 interface(REG_INTER);
4726 %}
4727
4728 // Pointer 64 bit Register not Special
4729 operand iRegPNoSp()
4730 %{
4731 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4732 match(RegP);
4733 // match(iRegP);
4734 // match(iRegP_R0);
4735 // match(iRegP_R2);
4736 // match(iRegP_R4);
4737 // match(iRegP_R5);
4738 // match(thread_RegP);
4739 op_cost(0);
4740 format %{ %}
4741 interface(REG_INTER);
4742 %}
4743
4744 // This operand is not allowed to use rfp even if
4745 // rfp is not used to hold the frame pointer.
4746 operand iRegPNoSpNoRfp()
4747 %{
4748 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4749 match(RegP);
4750 match(iRegPNoSp);
4751 op_cost(0);
4752 format %{ %}
4753 interface(REG_INTER);
4754 %}
4755
4756 // Pointer 64 bit Register R0 only
4757 operand iRegP_R0()
4758 %{
4759 constraint(ALLOC_IN_RC(r0_reg));
4760 match(RegP);
4761 // match(iRegP);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Pointer 64 bit Register R1 only
4769 operand iRegP_R1()
4770 %{
4771 constraint(ALLOC_IN_RC(r1_reg));
4772 match(RegP);
4773 // match(iRegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R2 only
4781 operand iRegP_R2()
4782 %{
4783 constraint(ALLOC_IN_RC(r2_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R3 only
4793 operand iRegP_R3()
4794 %{
4795 constraint(ALLOC_IN_RC(r3_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Pointer 64 bit Register R4 only
4805 operand iRegP_R4()
4806 %{
4807 constraint(ALLOC_IN_RC(r4_reg));
4808 match(RegP);
4809 // match(iRegP);
4810 match(iRegPNoSp);
4811 op_cost(0);
4812 format %{ %}
4813 interface(REG_INTER);
4814 %}
4815
4816 // Pointer 64 bit Register R5 only
4817 operand iRegP_R5()
4818 %{
4819 constraint(ALLOC_IN_RC(r5_reg));
4820 match(RegP);
4821 // match(iRegP);
4822 match(iRegPNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Pointer 64 bit Register R10 only
4829 operand iRegP_R10()
4830 %{
4831 constraint(ALLOC_IN_RC(r10_reg));
4832 match(RegP);
4833 // match(iRegP);
4834 match(iRegPNoSp);
4835 op_cost(0);
4836 format %{ %}
4837 interface(REG_INTER);
4838 %}
4839
4840 // Long 64 bit Register R0 only
4841 operand iRegL_R0()
4842 %{
4843 constraint(ALLOC_IN_RC(r0_reg));
4844 match(RegL);
4845 match(iRegLNoSp);
4846 op_cost(0);
4847 format %{ %}
4848 interface(REG_INTER);
4849 %}
4850
4851 // Long 64 bit Register R11 only
4852 operand iRegL_R11()
4853 %{
4854 constraint(ALLOC_IN_RC(r11_reg));
4855 match(RegL);
4856 match(iRegLNoSp);
4857 op_cost(0);
4858 format %{ %}
4859 interface(REG_INTER);
4860 %}
4861
4862 // Register R0 only
4863 operand iRegI_R0()
4864 %{
4865 constraint(ALLOC_IN_RC(int_r0_reg));
4866 match(RegI);
4867 match(iRegINoSp);
4868 op_cost(0);
4869 format %{ %}
4870 interface(REG_INTER);
4871 %}
4872
4873 // Register R2 only
4874 operand iRegI_R2()
4875 %{
4876 constraint(ALLOC_IN_RC(int_r2_reg));
4877 match(RegI);
4878 match(iRegINoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Register R3 only
4885 operand iRegI_R3()
4886 %{
4887 constraint(ALLOC_IN_RC(int_r3_reg));
4888 match(RegI);
4889 match(iRegINoSp);
4890 op_cost(0);
4891 format %{ %}
4892 interface(REG_INTER);
4893 %}
4894
4895
4896 // Register R4 only
4897 operand iRegI_R4()
4898 %{
4899 constraint(ALLOC_IN_RC(int_r4_reg));
4900 match(RegI);
4901 match(iRegINoSp);
4902 op_cost(0);
4903 format %{ %}
4904 interface(REG_INTER);
4905 %}
4906
4907
4908 // Pointer Register Operands
4909 // Narrow Pointer Register
4910 operand iRegN()
4911 %{
4912 constraint(ALLOC_IN_RC(any_reg32));
4913 match(RegN);
4914 match(iRegNNoSp);
4915 op_cost(0);
4916 format %{ %}
4917 interface(REG_INTER);
4918 %}
4919
4920 // Integer 64 bit Register not Special
4921 operand iRegNNoSp()
4922 %{
4923 constraint(ALLOC_IN_RC(no_special_reg32));
4924 match(RegN);
4925 op_cost(0);
4926 format %{ %}
4927 interface(REG_INTER);
4928 %}
4929
4930 // Float Register
4931 // Float register operands
4932 operand vRegF()
4933 %{
4934 constraint(ALLOC_IN_RC(float_reg));
4935 match(RegF);
4936
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 // Double Register
4943 // Double register operands
4944 operand vRegD()
4945 %{
4946 constraint(ALLOC_IN_RC(double_reg));
4947 match(RegD);
4948
4949 op_cost(0);
4950 format %{ %}
4951 interface(REG_INTER);
4952 %}
4953
4954 // Generic vector class. This will be used for
4955 // all vector operands, including NEON and SVE.
4956 operand vReg()
4957 %{
4958 constraint(ALLOC_IN_RC(dynamic));
4959 match(VecA);
4960 match(VecD);
4961 match(VecX);
4962
4963 op_cost(0);
4964 format %{ %}
4965 interface(REG_INTER);
4966 %}
4967
4968 operand vecA()
4969 %{
4970 constraint(ALLOC_IN_RC(vectora_reg));
4971 match(VecA);
4972
4973 op_cost(0);
4974 format %{ %}
4975 interface(REG_INTER);
4976 %}
4977
4978 operand vecD()
4979 %{
4980 constraint(ALLOC_IN_RC(vectord_reg));
4981 match(VecD);
4982
4983 op_cost(0);
4984 format %{ %}
4985 interface(REG_INTER);
4986 %}
4987
4988 operand vecX()
4989 %{
4990 constraint(ALLOC_IN_RC(vectorx_reg));
4991 match(VecX);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 operand vRegD_V0()
4999 %{
5000 constraint(ALLOC_IN_RC(v0_reg));
5001 match(RegD);
5002 op_cost(0);
5003 format %{ %}
5004 interface(REG_INTER);
5005 %}
5006
5007 operand vRegD_V1()
5008 %{
5009 constraint(ALLOC_IN_RC(v1_reg));
5010 match(RegD);
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 operand vRegD_V2()
5017 %{
5018 constraint(ALLOC_IN_RC(v2_reg));
5019 match(RegD);
5020 op_cost(0);
5021 format %{ %}
5022 interface(REG_INTER);
5023 %}
5024
5025 operand vRegD_V3()
5026 %{
5027 constraint(ALLOC_IN_RC(v3_reg));
5028 match(RegD);
5029 op_cost(0);
5030 format %{ %}
5031 interface(REG_INTER);
5032 %}
5033
5034 operand vRegD_V4()
5035 %{
5036 constraint(ALLOC_IN_RC(v4_reg));
5037 match(RegD);
5038 op_cost(0);
5039 format %{ %}
5040 interface(REG_INTER);
5041 %}
5042
5043 operand vRegD_V5()
5044 %{
5045 constraint(ALLOC_IN_RC(v5_reg));
5046 match(RegD);
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vRegD_V6()
5053 %{
5054 constraint(ALLOC_IN_RC(v6_reg));
5055 match(RegD);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vRegD_V7()
5062 %{
5063 constraint(ALLOC_IN_RC(v7_reg));
5064 match(RegD);
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vRegD_V12()
5071 %{
5072 constraint(ALLOC_IN_RC(v12_reg));
5073 match(RegD);
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vRegD_V13()
5080 %{
5081 constraint(ALLOC_IN_RC(v13_reg));
5082 match(RegD);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand pReg()
5089 %{
5090 constraint(ALLOC_IN_RC(pr_reg));
5091 match(RegVectMask);
5092 match(pRegGov);
5093 op_cost(0);
5094 format %{ %}
5095 interface(REG_INTER);
5096 %}
5097
5098 operand pRegGov()
5099 %{
5100 constraint(ALLOC_IN_RC(gov_pr));
5101 match(RegVectMask);
5102 match(pReg);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand pRegGov_P0()
5109 %{
5110 constraint(ALLOC_IN_RC(p0_reg));
5111 match(RegVectMask);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand pRegGov_P1()
5118 %{
5119 constraint(ALLOC_IN_RC(p1_reg));
5120 match(RegVectMask);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 // Flags register, used as output of signed compare instructions
5127
5128 // note that on AArch64 we also use this register as the output for
5129 // for floating point compare instructions (CmpF CmpD). this ensures
5130 // that ordered inequality tests use GT, GE, LT or LE none of which
5131 // pass through cases where the result is unordered i.e. one or both
5132 // inputs to the compare is a NaN. this means that the ideal code can
5133 // replace e.g. a GT with an LE and not end up capturing the NaN case
5134 // (where the comparison should always fail). EQ and NE tests are
5135 // always generated in ideal code so that unordered folds into the NE
5136 // case, matching the behaviour of AArch64 NE.
5137 //
5138 // This differs from x86 where the outputs of FP compares use a
5139 // special FP flags registers and where compares based on this
5140 // register are distinguished into ordered inequalities (cmpOpUCF) and
5141 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5142 // to explicitly handle the unordered case in branches. x86 also has
5143 // to include extra CMoveX rules to accept a cmpOpUCF input.
5144
5145 operand rFlagsReg()
5146 %{
5147 constraint(ALLOC_IN_RC(int_flags));
5148 match(RegFlags);
5149
5150 op_cost(0);
5151 format %{ "RFLAGS" %}
5152 interface(REG_INTER);
5153 %}
5154
5155 // Flags register, used as output of unsigned compare instructions
5156 operand rFlagsRegU()
5157 %{
5158 constraint(ALLOC_IN_RC(int_flags));
5159 match(RegFlags);
5160
5161 op_cost(0);
5162 format %{ "RFLAGSU" %}
5163 interface(REG_INTER);
5164 %}
5165
5166 // Special Registers
5167
5168 // Method Register
5169 operand inline_cache_RegP(iRegP reg)
5170 %{
5171 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5172 match(reg);
5173 match(iRegPNoSp);
5174 op_cost(0);
5175 format %{ %}
5176 interface(REG_INTER);
5177 %}
5178
5179 // Thread Register
5180 operand thread_RegP(iRegP reg)
5181 %{
5182 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5183 match(reg);
5184 op_cost(0);
5185 format %{ %}
5186 interface(REG_INTER);
5187 %}
5188
5189 //----------Memory Operands----------------------------------------------------
5190
5191 operand indirect(iRegP reg)
5192 %{
5193 constraint(ALLOC_IN_RC(ptr_reg));
5194 match(reg);
5195 op_cost(0);
5196 format %{ "[$reg]" %}
5197 interface(MEMORY_INTER) %{
5198 base($reg);
5199 index(0xffffffff);
5200 scale(0x0);
5201 disp(0x0);
5202 %}
5203 %}
5204
5205 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5206 %{
5207 constraint(ALLOC_IN_RC(ptr_reg));
5208 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5209 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5210 op_cost(0);
5211 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5212 interface(MEMORY_INTER) %{
5213 base($reg);
5214 index($ireg);
5215 scale($scale);
5216 disp(0x0);
5217 %}
5218 %}
5219
5220 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5221 %{
5222 constraint(ALLOC_IN_RC(ptr_reg));
5223 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5224 match(AddP reg (LShiftL lreg scale));
5225 op_cost(0);
5226 format %{ "$reg, $lreg lsl($scale)" %}
5227 interface(MEMORY_INTER) %{
5228 base($reg);
5229 index($lreg);
5230 scale($scale);
5231 disp(0x0);
5232 %}
5233 %}
5234
5235 operand indIndexI2L(iRegP reg, iRegI ireg)
5236 %{
5237 constraint(ALLOC_IN_RC(ptr_reg));
5238 match(AddP reg (ConvI2L ireg));
5239 op_cost(0);
5240 format %{ "$reg, $ireg, 0, I2L" %}
5241 interface(MEMORY_INTER) %{
5242 base($reg);
5243 index($ireg);
5244 scale(0x0);
5245 disp(0x0);
5246 %}
5247 %}
5248
5249 operand indIndex(iRegP reg, iRegL lreg)
5250 %{
5251 constraint(ALLOC_IN_RC(ptr_reg));
5252 match(AddP reg lreg);
5253 op_cost(0);
5254 format %{ "$reg, $lreg" %}
5255 interface(MEMORY_INTER) %{
5256 base($reg);
5257 index($lreg);
5258 scale(0x0);
5259 disp(0x0);
5260 %}
5261 %}
5262
5263 operand indOffI1(iRegP reg, immIOffset1 off)
5264 %{
5265 constraint(ALLOC_IN_RC(ptr_reg));
5266 match(AddP reg off);
5267 op_cost(0);
5268 format %{ "[$reg, $off]" %}
5269 interface(MEMORY_INTER) %{
5270 base($reg);
5271 index(0xffffffff);
5272 scale(0x0);
5273 disp($off);
5274 %}
5275 %}
5276
5277 operand indOffI2(iRegP reg, immIOffset2 off)
5278 %{
5279 constraint(ALLOC_IN_RC(ptr_reg));
5280 match(AddP reg off);
5281 op_cost(0);
5282 format %{ "[$reg, $off]" %}
5283 interface(MEMORY_INTER) %{
5284 base($reg);
5285 index(0xffffffff);
5286 scale(0x0);
5287 disp($off);
5288 %}
5289 %}
5290
5291 operand indOffI4(iRegP reg, immIOffset4 off)
5292 %{
5293 constraint(ALLOC_IN_RC(ptr_reg));
5294 match(AddP reg off);
5295 op_cost(0);
5296 format %{ "[$reg, $off]" %}
5297 interface(MEMORY_INTER) %{
5298 base($reg);
5299 index(0xffffffff);
5300 scale(0x0);
5301 disp($off);
5302 %}
5303 %}
5304
5305 operand indOffI8(iRegP reg, immIOffset8 off)
5306 %{
5307 constraint(ALLOC_IN_RC(ptr_reg));
5308 match(AddP reg off);
5309 op_cost(0);
5310 format %{ "[$reg, $off]" %}
5311 interface(MEMORY_INTER) %{
5312 base($reg);
5313 index(0xffffffff);
5314 scale(0x0);
5315 disp($off);
5316 %}
5317 %}
5318
5319 operand indOffI16(iRegP reg, immIOffset16 off)
5320 %{
5321 constraint(ALLOC_IN_RC(ptr_reg));
5322 match(AddP reg off);
5323 op_cost(0);
5324 format %{ "[$reg, $off]" %}
5325 interface(MEMORY_INTER) %{
5326 base($reg);
5327 index(0xffffffff);
5328 scale(0x0);
5329 disp($off);
5330 %}
5331 %}
5332
5333 operand indOffL1(iRegP reg, immLoffset1 off)
5334 %{
5335 constraint(ALLOC_IN_RC(ptr_reg));
5336 match(AddP reg off);
5337 op_cost(0);
5338 format %{ "[$reg, $off]" %}
5339 interface(MEMORY_INTER) %{
5340 base($reg);
5341 index(0xffffffff);
5342 scale(0x0);
5343 disp($off);
5344 %}
5345 %}
5346
5347 operand indOffL2(iRegP reg, immLoffset2 off)
5348 %{
5349 constraint(ALLOC_IN_RC(ptr_reg));
5350 match(AddP reg off);
5351 op_cost(0);
5352 format %{ "[$reg, $off]" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index(0xffffffff);
5356 scale(0x0);
5357 disp($off);
5358 %}
5359 %}
5360
5361 operand indOffL4(iRegP reg, immLoffset4 off)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 match(AddP reg off);
5365 op_cost(0);
5366 format %{ "[$reg, $off]" %}
5367 interface(MEMORY_INTER) %{
5368 base($reg);
5369 index(0xffffffff);
5370 scale(0x0);
5371 disp($off);
5372 %}
5373 %}
5374
5375 operand indOffL8(iRegP reg, immLoffset8 off)
5376 %{
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP reg off);
5379 op_cost(0);
5380 format %{ "[$reg, $off]" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index(0xffffffff);
5384 scale(0x0);
5385 disp($off);
5386 %}
5387 %}
5388
5389 operand indOffL16(iRegP reg, immLoffset16 off)
5390 %{
5391 constraint(ALLOC_IN_RC(ptr_reg));
5392 match(AddP reg off);
5393 op_cost(0);
5394 format %{ "[$reg, $off]" %}
5395 interface(MEMORY_INTER) %{
5396 base($reg);
5397 index(0xffffffff);
5398 scale(0x0);
5399 disp($off);
5400 %}
5401 %}
5402
5403 operand indirectX2P(iRegL reg)
5404 %{
5405 constraint(ALLOC_IN_RC(ptr_reg));
5406 match(CastX2P reg);
5407 op_cost(0);
5408 format %{ "[$reg]\t# long -> ptr" %}
5409 interface(MEMORY_INTER) %{
5410 base($reg);
5411 index(0xffffffff);
5412 scale(0x0);
5413 disp(0x0);
5414 %}
5415 %}
5416
5417 operand indOffX2P(iRegL reg, immLOffset off)
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(AddP (CastX2P reg) off);
5421 op_cost(0);
5422 format %{ "[$reg, $off]\t# long -> ptr" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp($off);
5428 %}
5429 %}
5430
5431 operand indirectN(iRegN reg)
5432 %{
5433 predicate(CompressedOops::shift() == 0);
5434 constraint(ALLOC_IN_RC(ptr_reg));
5435 match(DecodeN reg);
5436 op_cost(0);
5437 format %{ "[$reg]\t# narrow" %}
5438 interface(MEMORY_INTER) %{
5439 base($reg);
5440 index(0xffffffff);
5441 scale(0x0);
5442 disp(0x0);
5443 %}
5444 %}
5445
5446 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5447 %{
5448 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5449 constraint(ALLOC_IN_RC(ptr_reg));
5450 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5451 op_cost(0);
5452 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5453 interface(MEMORY_INTER) %{
5454 base($reg);
5455 index($ireg);
5456 scale($scale);
5457 disp(0x0);
5458 %}
5459 %}
5460
5461 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5462 %{
5463 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5464 constraint(ALLOC_IN_RC(ptr_reg));
5465 match(AddP (DecodeN reg) (LShiftL lreg scale));
5466 op_cost(0);
5467 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5468 interface(MEMORY_INTER) %{
5469 base($reg);
5470 index($lreg);
5471 scale($scale);
5472 disp(0x0);
5473 %}
5474 %}
5475
5476 operand indIndexI2LN(iRegN reg, iRegI ireg)
5477 %{
5478 predicate(CompressedOops::shift() == 0);
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(AddP (DecodeN reg) (ConvI2L ireg));
5481 op_cost(0);
5482 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index($ireg);
5486 scale(0x0);
5487 disp(0x0);
5488 %}
5489 %}
5490
5491 operand indIndexN(iRegN reg, iRegL lreg)
5492 %{
5493 predicate(CompressedOops::shift() == 0);
5494 constraint(ALLOC_IN_RC(ptr_reg));
5495 match(AddP (DecodeN reg) lreg);
5496 op_cost(0);
5497 format %{ "$reg, $lreg\t# narrow" %}
5498 interface(MEMORY_INTER) %{
5499 base($reg);
5500 index($lreg);
5501 scale(0x0);
5502 disp(0x0);
5503 %}
5504 %}
5505
5506 operand indOffIN(iRegN reg, immIOffset off)
5507 %{
5508 predicate(CompressedOops::shift() == 0);
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(AddP (DecodeN reg) off);
5511 op_cost(0);
5512 format %{ "[$reg, $off]\t# narrow" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index(0xffffffff);
5516 scale(0x0);
5517 disp($off);
5518 %}
5519 %}
5520
5521 operand indOffLN(iRegN reg, immLOffset off)
5522 %{
5523 predicate(CompressedOops::shift() == 0);
5524 constraint(ALLOC_IN_RC(ptr_reg));
5525 match(AddP (DecodeN reg) off);
5526 op_cost(0);
5527 format %{ "[$reg, $off]\t# narrow" %}
5528 interface(MEMORY_INTER) %{
5529 base($reg);
5530 index(0xffffffff);
5531 scale(0x0);
5532 disp($off);
5533 %}
5534 %}
5535
5536
5537 //----------Special Memory Operands--------------------------------------------
5538 // Stack Slot Operand - This operand is used for loading and storing temporary
5539 // values on the stack where a match requires a value to
5540 // flow through memory.
5541 operand stackSlotP(sRegP reg)
5542 %{
5543 constraint(ALLOC_IN_RC(stack_slots));
5544 op_cost(100);
5545 // No match rule because this operand is only generated in matching
5546 // match(RegP);
5547 format %{ "[$reg]" %}
5548 interface(MEMORY_INTER) %{
5549 base(0x1e); // RSP
5550 index(0x0); // No Index
5551 scale(0x0); // No Scale
5552 disp($reg); // Stack Offset
5553 %}
5554 %}
5555
5556 operand stackSlotI(sRegI reg)
5557 %{
5558 constraint(ALLOC_IN_RC(stack_slots));
5559 // No match rule because this operand is only generated in matching
5560 // match(RegI);
5561 format %{ "[$reg]" %}
5562 interface(MEMORY_INTER) %{
5563 base(0x1e); // RSP
5564 index(0x0); // No Index
5565 scale(0x0); // No Scale
5566 disp($reg); // Stack Offset
5567 %}
5568 %}
5569
5570 operand stackSlotF(sRegF reg)
5571 %{
5572 constraint(ALLOC_IN_RC(stack_slots));
5573 // No match rule because this operand is only generated in matching
5574 // match(RegF);
5575 format %{ "[$reg]" %}
5576 interface(MEMORY_INTER) %{
5577 base(0x1e); // RSP
5578 index(0x0); // No Index
5579 scale(0x0); // No Scale
5580 disp($reg); // Stack Offset
5581 %}
5582 %}
5583
5584 operand stackSlotD(sRegD reg)
5585 %{
5586 constraint(ALLOC_IN_RC(stack_slots));
5587 // No match rule because this operand is only generated in matching
5588 // match(RegD);
5589 format %{ "[$reg]" %}
5590 interface(MEMORY_INTER) %{
5591 base(0x1e); // RSP
5592 index(0x0); // No Index
5593 scale(0x0); // No Scale
5594 disp($reg); // Stack Offset
5595 %}
5596 %}
5597
5598 operand stackSlotL(sRegL reg)
5599 %{
5600 constraint(ALLOC_IN_RC(stack_slots));
5601 // No match rule because this operand is only generated in matching
5602 // match(RegL);
5603 format %{ "[$reg]" %}
5604 interface(MEMORY_INTER) %{
5605 base(0x1e); // RSP
5606 index(0x0); // No Index
5607 scale(0x0); // No Scale
5608 disp($reg); // Stack Offset
5609 %}
5610 %}
5611
5612 // Operands for expressing Control Flow
5613 // NOTE: Label is a predefined operand which should not be redefined in
5614 // the AD file. It is generically handled within the ADLC.
5615
5616 //----------Conditional Branch Operands----------------------------------------
5617 // Comparison Op - This is the operation of the comparison, and is limited to
5618 // the following set of codes:
5619 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5620 //
5621 // Other attributes of the comparison, such as unsignedness, are specified
5622 // by the comparison instruction that sets a condition code flags register.
5623 // That result is represented by a flags operand whose subtype is appropriate
5624 // to the unsignedness (etc.) of the comparison.
5625 //
5626 // Later, the instruction which matches both the Comparison Op (a Bool) and
5627 // the flags (produced by the Cmp) specifies the coding of the comparison op
5628 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5629
5630 // used for signed integral comparisons and fp comparisons
5631
5632 operand cmpOp()
5633 %{
5634 match(Bool);
5635
5636 format %{ "" %}
5637 interface(COND_INTER) %{
5638 equal(0x0, "eq");
5639 not_equal(0x1, "ne");
5640 less(0xb, "lt");
5641 greater_equal(0xa, "ge");
5642 less_equal(0xd, "le");
5643 greater(0xc, "gt");
5644 overflow(0x6, "vs");
5645 no_overflow(0x7, "vc");
5646 %}
5647 %}
5648
5649 // used for unsigned integral comparisons
5650
5651 operand cmpOpU()
5652 %{
5653 match(Bool);
5654
5655 format %{ "" %}
5656 interface(COND_INTER) %{
5657 equal(0x0, "eq");
5658 not_equal(0x1, "ne");
5659 less(0x3, "lo");
5660 greater_equal(0x2, "hs");
5661 less_equal(0x9, "ls");
5662 greater(0x8, "hi");
5663 overflow(0x6, "vs");
5664 no_overflow(0x7, "vc");
5665 %}
5666 %}
5667
5668 // used for certain integral comparisons which can be
5669 // converted to cbxx or tbxx instructions
5670
5671 operand cmpOpEqNe()
5672 %{
5673 match(Bool);
5674 op_cost(0);
5675 predicate(n->as_Bool()->_test._test == BoolTest::ne
5676 || n->as_Bool()->_test._test == BoolTest::eq);
5677
5678 format %{ "" %}
5679 interface(COND_INTER) %{
5680 equal(0x0, "eq");
5681 not_equal(0x1, "ne");
5682 less(0xb, "lt");
5683 greater_equal(0xa, "ge");
5684 less_equal(0xd, "le");
5685 greater(0xc, "gt");
5686 overflow(0x6, "vs");
5687 no_overflow(0x7, "vc");
5688 %}
5689 %}
5690
5691 // used for certain integral comparisons which can be
5692 // converted to cbxx or tbxx instructions
5693
5694 operand cmpOpLtGe()
5695 %{
5696 match(Bool);
5697 op_cost(0);
5698
5699 predicate(n->as_Bool()->_test._test == BoolTest::lt
5700 || n->as_Bool()->_test._test == BoolTest::ge);
5701
5702 format %{ "" %}
5703 interface(COND_INTER) %{
5704 equal(0x0, "eq");
5705 not_equal(0x1, "ne");
5706 less(0xb, "lt");
5707 greater_equal(0xa, "ge");
5708 less_equal(0xd, "le");
5709 greater(0xc, "gt");
5710 overflow(0x6, "vs");
5711 no_overflow(0x7, "vc");
5712 %}
5713 %}
5714
5715 // used for certain unsigned integral comparisons which can be
5716 // converted to cbxx or tbxx instructions
5717
5718 operand cmpOpUEqNeLeGt()
5719 %{
5720 match(Bool);
5721 op_cost(0);
5722
5723 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5724 n->as_Bool()->_test._test == BoolTest::ne ||
5725 n->as_Bool()->_test._test == BoolTest::le ||
5726 n->as_Bool()->_test._test == BoolTest::gt);
5727
5728 format %{ "" %}
5729 interface(COND_INTER) %{
5730 equal(0x0, "eq");
5731 not_equal(0x1, "ne");
5732 less(0x3, "lo");
5733 greater_equal(0x2, "hs");
5734 less_equal(0x9, "ls");
5735 greater(0x8, "hi");
5736 overflow(0x6, "vs");
5737 no_overflow(0x7, "vc");
5738 %}
5739 %}
5740
5741 // Special operand allowing long args to int ops to be truncated for free
5742
5743 operand iRegL2I(iRegL reg) %{
5744
5745 op_cost(0);
5746
5747 match(ConvL2I reg);
5748
5749 format %{ "l2i($reg)" %}
5750
5751 interface(REG_INTER)
5752 %}
5753
5754 operand iRegL2P(iRegL reg) %{
5755
5756 op_cost(0);
5757
5758 match(CastX2P reg);
5759
5760 format %{ "l2p($reg)" %}
5761
5762 interface(REG_INTER)
5763 %}
5764
5765 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5766 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5767 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5768 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5769
5770 //----------OPERAND CLASSES----------------------------------------------------
5771 // Operand Classes are groups of operands that are used as to simplify
5772 // instruction definitions by not requiring the AD writer to specify
5773 // separate instructions for every form of operand when the
5774 // instruction accepts multiple operand types with the same basic
5775 // encoding and format. The classic case of this is memory operands.
5776
5777 // memory is used to define read/write location for load/store
5778 // instruction defs. we can turn a memory op into an Address
5779
5780 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5781 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5782
5783 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5784 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5785
5786 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5787 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5788
5789 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5790 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5791
5792 // All of the memory operands. For the pipeline description.
5793 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5794 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5795 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5796
5797
5798 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5799 // operations. it allows the src to be either an iRegI or a (ConvL2I
5800 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5801 // can be elided because the 32-bit instruction will just employ the
5802 // lower 32 bits anyway.
5803 //
5804 // n.b. this does not elide all L2I conversions. if the truncated
5805 // value is consumed by more than one operation then the ConvL2I
5806 // cannot be bundled into the consuming nodes so an l2i gets planted
5807 // (actually a movw $dst $src) and the downstream instructions consume
5808 // the result of the l2i as an iRegI input. That's a shame since the
5809 // movw is actually redundant but its not too costly.
5810
5811 opclass iRegIorL2I(iRegI, iRegL2I);
5812 opclass iRegPorL2P(iRegP, iRegL2P);
5813
5814 //----------PIPELINE-----------------------------------------------------------
5815 // Rules which define the behavior of the target architectures pipeline.
5816
5817 // For specific pipelines, eg A53, define the stages of that pipeline
5818 //pipe_desc(ISS, EX1, EX2, WR);
5819 #define ISS S0
5820 #define EX1 S1
5821 #define EX2 S2
5822 #define WR S3
5823
5824 // Integer ALU reg operation
5825 pipeline %{
5826
5827 attributes %{
5828 // ARM instructions are of fixed length
5829 fixed_size_instructions; // Fixed size instructions TODO does
5830 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5831 // ARM instructions come in 32-bit word units
5832 instruction_unit_size = 4; // An instruction is 4 bytes long
5833 instruction_fetch_unit_size = 64; // The processor fetches one line
5834 instruction_fetch_units = 1; // of 64 bytes
5835
5836 // List of nop instructions
5837 nops( MachNop );
5838 %}
5839
5840 // We don't use an actual pipeline model so don't care about resources
5841 // or description. we do use pipeline classes to introduce fixed
5842 // latencies
5843
5844 //----------RESOURCES----------------------------------------------------------
5845 // Resources are the functional units available to the machine
5846
5847 resources( INS0, INS1, INS01 = INS0 | INS1,
5848 ALU0, ALU1, ALU = ALU0 | ALU1,
5849 MAC,
5850 DIV,
5851 BRANCH,
5852 LDST,
5853 NEON_FP);
5854
5855 //----------PIPELINE DESCRIPTION-----------------------------------------------
5856 // Pipeline Description specifies the stages in the machine's pipeline
5857
5858 // Define the pipeline as a generic 6 stage pipeline
5859 pipe_desc(S0, S1, S2, S3, S4, S5);
5860
5861 //----------PIPELINE CLASSES---------------------------------------------------
5862 // Pipeline Classes describe the stages in which input and output are
5863 // referenced by the hardware pipeline.
5864
5865 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5866 %{
5867 single_instruction;
5868 src1 : S1(read);
5869 src2 : S2(read);
5870 dst : S5(write);
5871 INS01 : ISS;
5872 NEON_FP : S5;
5873 %}
5874
5875 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5876 %{
5877 single_instruction;
5878 src1 : S1(read);
5879 src2 : S2(read);
5880 dst : S5(write);
5881 INS01 : ISS;
5882 NEON_FP : S5;
5883 %}
5884
5885 pipe_class fp_uop_s(vRegF dst, vRegF src)
5886 %{
5887 single_instruction;
5888 src : S1(read);
5889 dst : S5(write);
5890 INS01 : ISS;
5891 NEON_FP : S5;
5892 %}
5893
5894 pipe_class fp_uop_d(vRegD dst, vRegD src)
5895 %{
5896 single_instruction;
5897 src : S1(read);
5898 dst : S5(write);
5899 INS01 : ISS;
5900 NEON_FP : S5;
5901 %}
5902
5903 pipe_class fp_d2f(vRegF dst, vRegD src)
5904 %{
5905 single_instruction;
5906 src : S1(read);
5907 dst : S5(write);
5908 INS01 : ISS;
5909 NEON_FP : S5;
5910 %}
5911
5912 pipe_class fp_f2d(vRegD dst, vRegF src)
5913 %{
5914 single_instruction;
5915 src : S1(read);
5916 dst : S5(write);
5917 INS01 : ISS;
5918 NEON_FP : S5;
5919 %}
5920
5921 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5922 %{
5923 single_instruction;
5924 src : S1(read);
5925 dst : S5(write);
5926 INS01 : ISS;
5927 NEON_FP : S5;
5928 %}
5929
5930 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5931 %{
5932 single_instruction;
5933 src : S1(read);
5934 dst : S5(write);
5935 INS01 : ISS;
5936 NEON_FP : S5;
5937 %}
5938
5939 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5940 %{
5941 single_instruction;
5942 src : S1(read);
5943 dst : S5(write);
5944 INS01 : ISS;
5945 NEON_FP : S5;
5946 %}
5947
5948 pipe_class fp_l2f(vRegF dst, iRegL src)
5949 %{
5950 single_instruction;
5951 src : S1(read);
5952 dst : S5(write);
5953 INS01 : ISS;
5954 NEON_FP : S5;
5955 %}
5956
5957 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5958 %{
5959 single_instruction;
5960 src : S1(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5994 %{
5995 single_instruction;
5996 src1 : S1(read);
5997 src2 : S2(read);
5998 dst : S5(write);
5999 INS0 : ISS;
6000 NEON_FP : S5;
6001 %}
6002
6003 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6004 %{
6005 single_instruction;
6006 src1 : S1(read);
6007 src2 : S2(read);
6008 dst : S5(write);
6009 INS0 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6014 %{
6015 single_instruction;
6016 cr : S1(read);
6017 src1 : S1(read);
6018 src2 : S1(read);
6019 dst : S3(write);
6020 INS01 : ISS;
6021 NEON_FP : S3;
6022 %}
6023
6024 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6025 %{
6026 single_instruction;
6027 cr : S1(read);
6028 src1 : S1(read);
6029 src2 : S1(read);
6030 dst : S3(write);
6031 INS01 : ISS;
6032 NEON_FP : S3;
6033 %}
6034
6035 pipe_class fp_imm_s(vRegF dst)
6036 %{
6037 single_instruction;
6038 dst : S3(write);
6039 INS01 : ISS;
6040 NEON_FP : S3;
6041 %}
6042
6043 pipe_class fp_imm_d(vRegD dst)
6044 %{
6045 single_instruction;
6046 dst : S3(write);
6047 INS01 : ISS;
6048 NEON_FP : S3;
6049 %}
6050
6051 pipe_class fp_load_constant_s(vRegF dst)
6052 %{
6053 single_instruction;
6054 dst : S4(write);
6055 INS01 : ISS;
6056 NEON_FP : S4;
6057 %}
6058
6059 pipe_class fp_load_constant_d(vRegD dst)
6060 %{
6061 single_instruction;
6062 dst : S4(write);
6063 INS01 : ISS;
6064 NEON_FP : S4;
6065 %}
6066
6067 //------- Integer ALU operations --------------------------
6068
6069 // Integer ALU reg-reg operation
6070 // Operands needed in EX1, result generated in EX2
6071 // Eg. ADD x0, x1, x2
6072 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6073 %{
6074 single_instruction;
6075 dst : EX2(write);
6076 src1 : EX1(read);
6077 src2 : EX1(read);
6078 INS01 : ISS; // Dual issue as instruction 0 or 1
6079 ALU : EX2;
6080 %}
6081
6082 // Integer ALU reg-reg operation with constant shift
6083 // Shifted register must be available in LATE_ISS instead of EX1
6084 // Eg. ADD x0, x1, x2, LSL #2
6085 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6086 %{
6087 single_instruction;
6088 dst : EX2(write);
6089 src1 : EX1(read);
6090 src2 : ISS(read);
6091 INS01 : ISS;
6092 ALU : EX2;
6093 %}
6094
6095 // Integer ALU reg operation with constant shift
6096 // Eg. LSL x0, x1, #shift
6097 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6098 %{
6099 single_instruction;
6100 dst : EX2(write);
6101 src1 : ISS(read);
6102 INS01 : ISS;
6103 ALU : EX2;
6104 %}
6105
6106 // Integer ALU reg-reg operation with variable shift
6107 // Both operands must be available in LATE_ISS instead of EX1
6108 // Result is available in EX1 instead of EX2
6109 // Eg. LSLV x0, x1, x2
6110 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6111 %{
6112 single_instruction;
6113 dst : EX1(write);
6114 src1 : ISS(read);
6115 src2 : ISS(read);
6116 INS01 : ISS;
6117 ALU : EX1;
6118 %}
6119
6120 // Integer ALU reg-reg operation with extract
6121 // As for _vshift above, but result generated in EX2
6122 // Eg. EXTR x0, x1, x2, #N
6123 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6124 %{
6125 single_instruction;
6126 dst : EX2(write);
6127 src1 : ISS(read);
6128 src2 : ISS(read);
6129 INS1 : ISS; // Can only dual issue as Instruction 1
6130 ALU : EX1;
6131 %}
6132
6133 // Integer ALU reg operation
6134 // Eg. NEG x0, x1
6135 pipe_class ialu_reg(iRegI dst, iRegI src)
6136 %{
6137 single_instruction;
6138 dst : EX2(write);
6139 src : EX1(read);
6140 INS01 : ISS;
6141 ALU : EX2;
6142 %}
6143
6144 // Integer ALU reg mmediate operation
6145 // Eg. ADD x0, x1, #N
6146 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6147 %{
6148 single_instruction;
6149 dst : EX2(write);
6150 src1 : EX1(read);
6151 INS01 : ISS;
6152 ALU : EX2;
6153 %}
6154
6155 // Integer ALU immediate operation (no source operands)
6156 // Eg. MOV x0, #N
6157 pipe_class ialu_imm(iRegI dst)
6158 %{
6159 single_instruction;
6160 dst : EX1(write);
6161 INS01 : ISS;
6162 ALU : EX1;
6163 %}
6164
6165 //------- Compare operation -------------------------------
6166
6167 // Compare reg-reg
6168 // Eg. CMP x0, x1
6169 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6170 %{
6171 single_instruction;
6172 // fixed_latency(16);
6173 cr : EX2(write);
6174 op1 : EX1(read);
6175 op2 : EX1(read);
6176 INS01 : ISS;
6177 ALU : EX2;
6178 %}
6179
6180 // Compare reg-reg
6181 // Eg. CMP x0, #N
6182 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6183 %{
6184 single_instruction;
6185 // fixed_latency(16);
6186 cr : EX2(write);
6187 op1 : EX1(read);
6188 INS01 : ISS;
6189 ALU : EX2;
6190 %}
6191
6192 //------- Conditional instructions ------------------------
6193
6194 // Conditional no operands
6195 // Eg. CSINC x0, zr, zr, <cond>
6196 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6197 %{
6198 single_instruction;
6199 cr : EX1(read);
6200 dst : EX2(write);
6201 INS01 : ISS;
6202 ALU : EX2;
6203 %}
6204
6205 // Conditional 2 operand
6206 // EG. CSEL X0, X1, X2, <cond>
6207 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6208 %{
6209 single_instruction;
6210 cr : EX1(read);
6211 src1 : EX1(read);
6212 src2 : EX1(read);
6213 dst : EX2(write);
6214 INS01 : ISS;
6215 ALU : EX2;
6216 %}
6217
6218 // Conditional 2 operand
6219 // EG. CSEL X0, X1, X2, <cond>
6220 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6221 %{
6222 single_instruction;
6223 cr : EX1(read);
6224 src : EX1(read);
6225 dst : EX2(write);
6226 INS01 : ISS;
6227 ALU : EX2;
6228 %}
6229
6230 //------- Multiply pipeline operations --------------------
6231
6232 // Multiply reg-reg
6233 // Eg. MUL w0, w1, w2
6234 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6235 %{
6236 single_instruction;
6237 dst : WR(write);
6238 src1 : ISS(read);
6239 src2 : ISS(read);
6240 INS01 : ISS;
6241 MAC : WR;
6242 %}
6243
6244 // Multiply accumulate
6245 // Eg. MADD w0, w1, w2, w3
6246 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6247 %{
6248 single_instruction;
6249 dst : WR(write);
6250 src1 : ISS(read);
6251 src2 : ISS(read);
6252 src3 : ISS(read);
6253 INS01 : ISS;
6254 MAC : WR;
6255 %}
6256
6257 // Eg. MUL w0, w1, w2
6258 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6259 %{
6260 single_instruction;
6261 fixed_latency(3); // Maximum latency for 64 bit mul
6262 dst : WR(write);
6263 src1 : ISS(read);
6264 src2 : ISS(read);
6265 INS01 : ISS;
6266 MAC : WR;
6267 %}
6268
6269 // Multiply accumulate
6270 // Eg. MADD w0, w1, w2, w3
6271 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6272 %{
6273 single_instruction;
6274 fixed_latency(3); // Maximum latency for 64 bit mul
6275 dst : WR(write);
6276 src1 : ISS(read);
6277 src2 : ISS(read);
6278 src3 : ISS(read);
6279 INS01 : ISS;
6280 MAC : WR;
6281 %}
6282
6283 //------- Divide pipeline operations --------------------
6284
6285 // Eg. SDIV w0, w1, w2
6286 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6287 %{
6288 single_instruction;
6289 fixed_latency(8); // Maximum latency for 32 bit divide
6290 dst : WR(write);
6291 src1 : ISS(read);
6292 src2 : ISS(read);
6293 INS0 : ISS; // Can only dual issue as instruction 0
6294 DIV : WR;
6295 %}
6296
6297 // Eg. SDIV x0, x1, x2
6298 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6299 %{
6300 single_instruction;
6301 fixed_latency(16); // Maximum latency for 64 bit divide
6302 dst : WR(write);
6303 src1 : ISS(read);
6304 src2 : ISS(read);
6305 INS0 : ISS; // Can only dual issue as instruction 0
6306 DIV : WR;
6307 %}
6308
6309 //------- Load pipeline operations ------------------------
6310
6311 // Load - prefetch
6312 // Eg. PFRM <mem>
6313 pipe_class iload_prefetch(memory mem)
6314 %{
6315 single_instruction;
6316 mem : ISS(read);
6317 INS01 : ISS;
6318 LDST : WR;
6319 %}
6320
6321 // Load - reg, mem
6322 // Eg. LDR x0, <mem>
6323 pipe_class iload_reg_mem(iRegI dst, memory mem)
6324 %{
6325 single_instruction;
6326 dst : WR(write);
6327 mem : ISS(read);
6328 INS01 : ISS;
6329 LDST : WR;
6330 %}
6331
6332 // Load - reg, reg
6333 // Eg. LDR x0, [sp, x1]
6334 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6335 %{
6336 single_instruction;
6337 dst : WR(write);
6338 src : ISS(read);
6339 INS01 : ISS;
6340 LDST : WR;
6341 %}
6342
6343 //------- Store pipeline operations -----------------------
6344
6345 // Store - zr, mem
6346 // Eg. STR zr, <mem>
6347 pipe_class istore_mem(memory mem)
6348 %{
6349 single_instruction;
6350 mem : ISS(read);
6351 INS01 : ISS;
6352 LDST : WR;
6353 %}
6354
6355 // Store - reg, mem
6356 // Eg. STR x0, <mem>
6357 pipe_class istore_reg_mem(iRegI src, memory mem)
6358 %{
6359 single_instruction;
6360 mem : ISS(read);
6361 src : EX2(read);
6362 INS01 : ISS;
6363 LDST : WR;
6364 %}
6365
6366 // Store - reg, reg
6367 // Eg. STR x0, [sp, x1]
6368 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6369 %{
6370 single_instruction;
6371 dst : ISS(read);
6372 src : EX2(read);
6373 INS01 : ISS;
6374 LDST : WR;
6375 %}
6376
6377 //------- Store pipeline operations -----------------------
6378
6379 // Branch
6380 pipe_class pipe_branch()
6381 %{
6382 single_instruction;
6383 INS01 : ISS;
6384 BRANCH : EX1;
6385 %}
6386
6387 // Conditional branch
6388 pipe_class pipe_branch_cond(rFlagsReg cr)
6389 %{
6390 single_instruction;
6391 cr : EX1(read);
6392 INS01 : ISS;
6393 BRANCH : EX1;
6394 %}
6395
6396 // Compare & Branch
6397 // EG. CBZ/CBNZ
6398 pipe_class pipe_cmp_branch(iRegI op1)
6399 %{
6400 single_instruction;
6401 op1 : EX1(read);
6402 INS01 : ISS;
6403 BRANCH : EX1;
6404 %}
6405
6406 //------- Synchronisation operations ----------------------
6407
6408 // Any operation requiring serialization.
6409 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6410 pipe_class pipe_serial()
6411 %{
6412 single_instruction;
6413 force_serialization;
6414 fixed_latency(16);
6415 INS01 : ISS(2); // Cannot dual issue with any other instruction
6416 LDST : WR;
6417 %}
6418
6419 // Generic big/slow expanded idiom - also serialized
6420 pipe_class pipe_slow()
6421 %{
6422 instruction_count(10);
6423 multiple_bundles;
6424 force_serialization;
6425 fixed_latency(16);
6426 INS01 : ISS(2); // Cannot dual issue with any other instruction
6427 LDST : WR;
6428 %}
6429
6430 // Empty pipeline class
6431 pipe_class pipe_class_empty()
6432 %{
6433 single_instruction;
6434 fixed_latency(0);
6435 %}
6436
6437 // Default pipeline class.
6438 pipe_class pipe_class_default()
6439 %{
6440 single_instruction;
6441 fixed_latency(2);
6442 %}
6443
6444 // Pipeline class for compares.
6445 pipe_class pipe_class_compare()
6446 %{
6447 single_instruction;
6448 fixed_latency(16);
6449 %}
6450
6451 // Pipeline class for memory operations.
6452 pipe_class pipe_class_memory()
6453 %{
6454 single_instruction;
6455 fixed_latency(16);
6456 %}
6457
6458 // Pipeline class for call.
6459 pipe_class pipe_class_call()
6460 %{
6461 single_instruction;
6462 fixed_latency(100);
6463 %}
6464
6465 // Define the class for the Nop node.
6466 define %{
6467 MachNop = pipe_class_empty;
6468 %}
6469
6470 %}
6471 //----------INSTRUCTIONS-------------------------------------------------------
6472 //
6473 // match -- States which machine-independent subtree may be replaced
6474 // by this instruction.
6475 // ins_cost -- The estimated cost of this instruction is used by instruction
6476 // selection to identify a minimum cost tree of machine
6477 // instructions that matches a tree of machine-independent
6478 // instructions.
6479 // format -- A string providing the disassembly for this instruction.
6480 // The value of an instruction's operand may be inserted
6481 // by referring to it with a '$' prefix.
6482 // opcode -- Three instruction opcodes may be provided. These are referred
6483 // to within an encode class as $primary, $secondary, and $tertiary
6484 // rrspectively. The primary opcode is commonly used to
6485 // indicate the type of machine instruction, while secondary
6486 // and tertiary are often used for prefix options or addressing
6487 // modes.
6488 // ins_encode -- A list of encode classes with parameters. The encode class
6489 // name must have been defined in an 'enc_class' specification
6490 // in the encode section of the architecture description.
6491
6492 // ============================================================================
6493 // Memory (Load/Store) Instructions
6494
6495 // Load Instructions
6496
6497 // Load Byte (8 bit signed)
6498 instruct loadB(iRegINoSp dst, memory1 mem)
6499 %{
6500 match(Set dst (LoadB mem));
6501 predicate(!needs_acquiring_load(n));
6502
6503 ins_cost(4 * INSN_COST);
6504 format %{ "ldrsbw $dst, $mem\t# byte" %}
6505
6506 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6507
6508 ins_pipe(iload_reg_mem);
6509 %}
6510
6511 // Load Byte (8 bit signed) into long
6512 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6513 %{
6514 match(Set dst (ConvI2L (LoadB mem)));
6515 predicate(!needs_acquiring_load(n->in(1)));
6516
6517 ins_cost(4 * INSN_COST);
6518 format %{ "ldrsb $dst, $mem\t# byte" %}
6519
6520 ins_encode(aarch64_enc_ldrsb(dst, mem));
6521
6522 ins_pipe(iload_reg_mem);
6523 %}
6524
6525 // Load Byte (8 bit unsigned)
6526 instruct loadUB(iRegINoSp dst, memory1 mem)
6527 %{
6528 match(Set dst (LoadUB mem));
6529 predicate(!needs_acquiring_load(n));
6530
6531 ins_cost(4 * INSN_COST);
6532 format %{ "ldrbw $dst, $mem\t# byte" %}
6533
6534 ins_encode(aarch64_enc_ldrb(dst, mem));
6535
6536 ins_pipe(iload_reg_mem);
6537 %}
6538
6539 // Load Byte (8 bit unsigned) into long
6540 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6541 %{
6542 match(Set dst (ConvI2L (LoadUB mem)));
6543 predicate(!needs_acquiring_load(n->in(1)));
6544
6545 ins_cost(4 * INSN_COST);
6546 format %{ "ldrb $dst, $mem\t# byte" %}
6547
6548 ins_encode(aarch64_enc_ldrb(dst, mem));
6549
6550 ins_pipe(iload_reg_mem);
6551 %}
6552
6553 // Load Short (16 bit signed)
6554 instruct loadS(iRegINoSp dst, memory2 mem)
6555 %{
6556 match(Set dst (LoadS mem));
6557 predicate(!needs_acquiring_load(n));
6558
6559 ins_cost(4 * INSN_COST);
6560 format %{ "ldrshw $dst, $mem\t# short" %}
6561
6562 ins_encode(aarch64_enc_ldrshw(dst, mem));
6563
6564 ins_pipe(iload_reg_mem);
6565 %}
6566
6567 // Load Short (16 bit signed) into long
6568 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6569 %{
6570 match(Set dst (ConvI2L (LoadS mem)));
6571 predicate(!needs_acquiring_load(n->in(1)));
6572
6573 ins_cost(4 * INSN_COST);
6574 format %{ "ldrsh $dst, $mem\t# short" %}
6575
6576 ins_encode(aarch64_enc_ldrsh(dst, mem));
6577
6578 ins_pipe(iload_reg_mem);
6579 %}
6580
6581 // Load Char (16 bit unsigned)
6582 instruct loadUS(iRegINoSp dst, memory2 mem)
6583 %{
6584 match(Set dst (LoadUS mem));
6585 predicate(!needs_acquiring_load(n));
6586
6587 ins_cost(4 * INSN_COST);
6588 format %{ "ldrh $dst, $mem\t# short" %}
6589
6590 ins_encode(aarch64_enc_ldrh(dst, mem));
6591
6592 ins_pipe(iload_reg_mem);
6593 %}
6594
6595 // Load Short/Char (16 bit unsigned) into long
6596 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6597 %{
6598 match(Set dst (ConvI2L (LoadUS mem)));
6599 predicate(!needs_acquiring_load(n->in(1)));
6600
6601 ins_cost(4 * INSN_COST);
6602 format %{ "ldrh $dst, $mem\t# short" %}
6603
6604 ins_encode(aarch64_enc_ldrh(dst, mem));
6605
6606 ins_pipe(iload_reg_mem);
6607 %}
6608
6609 // Load Integer (32 bit signed)
6610 instruct loadI(iRegINoSp dst, memory4 mem)
6611 %{
6612 match(Set dst (LoadI mem));
6613 predicate(!needs_acquiring_load(n));
6614
6615 ins_cost(4 * INSN_COST);
6616 format %{ "ldrw $dst, $mem\t# int" %}
6617
6618 ins_encode(aarch64_enc_ldrw(dst, mem));
6619
6620 ins_pipe(iload_reg_mem);
6621 %}
6622
6623 // Load Integer (32 bit signed) into long
6624 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6625 %{
6626 match(Set dst (ConvI2L (LoadI mem)));
6627 predicate(!needs_acquiring_load(n->in(1)));
6628
6629 ins_cost(4 * INSN_COST);
6630 format %{ "ldrsw $dst, $mem\t# int" %}
6631
6632 ins_encode(aarch64_enc_ldrsw(dst, mem));
6633
6634 ins_pipe(iload_reg_mem);
6635 %}
6636
6637 // Load Integer (32 bit unsigned) into long
6638 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6639 %{
6640 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6641 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6642
6643 ins_cost(4 * INSN_COST);
6644 format %{ "ldrw $dst, $mem\t# int" %}
6645
6646 ins_encode(aarch64_enc_ldrw(dst, mem));
6647
6648 ins_pipe(iload_reg_mem);
6649 %}
6650
6651 // Load Long (64 bit signed)
6652 instruct loadL(iRegLNoSp dst, memory8 mem)
6653 %{
6654 match(Set dst (LoadL mem));
6655 predicate(!needs_acquiring_load(n));
6656
6657 ins_cost(4 * INSN_COST);
6658 format %{ "ldr $dst, $mem\t# int" %}
6659
6660 ins_encode(aarch64_enc_ldr(dst, mem));
6661
6662 ins_pipe(iload_reg_mem);
6663 %}
6664
6665 // Load Range
6666 instruct loadRange(iRegINoSp dst, memory4 mem)
6667 %{
6668 match(Set dst (LoadRange mem));
6669
6670 ins_cost(4 * INSN_COST);
6671 format %{ "ldrw $dst, $mem\t# range" %}
6672
6673 ins_encode(aarch64_enc_ldrw(dst, mem));
6674
6675 ins_pipe(iload_reg_mem);
6676 %}
6677
6678 // Load Pointer
6679 instruct loadP(iRegPNoSp dst, memory8 mem)
6680 %{
6681 match(Set dst (LoadP mem));
6682 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6683
6684 ins_cost(4 * INSN_COST);
6685 format %{ "ldr $dst, $mem\t# ptr" %}
6686
6687 ins_encode(aarch64_enc_ldr(dst, mem));
6688
6689 ins_pipe(iload_reg_mem);
6690 %}
6691
6692 // Load Compressed Pointer
6693 instruct loadN(iRegNNoSp dst, memory4 mem)
6694 %{
6695 match(Set dst (LoadN mem));
6696 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6697
6698 ins_cost(4 * INSN_COST);
6699 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6700
6701 ins_encode(aarch64_enc_ldrw(dst, mem));
6702
6703 ins_pipe(iload_reg_mem);
6704 %}
6705
6706 // Load Klass Pointer
6707 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6708 %{
6709 match(Set dst (LoadKlass mem));
6710 predicate(!needs_acquiring_load(n));
6711
6712 ins_cost(4 * INSN_COST);
6713 format %{ "ldr $dst, $mem\t# class" %}
6714
6715 ins_encode(aarch64_enc_ldr(dst, mem));
6716
6717 ins_pipe(iload_reg_mem);
6718 %}
6719
6720 // Load Narrow Klass Pointer
6721 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6722 %{
6723 match(Set dst (LoadNKlass mem));
6724 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6725
6726 ins_cost(4 * INSN_COST);
6727 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6728
6729 ins_encode(aarch64_enc_ldrw(dst, mem));
6730
6731 ins_pipe(iload_reg_mem);
6732 %}
6733
6734 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6735 %{
6736 match(Set dst (LoadNKlass mem));
6737 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6738
6739 ins_cost(4 * INSN_COST);
6740 format %{
6741 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6742 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6743 %}
6744 ins_encode %{
6745 // inlined aarch64_enc_ldrw
6746 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6747 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6748 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6749 %}
6750 ins_pipe(iload_reg_mem);
6751 %}
6752
6753 // Load Float
6754 instruct loadF(vRegF dst, memory4 mem)
6755 %{
6756 match(Set dst (LoadF mem));
6757 predicate(!needs_acquiring_load(n));
6758
6759 ins_cost(4 * INSN_COST);
6760 format %{ "ldrs $dst, $mem\t# float" %}
6761
6762 ins_encode( aarch64_enc_ldrs(dst, mem) );
6763
6764 ins_pipe(pipe_class_memory);
6765 %}
6766
6767 // Load Double
6768 instruct loadD(vRegD dst, memory8 mem)
6769 %{
6770 match(Set dst (LoadD mem));
6771 predicate(!needs_acquiring_load(n));
6772
6773 ins_cost(4 * INSN_COST);
6774 format %{ "ldrd $dst, $mem\t# double" %}
6775
6776 ins_encode( aarch64_enc_ldrd(dst, mem) );
6777
6778 ins_pipe(pipe_class_memory);
6779 %}
6780
6781
6782 // Load Int Constant
6783 instruct loadConI(iRegINoSp dst, immI src)
6784 %{
6785 match(Set dst src);
6786
6787 ins_cost(INSN_COST);
6788 format %{ "mov $dst, $src\t# int" %}
6789
6790 ins_encode( aarch64_enc_movw_imm(dst, src) );
6791
6792 ins_pipe(ialu_imm);
6793 %}
6794
6795 // Load Long Constant
6796 instruct loadConL(iRegLNoSp dst, immL src)
6797 %{
6798 match(Set dst src);
6799
6800 ins_cost(INSN_COST);
6801 format %{ "mov $dst, $src\t# long" %}
6802
6803 ins_encode( aarch64_enc_mov_imm(dst, src) );
6804
6805 ins_pipe(ialu_imm);
6806 %}
6807
6808 // Load Pointer Constant
6809
6810 instruct loadConP(iRegPNoSp dst, immP con)
6811 %{
6812 match(Set dst con);
6813
6814 ins_cost(INSN_COST * 4);
6815 format %{
6816 "mov $dst, $con\t# ptr"
6817 %}
6818
6819 ins_encode(aarch64_enc_mov_p(dst, con));
6820
6821 ins_pipe(ialu_imm);
6822 %}
6823
6824 // Load Null Pointer Constant
6825
6826 instruct loadConP0(iRegPNoSp dst, immP0 con)
6827 %{
6828 match(Set dst con);
6829
6830 ins_cost(INSN_COST);
6831 format %{ "mov $dst, $con\t# nullptr ptr" %}
6832
6833 ins_encode(aarch64_enc_mov_p0(dst, con));
6834
6835 ins_pipe(ialu_imm);
6836 %}
6837
6838 // Load Pointer Constant One
6839
6840 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6841 %{
6842 match(Set dst con);
6843
6844 ins_cost(INSN_COST);
6845 format %{ "mov $dst, $con\t# nullptr ptr" %}
6846
6847 ins_encode(aarch64_enc_mov_p1(dst, con));
6848
6849 ins_pipe(ialu_imm);
6850 %}
6851
6852 // Load Byte Map Base Constant
6853
6854 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6855 %{
6856 match(Set dst con);
6857
6858 ins_cost(INSN_COST);
6859 format %{ "adr $dst, $con\t# Byte Map Base" %}
6860
6861 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6862
6863 ins_pipe(ialu_imm);
6864 %}
6865
6866 // Load Narrow Pointer Constant
6867
6868 instruct loadConN(iRegNNoSp dst, immN con)
6869 %{
6870 match(Set dst con);
6871
6872 ins_cost(INSN_COST * 4);
6873 format %{ "mov $dst, $con\t# compressed ptr" %}
6874
6875 ins_encode(aarch64_enc_mov_n(dst, con));
6876
6877 ins_pipe(ialu_imm);
6878 %}
6879
6880 // Load Narrow Null Pointer Constant
6881
6882 instruct loadConN0(iRegNNoSp dst, immN0 con)
6883 %{
6884 match(Set dst con);
6885
6886 ins_cost(INSN_COST);
6887 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6888
6889 ins_encode(aarch64_enc_mov_n0(dst, con));
6890
6891 ins_pipe(ialu_imm);
6892 %}
6893
6894 // Load Narrow Klass Constant
6895
6896 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6897 %{
6898 match(Set dst con);
6899
6900 ins_cost(INSN_COST);
6901 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6902
6903 ins_encode(aarch64_enc_mov_nk(dst, con));
6904
6905 ins_pipe(ialu_imm);
6906 %}
6907
6908 // Load Packed Float Constant
6909
6910 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6911 match(Set dst con);
6912 ins_cost(INSN_COST * 4);
6913 format %{ "fmovs $dst, $con"%}
6914 ins_encode %{
6915 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6916 %}
6917
6918 ins_pipe(fp_imm_s);
6919 %}
6920
6921 // Load Float Constant
6922
6923 instruct loadConF(vRegF dst, immF con) %{
6924 match(Set dst con);
6925
6926 ins_cost(INSN_COST * 4);
6927
6928 format %{
6929 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6930 %}
6931
6932 ins_encode %{
6933 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6934 %}
6935
6936 ins_pipe(fp_load_constant_s);
6937 %}
6938
6939 // Load Packed Double Constant
6940
6941 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6942 match(Set dst con);
6943 ins_cost(INSN_COST);
6944 format %{ "fmovd $dst, $con"%}
6945 ins_encode %{
6946 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6947 %}
6948
6949 ins_pipe(fp_imm_d);
6950 %}
6951
6952 // Load Double Constant
6953
6954 instruct loadConD(vRegD dst, immD con) %{
6955 match(Set dst con);
6956
6957 ins_cost(INSN_COST * 5);
6958 format %{
6959 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6960 %}
6961
6962 ins_encode %{
6963 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6964 %}
6965
6966 ins_pipe(fp_load_constant_d);
6967 %}
6968
6969 // Store Instructions
6970
6971 // Store Byte
6972 instruct storeB(iRegIorL2I src, memory1 mem)
6973 %{
6974 match(Set mem (StoreB mem src));
6975 predicate(!needs_releasing_store(n));
6976
6977 ins_cost(INSN_COST);
6978 format %{ "strb $src, $mem\t# byte" %}
6979
6980 ins_encode(aarch64_enc_strb(src, mem));
6981
6982 ins_pipe(istore_reg_mem);
6983 %}
6984
6985
6986 instruct storeimmB0(immI0 zero, memory1 mem)
6987 %{
6988 match(Set mem (StoreB mem zero));
6989 predicate(!needs_releasing_store(n));
6990
6991 ins_cost(INSN_COST);
6992 format %{ "strb rscractch2, $mem\t# byte" %}
6993
6994 ins_encode(aarch64_enc_strb0(mem));
6995
6996 ins_pipe(istore_mem);
6997 %}
6998
6999 // Store Char/Short
7000 instruct storeC(iRegIorL2I src, memory2 mem)
7001 %{
7002 match(Set mem (StoreC mem src));
7003 predicate(!needs_releasing_store(n));
7004
7005 ins_cost(INSN_COST);
7006 format %{ "strh $src, $mem\t# short" %}
7007
7008 ins_encode(aarch64_enc_strh(src, mem));
7009
7010 ins_pipe(istore_reg_mem);
7011 %}
7012
7013 instruct storeimmC0(immI0 zero, memory2 mem)
7014 %{
7015 match(Set mem (StoreC mem zero));
7016 predicate(!needs_releasing_store(n));
7017
7018 ins_cost(INSN_COST);
7019 format %{ "strh zr, $mem\t# short" %}
7020
7021 ins_encode(aarch64_enc_strh0(mem));
7022
7023 ins_pipe(istore_mem);
7024 %}
7025
7026 // Store Integer
7027
7028 instruct storeI(iRegIorL2I src, memory4 mem)
7029 %{
7030 match(Set mem(StoreI mem src));
7031 predicate(!needs_releasing_store(n));
7032
7033 ins_cost(INSN_COST);
7034 format %{ "strw $src, $mem\t# int" %}
7035
7036 ins_encode(aarch64_enc_strw(src, mem));
7037
7038 ins_pipe(istore_reg_mem);
7039 %}
7040
7041 instruct storeimmI0(immI0 zero, memory4 mem)
7042 %{
7043 match(Set mem(StoreI mem zero));
7044 predicate(!needs_releasing_store(n));
7045
7046 ins_cost(INSN_COST);
7047 format %{ "strw zr, $mem\t# int" %}
7048
7049 ins_encode(aarch64_enc_strw0(mem));
7050
7051 ins_pipe(istore_mem);
7052 %}
7053
7054 // Store Long (64 bit signed)
7055 instruct storeL(iRegL src, memory8 mem)
7056 %{
7057 match(Set mem (StoreL mem src));
7058 predicate(!needs_releasing_store(n));
7059
7060 ins_cost(INSN_COST);
7061 format %{ "str $src, $mem\t# int" %}
7062
7063 ins_encode(aarch64_enc_str(src, mem));
7064
7065 ins_pipe(istore_reg_mem);
7066 %}
7067
7068 // Store Long (64 bit signed)
7069 instruct storeimmL0(immL0 zero, memory8 mem)
7070 %{
7071 match(Set mem (StoreL mem zero));
7072 predicate(!needs_releasing_store(n));
7073
7074 ins_cost(INSN_COST);
7075 format %{ "str zr, $mem\t# int" %}
7076
7077 ins_encode(aarch64_enc_str0(mem));
7078
7079 ins_pipe(istore_mem);
7080 %}
7081
7082 // Store Pointer
7083 instruct storeP(iRegP src, memory8 mem)
7084 %{
7085 match(Set mem (StoreP mem src));
7086 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7087
7088 ins_cost(INSN_COST);
7089 format %{ "str $src, $mem\t# ptr" %}
7090
7091 ins_encode(aarch64_enc_str(src, mem));
7092
7093 ins_pipe(istore_reg_mem);
7094 %}
7095
7096 // Store Pointer
7097 instruct storeimmP0(immP0 zero, memory8 mem)
7098 %{
7099 match(Set mem (StoreP mem zero));
7100 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7101
7102 ins_cost(INSN_COST);
7103 format %{ "str zr, $mem\t# ptr" %}
7104
7105 ins_encode(aarch64_enc_str0(mem));
7106
7107 ins_pipe(istore_mem);
7108 %}
7109
7110 // Store Compressed Pointer
7111 instruct storeN(iRegN src, memory4 mem)
7112 %{
7113 match(Set mem (StoreN mem src));
7114 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7115
7116 ins_cost(INSN_COST);
7117 format %{ "strw $src, $mem\t# compressed ptr" %}
7118
7119 ins_encode(aarch64_enc_strw(src, mem));
7120
7121 ins_pipe(istore_reg_mem);
7122 %}
7123
7124 instruct storeImmN0(immN0 zero, memory4 mem)
7125 %{
7126 match(Set mem (StoreN mem zero));
7127 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7128
7129 ins_cost(INSN_COST);
7130 format %{ "strw zr, $mem\t# compressed ptr" %}
7131
7132 ins_encode(aarch64_enc_strw0(mem));
7133
7134 ins_pipe(istore_mem);
7135 %}
7136
7137 // Store Float
7138 instruct storeF(vRegF src, memory4 mem)
7139 %{
7140 match(Set mem (StoreF mem src));
7141 predicate(!needs_releasing_store(n));
7142
7143 ins_cost(INSN_COST);
7144 format %{ "strs $src, $mem\t# float" %}
7145
7146 ins_encode( aarch64_enc_strs(src, mem) );
7147
7148 ins_pipe(pipe_class_memory);
7149 %}
7150
7151 // TODO
7152 // implement storeImmF0 and storeFImmPacked
7153
7154 // Store Double
7155 instruct storeD(vRegD src, memory8 mem)
7156 %{
7157 match(Set mem (StoreD mem src));
7158 predicate(!needs_releasing_store(n));
7159
7160 ins_cost(INSN_COST);
7161 format %{ "strd $src, $mem\t# double" %}
7162
7163 ins_encode( aarch64_enc_strd(src, mem) );
7164
7165 ins_pipe(pipe_class_memory);
7166 %}
7167
7168 // Store Compressed Klass Pointer
7169 instruct storeNKlass(iRegN src, memory4 mem)
7170 %{
7171 predicate(!needs_releasing_store(n));
7172 match(Set mem (StoreNKlass mem src));
7173
7174 ins_cost(INSN_COST);
7175 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7176
7177 ins_encode(aarch64_enc_strw(src, mem));
7178
7179 ins_pipe(istore_reg_mem);
7180 %}
7181
7182 // TODO
7183 // implement storeImmD0 and storeDImmPacked
7184
7185 // prefetch instructions
7186 // Must be safe to execute with invalid address (cannot fault).
7187
7188 instruct prefetchalloc( memory8 mem ) %{
7189 match(PrefetchAllocation mem);
7190
7191 ins_cost(INSN_COST);
7192 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7193
7194 ins_encode( aarch64_enc_prefetchw(mem) );
7195
7196 ins_pipe(iload_prefetch);
7197 %}
7198
7199 // ---------------- volatile loads and stores ----------------
7200
7201 // Load Byte (8 bit signed)
7202 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7203 %{
7204 match(Set dst (LoadB mem));
7205
7206 ins_cost(VOLATILE_REF_COST);
7207 format %{ "ldarsb $dst, $mem\t# byte" %}
7208
7209 ins_encode(aarch64_enc_ldarsb(dst, mem));
7210
7211 ins_pipe(pipe_serial);
7212 %}
7213
7214 // Load Byte (8 bit signed) into long
7215 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7216 %{
7217 match(Set dst (ConvI2L (LoadB mem)));
7218
7219 ins_cost(VOLATILE_REF_COST);
7220 format %{ "ldarsb $dst, $mem\t# byte" %}
7221
7222 ins_encode(aarch64_enc_ldarsb(dst, mem));
7223
7224 ins_pipe(pipe_serial);
7225 %}
7226
7227 // Load Byte (8 bit unsigned)
7228 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7229 %{
7230 match(Set dst (LoadUB mem));
7231
7232 ins_cost(VOLATILE_REF_COST);
7233 format %{ "ldarb $dst, $mem\t# byte" %}
7234
7235 ins_encode(aarch64_enc_ldarb(dst, mem));
7236
7237 ins_pipe(pipe_serial);
7238 %}
7239
7240 // Load Byte (8 bit unsigned) into long
7241 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7242 %{
7243 match(Set dst (ConvI2L (LoadUB mem)));
7244
7245 ins_cost(VOLATILE_REF_COST);
7246 format %{ "ldarb $dst, $mem\t# byte" %}
7247
7248 ins_encode(aarch64_enc_ldarb(dst, mem));
7249
7250 ins_pipe(pipe_serial);
7251 %}
7252
7253 // Load Short (16 bit signed)
7254 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7255 %{
7256 match(Set dst (LoadS mem));
7257
7258 ins_cost(VOLATILE_REF_COST);
7259 format %{ "ldarshw $dst, $mem\t# short" %}
7260
7261 ins_encode(aarch64_enc_ldarshw(dst, mem));
7262
7263 ins_pipe(pipe_serial);
7264 %}
7265
7266 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7267 %{
7268 match(Set dst (LoadUS mem));
7269
7270 ins_cost(VOLATILE_REF_COST);
7271 format %{ "ldarhw $dst, $mem\t# short" %}
7272
7273 ins_encode(aarch64_enc_ldarhw(dst, mem));
7274
7275 ins_pipe(pipe_serial);
7276 %}
7277
7278 // Load Short/Char (16 bit unsigned) into long
7279 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7280 %{
7281 match(Set dst (ConvI2L (LoadUS mem)));
7282
7283 ins_cost(VOLATILE_REF_COST);
7284 format %{ "ldarh $dst, $mem\t# short" %}
7285
7286 ins_encode(aarch64_enc_ldarh(dst, mem));
7287
7288 ins_pipe(pipe_serial);
7289 %}
7290
7291 // Load Short/Char (16 bit signed) into long
7292 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7293 %{
7294 match(Set dst (ConvI2L (LoadS mem)));
7295
7296 ins_cost(VOLATILE_REF_COST);
7297 format %{ "ldarh $dst, $mem\t# short" %}
7298
7299 ins_encode(aarch64_enc_ldarsh(dst, mem));
7300
7301 ins_pipe(pipe_serial);
7302 %}
7303
7304 // Load Integer (32 bit signed)
7305 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7306 %{
7307 match(Set dst (LoadI mem));
7308
7309 ins_cost(VOLATILE_REF_COST);
7310 format %{ "ldarw $dst, $mem\t# int" %}
7311
7312 ins_encode(aarch64_enc_ldarw(dst, mem));
7313
7314 ins_pipe(pipe_serial);
7315 %}
7316
7317 // Load Integer (32 bit unsigned) into long
7318 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7319 %{
7320 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7321
7322 ins_cost(VOLATILE_REF_COST);
7323 format %{ "ldarw $dst, $mem\t# int" %}
7324
7325 ins_encode(aarch64_enc_ldarw(dst, mem));
7326
7327 ins_pipe(pipe_serial);
7328 %}
7329
7330 // Load Long (64 bit signed)
7331 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7332 %{
7333 match(Set dst (LoadL mem));
7334
7335 ins_cost(VOLATILE_REF_COST);
7336 format %{ "ldar $dst, $mem\t# int" %}
7337
7338 ins_encode(aarch64_enc_ldar(dst, mem));
7339
7340 ins_pipe(pipe_serial);
7341 %}
7342
7343 // Load Pointer
7344 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7345 %{
7346 match(Set dst (LoadP mem));
7347 predicate(n->as_Load()->barrier_data() == 0);
7348
7349 ins_cost(VOLATILE_REF_COST);
7350 format %{ "ldar $dst, $mem\t# ptr" %}
7351
7352 ins_encode(aarch64_enc_ldar(dst, mem));
7353
7354 ins_pipe(pipe_serial);
7355 %}
7356
7357 // Load Compressed Pointer
7358 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7359 %{
7360 match(Set dst (LoadN mem));
7361 predicate(n->as_Load()->barrier_data() == 0);
7362
7363 ins_cost(VOLATILE_REF_COST);
7364 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7365
7366 ins_encode(aarch64_enc_ldarw(dst, mem));
7367
7368 ins_pipe(pipe_serial);
7369 %}
7370
7371 // Load Float
7372 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7373 %{
7374 match(Set dst (LoadF mem));
7375
7376 ins_cost(VOLATILE_REF_COST);
7377 format %{ "ldars $dst, $mem\t# float" %}
7378
7379 ins_encode( aarch64_enc_fldars(dst, mem) );
7380
7381 ins_pipe(pipe_serial);
7382 %}
7383
7384 // Load Double
7385 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7386 %{
7387 match(Set dst (LoadD mem));
7388
7389 ins_cost(VOLATILE_REF_COST);
7390 format %{ "ldard $dst, $mem\t# double" %}
7391
7392 ins_encode( aarch64_enc_fldard(dst, mem) );
7393
7394 ins_pipe(pipe_serial);
7395 %}
7396
7397 // Store Byte
7398 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7399 %{
7400 match(Set mem (StoreB mem src));
7401
7402 ins_cost(VOLATILE_REF_COST);
7403 format %{ "stlrb $src, $mem\t# byte" %}
7404
7405 ins_encode(aarch64_enc_stlrb(src, mem));
7406
7407 ins_pipe(pipe_class_memory);
7408 %}
7409
7410 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7411 %{
7412 match(Set mem (StoreB mem zero));
7413
7414 ins_cost(VOLATILE_REF_COST);
7415 format %{ "stlrb zr, $mem\t# byte" %}
7416
7417 ins_encode(aarch64_enc_stlrb0(mem));
7418
7419 ins_pipe(pipe_class_memory);
7420 %}
7421
7422 // Store Char/Short
7423 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7424 %{
7425 match(Set mem (StoreC mem src));
7426
7427 ins_cost(VOLATILE_REF_COST);
7428 format %{ "stlrh $src, $mem\t# short" %}
7429
7430 ins_encode(aarch64_enc_stlrh(src, mem));
7431
7432 ins_pipe(pipe_class_memory);
7433 %}
7434
7435 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7436 %{
7437 match(Set mem (StoreC mem zero));
7438
7439 ins_cost(VOLATILE_REF_COST);
7440 format %{ "stlrh zr, $mem\t# short" %}
7441
7442 ins_encode(aarch64_enc_stlrh0(mem));
7443
7444 ins_pipe(pipe_class_memory);
7445 %}
7446
7447 // Store Integer
7448
7449 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7450 %{
7451 match(Set mem(StoreI mem src));
7452
7453 ins_cost(VOLATILE_REF_COST);
7454 format %{ "stlrw $src, $mem\t# int" %}
7455
7456 ins_encode(aarch64_enc_stlrw(src, mem));
7457
7458 ins_pipe(pipe_class_memory);
7459 %}
7460
7461 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7462 %{
7463 match(Set mem(StoreI mem zero));
7464
7465 ins_cost(VOLATILE_REF_COST);
7466 format %{ "stlrw zr, $mem\t# int" %}
7467
7468 ins_encode(aarch64_enc_stlrw0(mem));
7469
7470 ins_pipe(pipe_class_memory);
7471 %}
7472
7473 // Store Long (64 bit signed)
7474 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7475 %{
7476 match(Set mem (StoreL mem src));
7477
7478 ins_cost(VOLATILE_REF_COST);
7479 format %{ "stlr $src, $mem\t# int" %}
7480
7481 ins_encode(aarch64_enc_stlr(src, mem));
7482
7483 ins_pipe(pipe_class_memory);
7484 %}
7485
7486 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7487 %{
7488 match(Set mem (StoreL mem zero));
7489
7490 ins_cost(VOLATILE_REF_COST);
7491 format %{ "stlr zr, $mem\t# int" %}
7492
7493 ins_encode(aarch64_enc_stlr0(mem));
7494
7495 ins_pipe(pipe_class_memory);
7496 %}
7497
7498 // Store Pointer
7499 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7500 %{
7501 match(Set mem (StoreP mem src));
7502 predicate(n->as_Store()->barrier_data() == 0);
7503
7504 ins_cost(VOLATILE_REF_COST);
7505 format %{ "stlr $src, $mem\t# ptr" %}
7506
7507 ins_encode(aarch64_enc_stlr(src, mem));
7508
7509 ins_pipe(pipe_class_memory);
7510 %}
7511
7512 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7513 %{
7514 match(Set mem (StoreP mem zero));
7515 predicate(n->as_Store()->barrier_data() == 0);
7516
7517 ins_cost(VOLATILE_REF_COST);
7518 format %{ "stlr zr, $mem\t# ptr" %}
7519
7520 ins_encode(aarch64_enc_stlr0(mem));
7521
7522 ins_pipe(pipe_class_memory);
7523 %}
7524
7525 // Store Compressed Pointer
7526 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7527 %{
7528 match(Set mem (StoreN mem src));
7529 predicate(n->as_Store()->barrier_data() == 0);
7530
7531 ins_cost(VOLATILE_REF_COST);
7532 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7533
7534 ins_encode(aarch64_enc_stlrw(src, mem));
7535
7536 ins_pipe(pipe_class_memory);
7537 %}
7538
7539 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7540 %{
7541 match(Set mem (StoreN mem zero));
7542 predicate(n->as_Store()->barrier_data() == 0);
7543
7544 ins_cost(VOLATILE_REF_COST);
7545 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7546
7547 ins_encode(aarch64_enc_stlrw0(mem));
7548
7549 ins_pipe(pipe_class_memory);
7550 %}
7551
7552 // Store Float
7553 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7554 %{
7555 match(Set mem (StoreF mem src));
7556
7557 ins_cost(VOLATILE_REF_COST);
7558 format %{ "stlrs $src, $mem\t# float" %}
7559
7560 ins_encode( aarch64_enc_fstlrs(src, mem) );
7561
7562 ins_pipe(pipe_class_memory);
7563 %}
7564
7565 // TODO
7566 // implement storeImmF0 and storeFImmPacked
7567
7568 // Store Double
7569 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem (StoreD mem src));
7572
7573 ins_cost(VOLATILE_REF_COST);
7574 format %{ "stlrd $src, $mem\t# double" %}
7575
7576 ins_encode( aarch64_enc_fstlrd(src, mem) );
7577
7578 ins_pipe(pipe_class_memory);
7579 %}
7580
7581 // ---------------- end of volatile loads and stores ----------------
7582
7583 instruct cacheWB(indirect addr)
7584 %{
7585 predicate(VM_Version::supports_data_cache_line_flush());
7586 match(CacheWB addr);
7587
7588 ins_cost(100);
7589 format %{"cache wb $addr" %}
7590 ins_encode %{
7591 assert($addr->index_position() < 0, "should be");
7592 assert($addr$$disp == 0, "should be");
7593 __ cache_wb(Address($addr$$base$$Register, 0));
7594 %}
7595 ins_pipe(pipe_slow); // XXX
7596 %}
7597
7598 instruct cacheWBPreSync()
7599 %{
7600 predicate(VM_Version::supports_data_cache_line_flush());
7601 match(CacheWBPreSync);
7602
7603 ins_cost(100);
7604 format %{"cache wb presync" %}
7605 ins_encode %{
7606 __ cache_wbsync(true);
7607 %}
7608 ins_pipe(pipe_slow); // XXX
7609 %}
7610
7611 instruct cacheWBPostSync()
7612 %{
7613 predicate(VM_Version::supports_data_cache_line_flush());
7614 match(CacheWBPostSync);
7615
7616 ins_cost(100);
7617 format %{"cache wb postsync" %}
7618 ins_encode %{
7619 __ cache_wbsync(false);
7620 %}
7621 ins_pipe(pipe_slow); // XXX
7622 %}
7623
7624 // ============================================================================
7625 // BSWAP Instructions
7626
7627 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7628 match(Set dst (ReverseBytesI src));
7629
7630 ins_cost(INSN_COST);
7631 format %{ "revw $dst, $src" %}
7632
7633 ins_encode %{
7634 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7635 %}
7636
7637 ins_pipe(ialu_reg);
7638 %}
7639
7640 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7641 match(Set dst (ReverseBytesL src));
7642
7643 ins_cost(INSN_COST);
7644 format %{ "rev $dst, $src" %}
7645
7646 ins_encode %{
7647 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7648 %}
7649
7650 ins_pipe(ialu_reg);
7651 %}
7652
7653 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7654 match(Set dst (ReverseBytesUS src));
7655
7656 ins_cost(INSN_COST);
7657 format %{ "rev16w $dst, $src" %}
7658
7659 ins_encode %{
7660 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7661 %}
7662
7663 ins_pipe(ialu_reg);
7664 %}
7665
7666 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7667 match(Set dst (ReverseBytesS src));
7668
7669 ins_cost(INSN_COST);
7670 format %{ "rev16w $dst, $src\n\t"
7671 "sbfmw $dst, $dst, #0, #15" %}
7672
7673 ins_encode %{
7674 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7675 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7676 %}
7677
7678 ins_pipe(ialu_reg);
7679 %}
7680
7681 // ============================================================================
7682 // Zero Count Instructions
7683
7684 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7685 match(Set dst (CountLeadingZerosI src));
7686
7687 ins_cost(INSN_COST);
7688 format %{ "clzw $dst, $src" %}
7689 ins_encode %{
7690 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7691 %}
7692
7693 ins_pipe(ialu_reg);
7694 %}
7695
7696 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7697 match(Set dst (CountLeadingZerosL src));
7698
7699 ins_cost(INSN_COST);
7700 format %{ "clz $dst, $src" %}
7701 ins_encode %{
7702 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7703 %}
7704
7705 ins_pipe(ialu_reg);
7706 %}
7707
7708 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7709 match(Set dst (CountTrailingZerosI src));
7710
7711 ins_cost(INSN_COST * 2);
7712 format %{ "rbitw $dst, $src\n\t"
7713 "clzw $dst, $dst" %}
7714 ins_encode %{
7715 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7716 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7717 %}
7718
7719 ins_pipe(ialu_reg);
7720 %}
7721
7722 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7723 match(Set dst (CountTrailingZerosL src));
7724
7725 ins_cost(INSN_COST * 2);
7726 format %{ "rbit $dst, $src\n\t"
7727 "clz $dst, $dst" %}
7728 ins_encode %{
7729 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7730 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7731 %}
7732
7733 ins_pipe(ialu_reg);
7734 %}
7735
7736 //---------- Population Count Instructions -------------------------------------
7737 //
7738
7739 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7740 match(Set dst (PopCountI src));
7741 effect(TEMP tmp);
7742 ins_cost(INSN_COST * 13);
7743
7744 format %{ "movw $src, $src\n\t"
7745 "mov $tmp, $src\t# vector (1D)\n\t"
7746 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7747 "addv $tmp, $tmp\t# vector (8B)\n\t"
7748 "mov $dst, $tmp\t# vector (1D)" %}
7749 ins_encode %{
7750 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7751 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7752 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7753 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7754 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7755 %}
7756
7757 ins_pipe(pipe_class_default);
7758 %}
7759
7760 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7761 match(Set dst (PopCountI (LoadI mem)));
7762 effect(TEMP tmp);
7763 ins_cost(INSN_COST * 13);
7764
7765 format %{ "ldrs $tmp, $mem\n\t"
7766 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7767 "addv $tmp, $tmp\t# vector (8B)\n\t"
7768 "mov $dst, $tmp\t# vector (1D)" %}
7769 ins_encode %{
7770 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7771 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7772 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7773 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7774 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7775 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7776 %}
7777
7778 ins_pipe(pipe_class_default);
7779 %}
7780
7781 // Note: Long.bitCount(long) returns an int.
7782 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7783 match(Set dst (PopCountL src));
7784 effect(TEMP tmp);
7785 ins_cost(INSN_COST * 13);
7786
7787 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7788 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7789 "addv $tmp, $tmp\t# vector (8B)\n\t"
7790 "mov $dst, $tmp\t# vector (1D)" %}
7791 ins_encode %{
7792 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7793 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7795 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7796 %}
7797
7798 ins_pipe(pipe_class_default);
7799 %}
7800
7801 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7802 match(Set dst (PopCountL (LoadL mem)));
7803 effect(TEMP tmp);
7804 ins_cost(INSN_COST * 13);
7805
7806 format %{ "ldrd $tmp, $mem\n\t"
7807 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7808 "addv $tmp, $tmp\t# vector (8B)\n\t"
7809 "mov $dst, $tmp\t# vector (1D)" %}
7810 ins_encode %{
7811 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7812 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7813 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7814 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7815 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7816 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7817 %}
7818
7819 ins_pipe(pipe_class_default);
7820 %}
7821
7822 // ============================================================================
7823 // VerifyVectorAlignment Instruction
7824
7825 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7826 match(Set addr (VerifyVectorAlignment addr mask));
7827 effect(KILL cr);
7828 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7829 ins_encode %{
7830 Label Lskip;
7831 // check if masked bits of addr are zero
7832 __ tst($addr$$Register, $mask$$constant);
7833 __ br(Assembler::EQ, Lskip);
7834 __ stop("verify_vector_alignment found a misaligned vector memory access");
7835 __ bind(Lskip);
7836 %}
7837 ins_pipe(pipe_slow);
7838 %}
7839
7840 // ============================================================================
7841 // MemBar Instruction
7842
7843 instruct load_fence() %{
7844 match(LoadFence);
7845 ins_cost(VOLATILE_REF_COST);
7846
7847 format %{ "load_fence" %}
7848
7849 ins_encode %{
7850 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7851 %}
7852 ins_pipe(pipe_serial);
7853 %}
7854
7855 instruct unnecessary_membar_acquire() %{
7856 predicate(unnecessary_acquire(n));
7857 match(MemBarAcquire);
7858 ins_cost(0);
7859
7860 format %{ "membar_acquire (elided)" %}
7861
7862 ins_encode %{
7863 __ block_comment("membar_acquire (elided)");
7864 %}
7865
7866 ins_pipe(pipe_class_empty);
7867 %}
7868
7869 instruct membar_acquire() %{
7870 match(MemBarAcquire);
7871 ins_cost(VOLATILE_REF_COST);
7872
7873 format %{ "membar_acquire\n\t"
7874 "dmb ishld" %}
7875
7876 ins_encode %{
7877 __ block_comment("membar_acquire");
7878 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7879 %}
7880
7881 ins_pipe(pipe_serial);
7882 %}
7883
7884
7885 instruct membar_acquire_lock() %{
7886 match(MemBarAcquireLock);
7887 ins_cost(VOLATILE_REF_COST);
7888
7889 format %{ "membar_acquire_lock (elided)" %}
7890
7891 ins_encode %{
7892 __ block_comment("membar_acquire_lock (elided)");
7893 %}
7894
7895 ins_pipe(pipe_serial);
7896 %}
7897
7898 instruct store_fence() %{
7899 match(StoreFence);
7900 ins_cost(VOLATILE_REF_COST);
7901
7902 format %{ "store_fence" %}
7903
7904 ins_encode %{
7905 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7906 %}
7907 ins_pipe(pipe_serial);
7908 %}
7909
7910 instruct unnecessary_membar_release() %{
7911 predicate(unnecessary_release(n));
7912 match(MemBarRelease);
7913 ins_cost(0);
7914
7915 format %{ "membar_release (elided)" %}
7916
7917 ins_encode %{
7918 __ block_comment("membar_release (elided)");
7919 %}
7920 ins_pipe(pipe_serial);
7921 %}
7922
7923 instruct membar_release() %{
7924 match(MemBarRelease);
7925 ins_cost(VOLATILE_REF_COST);
7926
7927 format %{ "membar_release\n\t"
7928 "dmb ishst\n\tdmb ishld" %}
7929
7930 ins_encode %{
7931 __ block_comment("membar_release");
7932 // These will be merged if AlwaysMergeDMB is enabled.
7933 __ membar(Assembler::StoreStore);
7934 __ membar(Assembler::LoadStore);
7935 %}
7936 ins_pipe(pipe_serial);
7937 %}
7938
7939 instruct membar_storestore() %{
7940 match(MemBarStoreStore);
7941 match(StoreStoreFence);
7942 ins_cost(VOLATILE_REF_COST);
7943
7944 format %{ "MEMBAR-store-store" %}
7945
7946 ins_encode %{
7947 __ membar(Assembler::StoreStore);
7948 %}
7949 ins_pipe(pipe_serial);
7950 %}
7951
7952 instruct membar_release_lock() %{
7953 match(MemBarReleaseLock);
7954 ins_cost(VOLATILE_REF_COST);
7955
7956 format %{ "membar_release_lock (elided)" %}
7957
7958 ins_encode %{
7959 __ block_comment("membar_release_lock (elided)");
7960 %}
7961
7962 ins_pipe(pipe_serial);
7963 %}
7964
7965 instruct unnecessary_membar_volatile() %{
7966 predicate(unnecessary_volatile(n));
7967 match(MemBarVolatile);
7968 ins_cost(0);
7969
7970 format %{ "membar_volatile (elided)" %}
7971
7972 ins_encode %{
7973 __ block_comment("membar_volatile (elided)");
7974 %}
7975
7976 ins_pipe(pipe_serial);
7977 %}
7978
7979 instruct membar_volatile() %{
7980 match(MemBarVolatile);
7981 ins_cost(VOLATILE_REF_COST*100);
7982
7983 format %{ "membar_volatile\n\t"
7984 "dmb ish"%}
7985
7986 ins_encode %{
7987 __ block_comment("membar_volatile");
7988 __ membar(Assembler::StoreLoad);
7989 %}
7990
7991 ins_pipe(pipe_serial);
7992 %}
7993
7994 // ============================================================================
7995 // Cast/Convert Instructions
7996
7997 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7998 match(Set dst (CastX2P src));
7999
8000 ins_cost(INSN_COST);
8001 format %{ "mov $dst, $src\t# long -> ptr" %}
8002
8003 ins_encode %{
8004 if ($dst$$reg != $src$$reg) {
8005 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8006 }
8007 %}
8008
8009 ins_pipe(ialu_reg);
8010 %}
8011
8012 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8013 match(Set dst (CastI2N src));
8014
8015 ins_cost(INSN_COST);
8016 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8017
8018 ins_encode %{
8019 if ($dst$$reg != $src$$reg) {
8020 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8021 }
8022 %}
8023
8024 ins_pipe(ialu_reg);
8025 %}
8026
8027 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8028 match(Set dst (CastP2X src));
8029
8030 ins_cost(INSN_COST);
8031 format %{ "mov $dst, $src\t# ptr -> long" %}
8032
8033 ins_encode %{
8034 if ($dst$$reg != $src$$reg) {
8035 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8036 }
8037 %}
8038
8039 ins_pipe(ialu_reg);
8040 %}
8041
8042 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8043 match(Set dst (CastP2X src));
8044
8045 ins_cost(INSN_COST);
8046 format %{ "mov $dst, $src\t# ptr -> long" %}
8047
8048 ins_encode %{
8049 if ($dst$$reg != $src$$reg) {
8050 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8051 }
8052 %}
8053
8054 ins_pipe(ialu_reg);
8055 %}
8056
8057 // Convert oop into int for vectors alignment masking
8058 instruct convP2I(iRegINoSp dst, iRegP src) %{
8059 match(Set dst (ConvL2I (CastP2X src)));
8060
8061 ins_cost(INSN_COST);
8062 format %{ "movw $dst, $src\t# ptr -> int" %}
8063 ins_encode %{
8064 __ movw($dst$$Register, $src$$Register);
8065 %}
8066
8067 ins_pipe(ialu_reg);
8068 %}
8069
8070 // Convert compressed oop into int for vectors alignment masking
8071 // in case of 32bit oops (heap < 4Gb).
8072 instruct convN2I(iRegINoSp dst, iRegN src)
8073 %{
8074 predicate(CompressedOops::shift() == 0);
8075 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8076
8077 ins_cost(INSN_COST);
8078 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8079 ins_encode %{
8080 __ movw($dst$$Register, $src$$Register);
8081 %}
8082
8083 ins_pipe(ialu_reg);
8084 %}
8085
8086
8087 // Convert oop pointer into compressed form
8088 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8089 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8090 match(Set dst (EncodeP src));
8091 effect(KILL cr);
8092 ins_cost(INSN_COST * 3);
8093 format %{ "encode_heap_oop $dst, $src" %}
8094 ins_encode %{
8095 Register s = $src$$Register;
8096 Register d = $dst$$Register;
8097 __ encode_heap_oop(d, s);
8098 %}
8099 ins_pipe(ialu_reg);
8100 %}
8101
8102 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8103 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8104 match(Set dst (EncodeP src));
8105 ins_cost(INSN_COST * 3);
8106 format %{ "encode_heap_oop_not_null $dst, $src" %}
8107 ins_encode %{
8108 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8109 %}
8110 ins_pipe(ialu_reg);
8111 %}
8112
8113 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8114 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8115 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8116 match(Set dst (DecodeN src));
8117 ins_cost(INSN_COST * 3);
8118 format %{ "decode_heap_oop $dst, $src" %}
8119 ins_encode %{
8120 Register s = $src$$Register;
8121 Register d = $dst$$Register;
8122 __ decode_heap_oop(d, s);
8123 %}
8124 ins_pipe(ialu_reg);
8125 %}
8126
8127 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8128 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8129 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8130 match(Set dst (DecodeN src));
8131 ins_cost(INSN_COST * 3);
8132 format %{ "decode_heap_oop_not_null $dst, $src" %}
8133 ins_encode %{
8134 Register s = $src$$Register;
8135 Register d = $dst$$Register;
8136 __ decode_heap_oop_not_null(d, s);
8137 %}
8138 ins_pipe(ialu_reg);
8139 %}
8140
8141 // n.b. AArch64 implementations of encode_klass_not_null and
8142 // decode_klass_not_null do not modify the flags register so, unlike
8143 // Intel, we don't kill CR as a side effect here
8144
8145 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8146 match(Set dst (EncodePKlass src));
8147
8148 ins_cost(INSN_COST * 3);
8149 format %{ "encode_klass_not_null $dst,$src" %}
8150
8151 ins_encode %{
8152 Register src_reg = as_Register($src$$reg);
8153 Register dst_reg = as_Register($dst$$reg);
8154 __ encode_klass_not_null(dst_reg, src_reg);
8155 %}
8156
8157 ins_pipe(ialu_reg);
8158 %}
8159
8160 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8161 match(Set dst (DecodeNKlass src));
8162
8163 ins_cost(INSN_COST * 3);
8164 format %{ "decode_klass_not_null $dst,$src" %}
8165
8166 ins_encode %{
8167 Register src_reg = as_Register($src$$reg);
8168 Register dst_reg = as_Register($dst$$reg);
8169 if (dst_reg != src_reg) {
8170 __ decode_klass_not_null(dst_reg, src_reg);
8171 } else {
8172 __ decode_klass_not_null(dst_reg);
8173 }
8174 %}
8175
8176 ins_pipe(ialu_reg);
8177 %}
8178
8179 instruct checkCastPP(iRegPNoSp dst)
8180 %{
8181 match(Set dst (CheckCastPP dst));
8182
8183 size(0);
8184 format %{ "# checkcastPP of $dst" %}
8185 ins_encode(/* empty encoding */);
8186 ins_pipe(pipe_class_empty);
8187 %}
8188
8189 instruct castPP(iRegPNoSp dst)
8190 %{
8191 match(Set dst (CastPP dst));
8192
8193 size(0);
8194 format %{ "# castPP of $dst" %}
8195 ins_encode(/* empty encoding */);
8196 ins_pipe(pipe_class_empty);
8197 %}
8198
8199 instruct castII(iRegI dst)
8200 %{
8201 match(Set dst (CastII dst));
8202
8203 size(0);
8204 format %{ "# castII of $dst" %}
8205 ins_encode(/* empty encoding */);
8206 ins_cost(0);
8207 ins_pipe(pipe_class_empty);
8208 %}
8209
8210 instruct castLL(iRegL dst)
8211 %{
8212 match(Set dst (CastLL dst));
8213
8214 size(0);
8215 format %{ "# castLL of $dst" %}
8216 ins_encode(/* empty encoding */);
8217 ins_cost(0);
8218 ins_pipe(pipe_class_empty);
8219 %}
8220
8221 instruct castFF(vRegF dst)
8222 %{
8223 match(Set dst (CastFF dst));
8224
8225 size(0);
8226 format %{ "# castFF of $dst" %}
8227 ins_encode(/* empty encoding */);
8228 ins_cost(0);
8229 ins_pipe(pipe_class_empty);
8230 %}
8231
8232 instruct castDD(vRegD dst)
8233 %{
8234 match(Set dst (CastDD dst));
8235
8236 size(0);
8237 format %{ "# castDD of $dst" %}
8238 ins_encode(/* empty encoding */);
8239 ins_cost(0);
8240 ins_pipe(pipe_class_empty);
8241 %}
8242
8243 instruct castVV(vReg dst)
8244 %{
8245 match(Set dst (CastVV dst));
8246
8247 size(0);
8248 format %{ "# castVV of $dst" %}
8249 ins_encode(/* empty encoding */);
8250 ins_cost(0);
8251 ins_pipe(pipe_class_empty);
8252 %}
8253
8254 instruct castVVMask(pRegGov dst)
8255 %{
8256 match(Set dst (CastVV dst));
8257
8258 size(0);
8259 format %{ "# castVV of $dst" %}
8260 ins_encode(/* empty encoding */);
8261 ins_cost(0);
8262 ins_pipe(pipe_class_empty);
8263 %}
8264
8265 // ============================================================================
8266 // Atomic operation instructions
8267 //
8268
8269 // standard CompareAndSwapX when we are using barriers
8270 // these have higher priority than the rules selected by a predicate
8271
8272 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8273 // can't match them
8274
8275 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8276
8277 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8278 ins_cost(2 * VOLATILE_REF_COST);
8279
8280 effect(KILL cr);
8281
8282 format %{
8283 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8284 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8285 %}
8286
8287 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8288 aarch64_enc_cset_eq(res));
8289
8290 ins_pipe(pipe_slow);
8291 %}
8292
8293 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8294
8295 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8296 ins_cost(2 * VOLATILE_REF_COST);
8297
8298 effect(KILL cr);
8299
8300 format %{
8301 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8302 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8303 %}
8304
8305 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8306 aarch64_enc_cset_eq(res));
8307
8308 ins_pipe(pipe_slow);
8309 %}
8310
8311 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8312
8313 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8314 ins_cost(2 * VOLATILE_REF_COST);
8315
8316 effect(KILL cr);
8317
8318 format %{
8319 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8320 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8321 %}
8322
8323 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8324 aarch64_enc_cset_eq(res));
8325
8326 ins_pipe(pipe_slow);
8327 %}
8328
8329 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8330
8331 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8332 ins_cost(2 * VOLATILE_REF_COST);
8333
8334 effect(KILL cr);
8335
8336 format %{
8337 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8338 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8339 %}
8340
8341 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8342 aarch64_enc_cset_eq(res));
8343
8344 ins_pipe(pipe_slow);
8345 %}
8346
8347 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8348
8349 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8350 predicate(n->as_LoadStore()->barrier_data() == 0);
8351 ins_cost(2 * VOLATILE_REF_COST);
8352
8353 effect(KILL cr);
8354
8355 format %{
8356 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8357 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8358 %}
8359
8360 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8361 aarch64_enc_cset_eq(res));
8362
8363 ins_pipe(pipe_slow);
8364 %}
8365
8366 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8367
8368 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8369 predicate(n->as_LoadStore()->barrier_data() == 0);
8370 ins_cost(2 * VOLATILE_REF_COST);
8371
8372 effect(KILL cr);
8373
8374 format %{
8375 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8376 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8377 %}
8378
8379 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8380 aarch64_enc_cset_eq(res));
8381
8382 ins_pipe(pipe_slow);
8383 %}
8384
8385 // alternative CompareAndSwapX when we are eliding barriers
8386
8387 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8388
8389 predicate(needs_acquiring_load_exclusive(n));
8390 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8391 ins_cost(VOLATILE_REF_COST);
8392
8393 effect(KILL cr);
8394
8395 format %{
8396 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8397 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8398 %}
8399
8400 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8401 aarch64_enc_cset_eq(res));
8402
8403 ins_pipe(pipe_slow);
8404 %}
8405
8406 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8407
8408 predicate(needs_acquiring_load_exclusive(n));
8409 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8410 ins_cost(VOLATILE_REF_COST);
8411
8412 effect(KILL cr);
8413
8414 format %{
8415 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8416 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8417 %}
8418
8419 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8420 aarch64_enc_cset_eq(res));
8421
8422 ins_pipe(pipe_slow);
8423 %}
8424
8425 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8426
8427 predicate(needs_acquiring_load_exclusive(n));
8428 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8429 ins_cost(VOLATILE_REF_COST);
8430
8431 effect(KILL cr);
8432
8433 format %{
8434 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8435 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8436 %}
8437
8438 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8439 aarch64_enc_cset_eq(res));
8440
8441 ins_pipe(pipe_slow);
8442 %}
8443
8444 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8445
8446 predicate(needs_acquiring_load_exclusive(n));
8447 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8448 ins_cost(VOLATILE_REF_COST);
8449
8450 effect(KILL cr);
8451
8452 format %{
8453 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8454 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8455 %}
8456
8457 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8458 aarch64_enc_cset_eq(res));
8459
8460 ins_pipe(pipe_slow);
8461 %}
8462
8463 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8464
8465 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8466 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8467 ins_cost(VOLATILE_REF_COST);
8468
8469 effect(KILL cr);
8470
8471 format %{
8472 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8473 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8474 %}
8475
8476 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8477 aarch64_enc_cset_eq(res));
8478
8479 ins_pipe(pipe_slow);
8480 %}
8481
8482 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8483
8484 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8485 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8486 ins_cost(VOLATILE_REF_COST);
8487
8488 effect(KILL cr);
8489
8490 format %{
8491 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8492 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8493 %}
8494
8495 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8496 aarch64_enc_cset_eq(res));
8497
8498 ins_pipe(pipe_slow);
8499 %}
8500
8501
8502 // ---------------------------------------------------------------------
8503
8504 // BEGIN This section of the file is automatically generated. Do not edit --------------
8505
8506 // Sundry CAS operations. Note that release is always true,
8507 // regardless of the memory ordering of the CAS. This is because we
8508 // need the volatile case to be sequentially consistent but there is
8509 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8510 // can't check the type of memory ordering here, so we always emit a
8511 // STLXR.
8512
8513 // This section is generated from cas.m4
8514
8515
8516 // This pattern is generated automatically from cas.m4.
8517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8518 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8519 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8520 ins_cost(2 * VOLATILE_REF_COST);
8521 effect(TEMP_DEF res, KILL cr);
8522 format %{
8523 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8524 %}
8525 ins_encode %{
8526 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8527 Assembler::byte, /*acquire*/ false, /*release*/ true,
8528 /*weak*/ false, $res$$Register);
8529 __ sxtbw($res$$Register, $res$$Register);
8530 %}
8531 ins_pipe(pipe_slow);
8532 %}
8533
8534 // This pattern is generated automatically from cas.m4.
8535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8536 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8537 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8538 ins_cost(2 * VOLATILE_REF_COST);
8539 effect(TEMP_DEF res, KILL cr);
8540 format %{
8541 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8542 %}
8543 ins_encode %{
8544 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8545 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8546 /*weak*/ false, $res$$Register);
8547 __ sxthw($res$$Register, $res$$Register);
8548 %}
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552 // This pattern is generated automatically from cas.m4.
8553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8554 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8555 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8556 ins_cost(2 * VOLATILE_REF_COST);
8557 effect(TEMP_DEF res, KILL cr);
8558 format %{
8559 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8560 %}
8561 ins_encode %{
8562 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8563 Assembler::word, /*acquire*/ false, /*release*/ true,
8564 /*weak*/ false, $res$$Register);
8565 %}
8566 ins_pipe(pipe_slow);
8567 %}
8568
8569 // This pattern is generated automatically from cas.m4.
8570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8571 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8572 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8573 ins_cost(2 * VOLATILE_REF_COST);
8574 effect(TEMP_DEF res, KILL cr);
8575 format %{
8576 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8577 %}
8578 ins_encode %{
8579 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8580 Assembler::xword, /*acquire*/ false, /*release*/ true,
8581 /*weak*/ false, $res$$Register);
8582 %}
8583 ins_pipe(pipe_slow);
8584 %}
8585
8586 // This pattern is generated automatically from cas.m4.
8587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8588 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8589 predicate(n->as_LoadStore()->barrier_data() == 0);
8590 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8591 ins_cost(2 * VOLATILE_REF_COST);
8592 effect(TEMP_DEF res, KILL cr);
8593 format %{
8594 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8595 %}
8596 ins_encode %{
8597 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8598 Assembler::word, /*acquire*/ false, /*release*/ true,
8599 /*weak*/ false, $res$$Register);
8600 %}
8601 ins_pipe(pipe_slow);
8602 %}
8603
8604 // This pattern is generated automatically from cas.m4.
8605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8606 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8607 predicate(n->as_LoadStore()->barrier_data() == 0);
8608 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8609 ins_cost(2 * VOLATILE_REF_COST);
8610 effect(TEMP_DEF res, KILL cr);
8611 format %{
8612 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8613 %}
8614 ins_encode %{
8615 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8616 Assembler::xword, /*acquire*/ false, /*release*/ true,
8617 /*weak*/ false, $res$$Register);
8618 %}
8619 ins_pipe(pipe_slow);
8620 %}
8621
8622 // This pattern is generated automatically from cas.m4.
8623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8624 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8625 predicate(needs_acquiring_load_exclusive(n));
8626 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8627 ins_cost(VOLATILE_REF_COST);
8628 effect(TEMP_DEF res, KILL cr);
8629 format %{
8630 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8631 %}
8632 ins_encode %{
8633 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8634 Assembler::byte, /*acquire*/ true, /*release*/ true,
8635 /*weak*/ false, $res$$Register);
8636 __ sxtbw($res$$Register, $res$$Register);
8637 %}
8638 ins_pipe(pipe_slow);
8639 %}
8640
8641 // This pattern is generated automatically from cas.m4.
8642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8643 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8644 predicate(needs_acquiring_load_exclusive(n));
8645 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8646 ins_cost(VOLATILE_REF_COST);
8647 effect(TEMP_DEF res, KILL cr);
8648 format %{
8649 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8650 %}
8651 ins_encode %{
8652 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8653 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8654 /*weak*/ false, $res$$Register);
8655 __ sxthw($res$$Register, $res$$Register);
8656 %}
8657 ins_pipe(pipe_slow);
8658 %}
8659
8660 // This pattern is generated automatically from cas.m4.
8661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8662 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8663 predicate(needs_acquiring_load_exclusive(n));
8664 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8665 ins_cost(VOLATILE_REF_COST);
8666 effect(TEMP_DEF res, KILL cr);
8667 format %{
8668 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8669 %}
8670 ins_encode %{
8671 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8672 Assembler::word, /*acquire*/ true, /*release*/ true,
8673 /*weak*/ false, $res$$Register);
8674 %}
8675 ins_pipe(pipe_slow);
8676 %}
8677
8678 // This pattern is generated automatically from cas.m4.
8679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8680 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8681 predicate(needs_acquiring_load_exclusive(n));
8682 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8683 ins_cost(VOLATILE_REF_COST);
8684 effect(TEMP_DEF res, KILL cr);
8685 format %{
8686 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8687 %}
8688 ins_encode %{
8689 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8690 Assembler::xword, /*acquire*/ true, /*release*/ true,
8691 /*weak*/ false, $res$$Register);
8692 %}
8693 ins_pipe(pipe_slow);
8694 %}
8695
8696 // This pattern is generated automatically from cas.m4.
8697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8698 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8699 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8700 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8701 ins_cost(VOLATILE_REF_COST);
8702 effect(TEMP_DEF res, KILL cr);
8703 format %{
8704 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8705 %}
8706 ins_encode %{
8707 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8708 Assembler::word, /*acquire*/ true, /*release*/ true,
8709 /*weak*/ false, $res$$Register);
8710 %}
8711 ins_pipe(pipe_slow);
8712 %}
8713
8714 // This pattern is generated automatically from cas.m4.
8715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8716 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8717 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8718 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8719 ins_cost(VOLATILE_REF_COST);
8720 effect(TEMP_DEF res, KILL cr);
8721 format %{
8722 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8723 %}
8724 ins_encode %{
8725 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8726 Assembler::xword, /*acquire*/ true, /*release*/ true,
8727 /*weak*/ false, $res$$Register);
8728 %}
8729 ins_pipe(pipe_slow);
8730 %}
8731
8732 // This pattern is generated automatically from cas.m4.
8733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8734 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8735 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8736 ins_cost(2 * VOLATILE_REF_COST);
8737 effect(KILL cr);
8738 format %{
8739 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8740 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8741 %}
8742 ins_encode %{
8743 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8744 Assembler::byte, /*acquire*/ false, /*release*/ true,
8745 /*weak*/ true, noreg);
8746 __ csetw($res$$Register, Assembler::EQ);
8747 %}
8748 ins_pipe(pipe_slow);
8749 %}
8750
8751 // This pattern is generated automatically from cas.m4.
8752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8753 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8754 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8755 ins_cost(2 * VOLATILE_REF_COST);
8756 effect(KILL cr);
8757 format %{
8758 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8759 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8760 %}
8761 ins_encode %{
8762 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8763 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8764 /*weak*/ true, noreg);
8765 __ csetw($res$$Register, Assembler::EQ);
8766 %}
8767 ins_pipe(pipe_slow);
8768 %}
8769
8770 // This pattern is generated automatically from cas.m4.
8771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8772 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8773 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8774 ins_cost(2 * VOLATILE_REF_COST);
8775 effect(KILL cr);
8776 format %{
8777 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8778 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8779 %}
8780 ins_encode %{
8781 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8782 Assembler::word, /*acquire*/ false, /*release*/ true,
8783 /*weak*/ true, noreg);
8784 __ csetw($res$$Register, Assembler::EQ);
8785 %}
8786 ins_pipe(pipe_slow);
8787 %}
8788
8789 // This pattern is generated automatically from cas.m4.
8790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8791 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8792 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8793 ins_cost(2 * VOLATILE_REF_COST);
8794 effect(KILL cr);
8795 format %{
8796 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8797 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8798 %}
8799 ins_encode %{
8800 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8801 Assembler::xword, /*acquire*/ false, /*release*/ true,
8802 /*weak*/ true, noreg);
8803 __ csetw($res$$Register, Assembler::EQ);
8804 %}
8805 ins_pipe(pipe_slow);
8806 %}
8807
8808 // This pattern is generated automatically from cas.m4.
8809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8810 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8811 predicate(n->as_LoadStore()->barrier_data() == 0);
8812 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8813 ins_cost(2 * VOLATILE_REF_COST);
8814 effect(KILL cr);
8815 format %{
8816 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8817 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8818 %}
8819 ins_encode %{
8820 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8821 Assembler::word, /*acquire*/ false, /*release*/ true,
8822 /*weak*/ true, noreg);
8823 __ csetw($res$$Register, Assembler::EQ);
8824 %}
8825 ins_pipe(pipe_slow);
8826 %}
8827
8828 // This pattern is generated automatically from cas.m4.
8829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8830 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8831 predicate(n->as_LoadStore()->barrier_data() == 0);
8832 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8833 ins_cost(2 * VOLATILE_REF_COST);
8834 effect(KILL cr);
8835 format %{
8836 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8837 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8838 %}
8839 ins_encode %{
8840 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8841 Assembler::xword, /*acquire*/ false, /*release*/ true,
8842 /*weak*/ true, noreg);
8843 __ csetw($res$$Register, Assembler::EQ);
8844 %}
8845 ins_pipe(pipe_slow);
8846 %}
8847
8848 // This pattern is generated automatically from cas.m4.
8849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8850 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8851 predicate(needs_acquiring_load_exclusive(n));
8852 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8853 ins_cost(VOLATILE_REF_COST);
8854 effect(KILL cr);
8855 format %{
8856 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8857 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8858 %}
8859 ins_encode %{
8860 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8861 Assembler::byte, /*acquire*/ true, /*release*/ true,
8862 /*weak*/ true, noreg);
8863 __ csetw($res$$Register, Assembler::EQ);
8864 %}
8865 ins_pipe(pipe_slow);
8866 %}
8867
8868 // This pattern is generated automatically from cas.m4.
8869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8870 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8871 predicate(needs_acquiring_load_exclusive(n));
8872 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8873 ins_cost(VOLATILE_REF_COST);
8874 effect(KILL cr);
8875 format %{
8876 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8877 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8878 %}
8879 ins_encode %{
8880 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8881 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8882 /*weak*/ true, noreg);
8883 __ csetw($res$$Register, Assembler::EQ);
8884 %}
8885 ins_pipe(pipe_slow);
8886 %}
8887
8888 // This pattern is generated automatically from cas.m4.
8889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8890 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8891 predicate(needs_acquiring_load_exclusive(n));
8892 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8893 ins_cost(VOLATILE_REF_COST);
8894 effect(KILL cr);
8895 format %{
8896 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8897 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8898 %}
8899 ins_encode %{
8900 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8901 Assembler::word, /*acquire*/ true, /*release*/ true,
8902 /*weak*/ true, noreg);
8903 __ csetw($res$$Register, Assembler::EQ);
8904 %}
8905 ins_pipe(pipe_slow);
8906 %}
8907
8908 // This pattern is generated automatically from cas.m4.
8909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8910 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8911 predicate(needs_acquiring_load_exclusive(n));
8912 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8913 ins_cost(VOLATILE_REF_COST);
8914 effect(KILL cr);
8915 format %{
8916 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8917 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8918 %}
8919 ins_encode %{
8920 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8921 Assembler::xword, /*acquire*/ true, /*release*/ true,
8922 /*weak*/ true, noreg);
8923 __ csetw($res$$Register, Assembler::EQ);
8924 %}
8925 ins_pipe(pipe_slow);
8926 %}
8927
8928 // This pattern is generated automatically from cas.m4.
8929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8930 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8931 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8932 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8933 ins_cost(VOLATILE_REF_COST);
8934 effect(KILL cr);
8935 format %{
8936 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8937 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8938 %}
8939 ins_encode %{
8940 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8941 Assembler::word, /*acquire*/ true, /*release*/ true,
8942 /*weak*/ true, noreg);
8943 __ csetw($res$$Register, Assembler::EQ);
8944 %}
8945 ins_pipe(pipe_slow);
8946 %}
8947
8948 // This pattern is generated automatically from cas.m4.
8949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8950 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8951 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8952 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8953 ins_cost(VOLATILE_REF_COST);
8954 effect(KILL cr);
8955 format %{
8956 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8957 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8958 %}
8959 ins_encode %{
8960 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8961 Assembler::xword, /*acquire*/ true, /*release*/ true,
8962 /*weak*/ true, noreg);
8963 __ csetw($res$$Register, Assembler::EQ);
8964 %}
8965 ins_pipe(pipe_slow);
8966 %}
8967
8968 // END This section of the file is automatically generated. Do not edit --------------
8969 // ---------------------------------------------------------------------
8970
8971 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8972 match(Set prev (GetAndSetI mem newv));
8973 ins_cost(2 * VOLATILE_REF_COST);
8974 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8975 ins_encode %{
8976 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8977 %}
8978 ins_pipe(pipe_serial);
8979 %}
8980
8981 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8982 match(Set prev (GetAndSetL mem newv));
8983 ins_cost(2 * VOLATILE_REF_COST);
8984 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8985 ins_encode %{
8986 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8987 %}
8988 ins_pipe(pipe_serial);
8989 %}
8990
8991 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8992 predicate(n->as_LoadStore()->barrier_data() == 0);
8993 match(Set prev (GetAndSetN mem newv));
8994 ins_cost(2 * VOLATILE_REF_COST);
8995 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8996 ins_encode %{
8997 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8998 %}
8999 ins_pipe(pipe_serial);
9000 %}
9001
9002 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9003 predicate(n->as_LoadStore()->barrier_data() == 0);
9004 match(Set prev (GetAndSetP mem newv));
9005 ins_cost(2 * VOLATILE_REF_COST);
9006 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9007 ins_encode %{
9008 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9009 %}
9010 ins_pipe(pipe_serial);
9011 %}
9012
9013 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9014 predicate(needs_acquiring_load_exclusive(n));
9015 match(Set prev (GetAndSetI mem newv));
9016 ins_cost(VOLATILE_REF_COST);
9017 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9018 ins_encode %{
9019 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9020 %}
9021 ins_pipe(pipe_serial);
9022 %}
9023
9024 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9025 predicate(needs_acquiring_load_exclusive(n));
9026 match(Set prev (GetAndSetL mem newv));
9027 ins_cost(VOLATILE_REF_COST);
9028 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9029 ins_encode %{
9030 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9031 %}
9032 ins_pipe(pipe_serial);
9033 %}
9034
9035 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9036 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9037 match(Set prev (GetAndSetN mem newv));
9038 ins_cost(VOLATILE_REF_COST);
9039 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9040 ins_encode %{
9041 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9042 %}
9043 ins_pipe(pipe_serial);
9044 %}
9045
9046 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9047 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9048 match(Set prev (GetAndSetP mem newv));
9049 ins_cost(VOLATILE_REF_COST);
9050 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9051 ins_encode %{
9052 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9053 %}
9054 ins_pipe(pipe_serial);
9055 %}
9056
9057
9058 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9059 match(Set newval (GetAndAddL mem incr));
9060 ins_cost(2 * VOLATILE_REF_COST + 1);
9061 format %{ "get_and_addL $newval, [$mem], $incr" %}
9062 ins_encode %{
9063 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9064 %}
9065 ins_pipe(pipe_serial);
9066 %}
9067
9068 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9069 predicate(n->as_LoadStore()->result_not_used());
9070 match(Set dummy (GetAndAddL mem incr));
9071 ins_cost(2 * VOLATILE_REF_COST);
9072 format %{ "get_and_addL [$mem], $incr" %}
9073 ins_encode %{
9074 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9075 %}
9076 ins_pipe(pipe_serial);
9077 %}
9078
9079 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9080 match(Set newval (GetAndAddL mem incr));
9081 ins_cost(2 * VOLATILE_REF_COST + 1);
9082 format %{ "get_and_addL $newval, [$mem], $incr" %}
9083 ins_encode %{
9084 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9085 %}
9086 ins_pipe(pipe_serial);
9087 %}
9088
9089 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9090 predicate(n->as_LoadStore()->result_not_used());
9091 match(Set dummy (GetAndAddL mem incr));
9092 ins_cost(2 * VOLATILE_REF_COST);
9093 format %{ "get_and_addL [$mem], $incr" %}
9094 ins_encode %{
9095 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9096 %}
9097 ins_pipe(pipe_serial);
9098 %}
9099
9100 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9101 match(Set newval (GetAndAddI mem incr));
9102 ins_cost(2 * VOLATILE_REF_COST + 1);
9103 format %{ "get_and_addI $newval, [$mem], $incr" %}
9104 ins_encode %{
9105 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9106 %}
9107 ins_pipe(pipe_serial);
9108 %}
9109
9110 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9111 predicate(n->as_LoadStore()->result_not_used());
9112 match(Set dummy (GetAndAddI mem incr));
9113 ins_cost(2 * VOLATILE_REF_COST);
9114 format %{ "get_and_addI [$mem], $incr" %}
9115 ins_encode %{
9116 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9117 %}
9118 ins_pipe(pipe_serial);
9119 %}
9120
9121 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9122 match(Set newval (GetAndAddI mem incr));
9123 ins_cost(2 * VOLATILE_REF_COST + 1);
9124 format %{ "get_and_addI $newval, [$mem], $incr" %}
9125 ins_encode %{
9126 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9127 %}
9128 ins_pipe(pipe_serial);
9129 %}
9130
9131 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9132 predicate(n->as_LoadStore()->result_not_used());
9133 match(Set dummy (GetAndAddI mem incr));
9134 ins_cost(2 * VOLATILE_REF_COST);
9135 format %{ "get_and_addI [$mem], $incr" %}
9136 ins_encode %{
9137 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9138 %}
9139 ins_pipe(pipe_serial);
9140 %}
9141
9142 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9143 predicate(needs_acquiring_load_exclusive(n));
9144 match(Set newval (GetAndAddL mem incr));
9145 ins_cost(VOLATILE_REF_COST + 1);
9146 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9147 ins_encode %{
9148 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9149 %}
9150 ins_pipe(pipe_serial);
9151 %}
9152
9153 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9154 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9155 match(Set dummy (GetAndAddL mem incr));
9156 ins_cost(VOLATILE_REF_COST);
9157 format %{ "get_and_addL_acq [$mem], $incr" %}
9158 ins_encode %{
9159 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9160 %}
9161 ins_pipe(pipe_serial);
9162 %}
9163
9164 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9165 predicate(needs_acquiring_load_exclusive(n));
9166 match(Set newval (GetAndAddL mem incr));
9167 ins_cost(VOLATILE_REF_COST + 1);
9168 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9169 ins_encode %{
9170 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9171 %}
9172 ins_pipe(pipe_serial);
9173 %}
9174
9175 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9176 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9177 match(Set dummy (GetAndAddL mem incr));
9178 ins_cost(VOLATILE_REF_COST);
9179 format %{ "get_and_addL_acq [$mem], $incr" %}
9180 ins_encode %{
9181 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9182 %}
9183 ins_pipe(pipe_serial);
9184 %}
9185
9186 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9187 predicate(needs_acquiring_load_exclusive(n));
9188 match(Set newval (GetAndAddI mem incr));
9189 ins_cost(VOLATILE_REF_COST + 1);
9190 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9191 ins_encode %{
9192 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9193 %}
9194 ins_pipe(pipe_serial);
9195 %}
9196
9197 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9198 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9199 match(Set dummy (GetAndAddI mem incr));
9200 ins_cost(VOLATILE_REF_COST);
9201 format %{ "get_and_addI_acq [$mem], $incr" %}
9202 ins_encode %{
9203 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9204 %}
9205 ins_pipe(pipe_serial);
9206 %}
9207
9208 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9209 predicate(needs_acquiring_load_exclusive(n));
9210 match(Set newval (GetAndAddI mem incr));
9211 ins_cost(VOLATILE_REF_COST + 1);
9212 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9213 ins_encode %{
9214 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9215 %}
9216 ins_pipe(pipe_serial);
9217 %}
9218
9219 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9220 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9221 match(Set dummy (GetAndAddI mem incr));
9222 ins_cost(VOLATILE_REF_COST);
9223 format %{ "get_and_addI_acq [$mem], $incr" %}
9224 ins_encode %{
9225 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9226 %}
9227 ins_pipe(pipe_serial);
9228 %}
9229
9230 // Manifest a CmpU result in an integer register.
9231 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9232 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9233 %{
9234 match(Set dst (CmpU3 src1 src2));
9235 effect(KILL flags);
9236
9237 ins_cost(INSN_COST * 3);
9238 format %{
9239 "cmpw $src1, $src2\n\t"
9240 "csetw $dst, ne\n\t"
9241 "cnegw $dst, lo\t# CmpU3(reg)"
9242 %}
9243 ins_encode %{
9244 __ cmpw($src1$$Register, $src2$$Register);
9245 __ csetw($dst$$Register, Assembler::NE);
9246 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9247 %}
9248
9249 ins_pipe(pipe_class_default);
9250 %}
9251
9252 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9253 %{
9254 match(Set dst (CmpU3 src1 src2));
9255 effect(KILL flags);
9256
9257 ins_cost(INSN_COST * 3);
9258 format %{
9259 "subsw zr, $src1, $src2\n\t"
9260 "csetw $dst, ne\n\t"
9261 "cnegw $dst, lo\t# CmpU3(imm)"
9262 %}
9263 ins_encode %{
9264 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9265 __ csetw($dst$$Register, Assembler::NE);
9266 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9267 %}
9268
9269 ins_pipe(pipe_class_default);
9270 %}
9271
9272 // Manifest a CmpUL result in an integer register.
9273 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9274 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9275 %{
9276 match(Set dst (CmpUL3 src1 src2));
9277 effect(KILL flags);
9278
9279 ins_cost(INSN_COST * 3);
9280 format %{
9281 "cmp $src1, $src2\n\t"
9282 "csetw $dst, ne\n\t"
9283 "cnegw $dst, lo\t# CmpUL3(reg)"
9284 %}
9285 ins_encode %{
9286 __ cmp($src1$$Register, $src2$$Register);
9287 __ csetw($dst$$Register, Assembler::NE);
9288 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9289 %}
9290
9291 ins_pipe(pipe_class_default);
9292 %}
9293
9294 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9295 %{
9296 match(Set dst (CmpUL3 src1 src2));
9297 effect(KILL flags);
9298
9299 ins_cost(INSN_COST * 3);
9300 format %{
9301 "subs zr, $src1, $src2\n\t"
9302 "csetw $dst, ne\n\t"
9303 "cnegw $dst, lo\t# CmpUL3(imm)"
9304 %}
9305 ins_encode %{
9306 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9307 __ csetw($dst$$Register, Assembler::NE);
9308 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9309 %}
9310
9311 ins_pipe(pipe_class_default);
9312 %}
9313
9314 // Manifest a CmpL result in an integer register.
9315 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9316 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9317 %{
9318 match(Set dst (CmpL3 src1 src2));
9319 effect(KILL flags);
9320
9321 ins_cost(INSN_COST * 3);
9322 format %{
9323 "cmp $src1, $src2\n\t"
9324 "csetw $dst, ne\n\t"
9325 "cnegw $dst, lt\t# CmpL3(reg)"
9326 %}
9327 ins_encode %{
9328 __ cmp($src1$$Register, $src2$$Register);
9329 __ csetw($dst$$Register, Assembler::NE);
9330 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9331 %}
9332
9333 ins_pipe(pipe_class_default);
9334 %}
9335
9336 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9337 %{
9338 match(Set dst (CmpL3 src1 src2));
9339 effect(KILL flags);
9340
9341 ins_cost(INSN_COST * 3);
9342 format %{
9343 "subs zr, $src1, $src2\n\t"
9344 "csetw $dst, ne\n\t"
9345 "cnegw $dst, lt\t# CmpL3(imm)"
9346 %}
9347 ins_encode %{
9348 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9349 __ csetw($dst$$Register, Assembler::NE);
9350 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9351 %}
9352
9353 ins_pipe(pipe_class_default);
9354 %}
9355
9356 // ============================================================================
9357 // Conditional Move Instructions
9358
9359 // n.b. we have identical rules for both a signed compare op (cmpOp)
9360 // and an unsigned compare op (cmpOpU). it would be nice if we could
9361 // define an op class which merged both inputs and use it to type the
9362 // argument to a single rule. unfortunatelyt his fails because the
9363 // opclass does not live up to the COND_INTER interface of its
9364 // component operands. When the generic code tries to negate the
9365 // operand it ends up running the generci Machoper::negate method
9366 // which throws a ShouldNotHappen. So, we have to provide two flavours
9367 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9368
9369 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9370 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9371
9372 ins_cost(INSN_COST * 2);
9373 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9374
9375 ins_encode %{
9376 __ cselw(as_Register($dst$$reg),
9377 as_Register($src2$$reg),
9378 as_Register($src1$$reg),
9379 (Assembler::Condition)$cmp$$cmpcode);
9380 %}
9381
9382 ins_pipe(icond_reg_reg);
9383 %}
9384
9385 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9386 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9387
9388 ins_cost(INSN_COST * 2);
9389 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9390
9391 ins_encode %{
9392 __ cselw(as_Register($dst$$reg),
9393 as_Register($src2$$reg),
9394 as_Register($src1$$reg),
9395 (Assembler::Condition)$cmp$$cmpcode);
9396 %}
9397
9398 ins_pipe(icond_reg_reg);
9399 %}
9400
9401 // special cases where one arg is zero
9402
9403 // n.b. this is selected in preference to the rule above because it
9404 // avoids loading constant 0 into a source register
9405
9406 // TODO
9407 // we ought only to be able to cull one of these variants as the ideal
9408 // transforms ought always to order the zero consistently (to left/right?)
9409
9410 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9411 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9412
9413 ins_cost(INSN_COST * 2);
9414 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9415
9416 ins_encode %{
9417 __ cselw(as_Register($dst$$reg),
9418 as_Register($src$$reg),
9419 zr,
9420 (Assembler::Condition)$cmp$$cmpcode);
9421 %}
9422
9423 ins_pipe(icond_reg);
9424 %}
9425
9426 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9427 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9428
9429 ins_cost(INSN_COST * 2);
9430 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9431
9432 ins_encode %{
9433 __ cselw(as_Register($dst$$reg),
9434 as_Register($src$$reg),
9435 zr,
9436 (Assembler::Condition)$cmp$$cmpcode);
9437 %}
9438
9439 ins_pipe(icond_reg);
9440 %}
9441
9442 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9443 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9444
9445 ins_cost(INSN_COST * 2);
9446 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9447
9448 ins_encode %{
9449 __ cselw(as_Register($dst$$reg),
9450 zr,
9451 as_Register($src$$reg),
9452 (Assembler::Condition)$cmp$$cmpcode);
9453 %}
9454
9455 ins_pipe(icond_reg);
9456 %}
9457
9458 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9459 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9460
9461 ins_cost(INSN_COST * 2);
9462 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9463
9464 ins_encode %{
9465 __ cselw(as_Register($dst$$reg),
9466 zr,
9467 as_Register($src$$reg),
9468 (Assembler::Condition)$cmp$$cmpcode);
9469 %}
9470
9471 ins_pipe(icond_reg);
9472 %}
9473
9474 // special case for creating a boolean 0 or 1
9475
9476 // n.b. this is selected in preference to the rule above because it
9477 // avoids loading constants 0 and 1 into a source register
9478
9479 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9480 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9481
9482 ins_cost(INSN_COST * 2);
9483 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9484
9485 ins_encode %{
9486 // equivalently
9487 // cset(as_Register($dst$$reg),
9488 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9489 __ csincw(as_Register($dst$$reg),
9490 zr,
9491 zr,
9492 (Assembler::Condition)$cmp$$cmpcode);
9493 %}
9494
9495 ins_pipe(icond_none);
9496 %}
9497
9498 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9499 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9500
9501 ins_cost(INSN_COST * 2);
9502 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9503
9504 ins_encode %{
9505 // equivalently
9506 // cset(as_Register($dst$$reg),
9507 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9508 __ csincw(as_Register($dst$$reg),
9509 zr,
9510 zr,
9511 (Assembler::Condition)$cmp$$cmpcode);
9512 %}
9513
9514 ins_pipe(icond_none);
9515 %}
9516
9517 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9518 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9519
9520 ins_cost(INSN_COST * 2);
9521 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9522
9523 ins_encode %{
9524 __ csel(as_Register($dst$$reg),
9525 as_Register($src2$$reg),
9526 as_Register($src1$$reg),
9527 (Assembler::Condition)$cmp$$cmpcode);
9528 %}
9529
9530 ins_pipe(icond_reg_reg);
9531 %}
9532
9533 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9534 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9535
9536 ins_cost(INSN_COST * 2);
9537 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9538
9539 ins_encode %{
9540 __ csel(as_Register($dst$$reg),
9541 as_Register($src2$$reg),
9542 as_Register($src1$$reg),
9543 (Assembler::Condition)$cmp$$cmpcode);
9544 %}
9545
9546 ins_pipe(icond_reg_reg);
9547 %}
9548
9549 // special cases where one arg is zero
9550
9551 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9552 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9553
9554 ins_cost(INSN_COST * 2);
9555 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9556
9557 ins_encode %{
9558 __ csel(as_Register($dst$$reg),
9559 zr,
9560 as_Register($src$$reg),
9561 (Assembler::Condition)$cmp$$cmpcode);
9562 %}
9563
9564 ins_pipe(icond_reg);
9565 %}
9566
9567 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9568 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9569
9570 ins_cost(INSN_COST * 2);
9571 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9572
9573 ins_encode %{
9574 __ csel(as_Register($dst$$reg),
9575 zr,
9576 as_Register($src$$reg),
9577 (Assembler::Condition)$cmp$$cmpcode);
9578 %}
9579
9580 ins_pipe(icond_reg);
9581 %}
9582
9583 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9584 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9585
9586 ins_cost(INSN_COST * 2);
9587 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9588
9589 ins_encode %{
9590 __ csel(as_Register($dst$$reg),
9591 as_Register($src$$reg),
9592 zr,
9593 (Assembler::Condition)$cmp$$cmpcode);
9594 %}
9595
9596 ins_pipe(icond_reg);
9597 %}
9598
9599 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9600 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9601
9602 ins_cost(INSN_COST * 2);
9603 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9604
9605 ins_encode %{
9606 __ csel(as_Register($dst$$reg),
9607 as_Register($src$$reg),
9608 zr,
9609 (Assembler::Condition)$cmp$$cmpcode);
9610 %}
9611
9612 ins_pipe(icond_reg);
9613 %}
9614
9615 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9616 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9617
9618 ins_cost(INSN_COST * 2);
9619 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9620
9621 ins_encode %{
9622 __ csel(as_Register($dst$$reg),
9623 as_Register($src2$$reg),
9624 as_Register($src1$$reg),
9625 (Assembler::Condition)$cmp$$cmpcode);
9626 %}
9627
9628 ins_pipe(icond_reg_reg);
9629 %}
9630
9631 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9632 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9633
9634 ins_cost(INSN_COST * 2);
9635 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9636
9637 ins_encode %{
9638 __ csel(as_Register($dst$$reg),
9639 as_Register($src2$$reg),
9640 as_Register($src1$$reg),
9641 (Assembler::Condition)$cmp$$cmpcode);
9642 %}
9643
9644 ins_pipe(icond_reg_reg);
9645 %}
9646
9647 // special cases where one arg is zero
9648
9649 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9650 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9651
9652 ins_cost(INSN_COST * 2);
9653 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9654
9655 ins_encode %{
9656 __ csel(as_Register($dst$$reg),
9657 zr,
9658 as_Register($src$$reg),
9659 (Assembler::Condition)$cmp$$cmpcode);
9660 %}
9661
9662 ins_pipe(icond_reg);
9663 %}
9664
9665 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9666 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9667
9668 ins_cost(INSN_COST * 2);
9669 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9670
9671 ins_encode %{
9672 __ csel(as_Register($dst$$reg),
9673 zr,
9674 as_Register($src$$reg),
9675 (Assembler::Condition)$cmp$$cmpcode);
9676 %}
9677
9678 ins_pipe(icond_reg);
9679 %}
9680
9681 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9682 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9683
9684 ins_cost(INSN_COST * 2);
9685 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9686
9687 ins_encode %{
9688 __ csel(as_Register($dst$$reg),
9689 as_Register($src$$reg),
9690 zr,
9691 (Assembler::Condition)$cmp$$cmpcode);
9692 %}
9693
9694 ins_pipe(icond_reg);
9695 %}
9696
9697 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9698 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9699
9700 ins_cost(INSN_COST * 2);
9701 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9702
9703 ins_encode %{
9704 __ csel(as_Register($dst$$reg),
9705 as_Register($src$$reg),
9706 zr,
9707 (Assembler::Condition)$cmp$$cmpcode);
9708 %}
9709
9710 ins_pipe(icond_reg);
9711 %}
9712
9713 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9714 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9715
9716 ins_cost(INSN_COST * 2);
9717 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9718
9719 ins_encode %{
9720 __ cselw(as_Register($dst$$reg),
9721 as_Register($src2$$reg),
9722 as_Register($src1$$reg),
9723 (Assembler::Condition)$cmp$$cmpcode);
9724 %}
9725
9726 ins_pipe(icond_reg_reg);
9727 %}
9728
9729 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9730 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9731
9732 ins_cost(INSN_COST * 2);
9733 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9734
9735 ins_encode %{
9736 __ cselw(as_Register($dst$$reg),
9737 as_Register($src2$$reg),
9738 as_Register($src1$$reg),
9739 (Assembler::Condition)$cmp$$cmpcode);
9740 %}
9741
9742 ins_pipe(icond_reg_reg);
9743 %}
9744
9745 // special cases where one arg is zero
9746
9747 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9748 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9749
9750 ins_cost(INSN_COST * 2);
9751 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9752
9753 ins_encode %{
9754 __ cselw(as_Register($dst$$reg),
9755 zr,
9756 as_Register($src$$reg),
9757 (Assembler::Condition)$cmp$$cmpcode);
9758 %}
9759
9760 ins_pipe(icond_reg);
9761 %}
9762
9763 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9764 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9765
9766 ins_cost(INSN_COST * 2);
9767 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9768
9769 ins_encode %{
9770 __ cselw(as_Register($dst$$reg),
9771 zr,
9772 as_Register($src$$reg),
9773 (Assembler::Condition)$cmp$$cmpcode);
9774 %}
9775
9776 ins_pipe(icond_reg);
9777 %}
9778
9779 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9780 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9781
9782 ins_cost(INSN_COST * 2);
9783 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9784
9785 ins_encode %{
9786 __ cselw(as_Register($dst$$reg),
9787 as_Register($src$$reg),
9788 zr,
9789 (Assembler::Condition)$cmp$$cmpcode);
9790 %}
9791
9792 ins_pipe(icond_reg);
9793 %}
9794
9795 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9796 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9797
9798 ins_cost(INSN_COST * 2);
9799 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9800
9801 ins_encode %{
9802 __ cselw(as_Register($dst$$reg),
9803 as_Register($src$$reg),
9804 zr,
9805 (Assembler::Condition)$cmp$$cmpcode);
9806 %}
9807
9808 ins_pipe(icond_reg);
9809 %}
9810
9811 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9812 %{
9813 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9814
9815 ins_cost(INSN_COST * 3);
9816
9817 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9818 ins_encode %{
9819 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9820 __ fcsels(as_FloatRegister($dst$$reg),
9821 as_FloatRegister($src2$$reg),
9822 as_FloatRegister($src1$$reg),
9823 cond);
9824 %}
9825
9826 ins_pipe(fp_cond_reg_reg_s);
9827 %}
9828
9829 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9830 %{
9831 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9832
9833 ins_cost(INSN_COST * 3);
9834
9835 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9836 ins_encode %{
9837 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9838 __ fcsels(as_FloatRegister($dst$$reg),
9839 as_FloatRegister($src2$$reg),
9840 as_FloatRegister($src1$$reg),
9841 cond);
9842 %}
9843
9844 ins_pipe(fp_cond_reg_reg_s);
9845 %}
9846
9847 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9848 %{
9849 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9850
9851 ins_cost(INSN_COST * 3);
9852
9853 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9854 ins_encode %{
9855 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9856 __ fcseld(as_FloatRegister($dst$$reg),
9857 as_FloatRegister($src2$$reg),
9858 as_FloatRegister($src1$$reg),
9859 cond);
9860 %}
9861
9862 ins_pipe(fp_cond_reg_reg_d);
9863 %}
9864
9865 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9866 %{
9867 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9868
9869 ins_cost(INSN_COST * 3);
9870
9871 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9872 ins_encode %{
9873 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9874 __ fcseld(as_FloatRegister($dst$$reg),
9875 as_FloatRegister($src2$$reg),
9876 as_FloatRegister($src1$$reg),
9877 cond);
9878 %}
9879
9880 ins_pipe(fp_cond_reg_reg_d);
9881 %}
9882
9883 // ============================================================================
9884 // Arithmetic Instructions
9885 //
9886
9887 // Integer Addition
9888
9889 // TODO
9890 // these currently employ operations which do not set CR and hence are
9891 // not flagged as killing CR but we would like to isolate the cases
9892 // where we want to set flags from those where we don't. need to work
9893 // out how to do that.
9894
9895 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9896 match(Set dst (AddI src1 src2));
9897
9898 ins_cost(INSN_COST);
9899 format %{ "addw $dst, $src1, $src2" %}
9900
9901 ins_encode %{
9902 __ addw(as_Register($dst$$reg),
9903 as_Register($src1$$reg),
9904 as_Register($src2$$reg));
9905 %}
9906
9907 ins_pipe(ialu_reg_reg);
9908 %}
9909
9910 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9911 match(Set dst (AddI src1 src2));
9912
9913 ins_cost(INSN_COST);
9914 format %{ "addw $dst, $src1, $src2" %}
9915
9916 // use opcode to indicate that this is an add not a sub
9917 opcode(0x0);
9918
9919 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9920
9921 ins_pipe(ialu_reg_imm);
9922 %}
9923
9924 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9925 match(Set dst (AddI (ConvL2I src1) src2));
9926
9927 ins_cost(INSN_COST);
9928 format %{ "addw $dst, $src1, $src2" %}
9929
9930 // use opcode to indicate that this is an add not a sub
9931 opcode(0x0);
9932
9933 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9934
9935 ins_pipe(ialu_reg_imm);
9936 %}
9937
9938 // Pointer Addition
9939 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9940 match(Set dst (AddP src1 src2));
9941
9942 ins_cost(INSN_COST);
9943 format %{ "add $dst, $src1, $src2\t# ptr" %}
9944
9945 ins_encode %{
9946 __ add(as_Register($dst$$reg),
9947 as_Register($src1$$reg),
9948 as_Register($src2$$reg));
9949 %}
9950
9951 ins_pipe(ialu_reg_reg);
9952 %}
9953
9954 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9955 match(Set dst (AddP src1 (ConvI2L src2)));
9956
9957 ins_cost(1.9 * INSN_COST);
9958 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9959
9960 ins_encode %{
9961 __ add(as_Register($dst$$reg),
9962 as_Register($src1$$reg),
9963 as_Register($src2$$reg), ext::sxtw);
9964 %}
9965
9966 ins_pipe(ialu_reg_reg);
9967 %}
9968
9969 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9970 match(Set dst (AddP src1 (LShiftL src2 scale)));
9971
9972 ins_cost(1.9 * INSN_COST);
9973 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9974
9975 ins_encode %{
9976 __ lea(as_Register($dst$$reg),
9977 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9978 Address::lsl($scale$$constant)));
9979 %}
9980
9981 ins_pipe(ialu_reg_reg_shift);
9982 %}
9983
9984 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9985 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9986
9987 ins_cost(1.9 * INSN_COST);
9988 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9989
9990 ins_encode %{
9991 __ lea(as_Register($dst$$reg),
9992 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9993 Address::sxtw($scale$$constant)));
9994 %}
9995
9996 ins_pipe(ialu_reg_reg_shift);
9997 %}
9998
9999 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10000 match(Set dst (LShiftL (ConvI2L src) scale));
10001
10002 ins_cost(INSN_COST);
10003 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10004
10005 ins_encode %{
10006 __ sbfiz(as_Register($dst$$reg),
10007 as_Register($src$$reg),
10008 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10009 %}
10010
10011 ins_pipe(ialu_reg_shift);
10012 %}
10013
10014 // Pointer Immediate Addition
10015 // n.b. this needs to be more expensive than using an indirect memory
10016 // operand
10017 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10018 match(Set dst (AddP src1 src2));
10019
10020 ins_cost(INSN_COST);
10021 format %{ "add $dst, $src1, $src2\t# ptr" %}
10022
10023 // use opcode to indicate that this is an add not a sub
10024 opcode(0x0);
10025
10026 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10027
10028 ins_pipe(ialu_reg_imm);
10029 %}
10030
10031 // Long Addition
10032 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10033
10034 match(Set dst (AddL src1 src2));
10035
10036 ins_cost(INSN_COST);
10037 format %{ "add $dst, $src1, $src2" %}
10038
10039 ins_encode %{
10040 __ add(as_Register($dst$$reg),
10041 as_Register($src1$$reg),
10042 as_Register($src2$$reg));
10043 %}
10044
10045 ins_pipe(ialu_reg_reg);
10046 %}
10047
10048 // No constant pool entries requiredLong Immediate Addition.
10049 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10050 match(Set dst (AddL src1 src2));
10051
10052 ins_cost(INSN_COST);
10053 format %{ "add $dst, $src1, $src2" %}
10054
10055 // use opcode to indicate that this is an add not a sub
10056 opcode(0x0);
10057
10058 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10059
10060 ins_pipe(ialu_reg_imm);
10061 %}
10062
10063 // Integer Subtraction
10064 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10065 match(Set dst (SubI src1 src2));
10066
10067 ins_cost(INSN_COST);
10068 format %{ "subw $dst, $src1, $src2" %}
10069
10070 ins_encode %{
10071 __ subw(as_Register($dst$$reg),
10072 as_Register($src1$$reg),
10073 as_Register($src2$$reg));
10074 %}
10075
10076 ins_pipe(ialu_reg_reg);
10077 %}
10078
10079 // Immediate Subtraction
10080 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10081 match(Set dst (SubI src1 src2));
10082
10083 ins_cost(INSN_COST);
10084 format %{ "subw $dst, $src1, $src2" %}
10085
10086 // use opcode to indicate that this is a sub not an add
10087 opcode(0x1);
10088
10089 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10090
10091 ins_pipe(ialu_reg_imm);
10092 %}
10093
10094 // Long Subtraction
10095 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10096
10097 match(Set dst (SubL src1 src2));
10098
10099 ins_cost(INSN_COST);
10100 format %{ "sub $dst, $src1, $src2" %}
10101
10102 ins_encode %{
10103 __ sub(as_Register($dst$$reg),
10104 as_Register($src1$$reg),
10105 as_Register($src2$$reg));
10106 %}
10107
10108 ins_pipe(ialu_reg_reg);
10109 %}
10110
10111 // No constant pool entries requiredLong Immediate Subtraction.
10112 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10113 match(Set dst (SubL src1 src2));
10114
10115 ins_cost(INSN_COST);
10116 format %{ "sub$dst, $src1, $src2" %}
10117
10118 // use opcode to indicate that this is a sub not an add
10119 opcode(0x1);
10120
10121 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10122
10123 ins_pipe(ialu_reg_imm);
10124 %}
10125
10126 // Integer Negation (special case for sub)
10127
10128 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10129 match(Set dst (SubI zero src));
10130
10131 ins_cost(INSN_COST);
10132 format %{ "negw $dst, $src\t# int" %}
10133
10134 ins_encode %{
10135 __ negw(as_Register($dst$$reg),
10136 as_Register($src$$reg));
10137 %}
10138
10139 ins_pipe(ialu_reg);
10140 %}
10141
10142 // Long Negation
10143
10144 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10145 match(Set dst (SubL zero src));
10146
10147 ins_cost(INSN_COST);
10148 format %{ "neg $dst, $src\t# long" %}
10149
10150 ins_encode %{
10151 __ neg(as_Register($dst$$reg),
10152 as_Register($src$$reg));
10153 %}
10154
10155 ins_pipe(ialu_reg);
10156 %}
10157
10158 // Integer Multiply
10159
10160 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10161 match(Set dst (MulI src1 src2));
10162
10163 ins_cost(INSN_COST * 3);
10164 format %{ "mulw $dst, $src1, $src2" %}
10165
10166 ins_encode %{
10167 __ mulw(as_Register($dst$$reg),
10168 as_Register($src1$$reg),
10169 as_Register($src2$$reg));
10170 %}
10171
10172 ins_pipe(imul_reg_reg);
10173 %}
10174
10175 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10176 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10177
10178 ins_cost(INSN_COST * 3);
10179 format %{ "smull $dst, $src1, $src2" %}
10180
10181 ins_encode %{
10182 __ smull(as_Register($dst$$reg),
10183 as_Register($src1$$reg),
10184 as_Register($src2$$reg));
10185 %}
10186
10187 ins_pipe(imul_reg_reg);
10188 %}
10189
10190 // Long Multiply
10191
10192 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10193 match(Set dst (MulL src1 src2));
10194
10195 ins_cost(INSN_COST * 5);
10196 format %{ "mul $dst, $src1, $src2" %}
10197
10198 ins_encode %{
10199 __ mul(as_Register($dst$$reg),
10200 as_Register($src1$$reg),
10201 as_Register($src2$$reg));
10202 %}
10203
10204 ins_pipe(lmul_reg_reg);
10205 %}
10206
10207 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10208 %{
10209 match(Set dst (MulHiL src1 src2));
10210
10211 ins_cost(INSN_COST * 7);
10212 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10213
10214 ins_encode %{
10215 __ smulh(as_Register($dst$$reg),
10216 as_Register($src1$$reg),
10217 as_Register($src2$$reg));
10218 %}
10219
10220 ins_pipe(lmul_reg_reg);
10221 %}
10222
10223 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10224 %{
10225 match(Set dst (UMulHiL src1 src2));
10226
10227 ins_cost(INSN_COST * 7);
10228 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10229
10230 ins_encode %{
10231 __ umulh(as_Register($dst$$reg),
10232 as_Register($src1$$reg),
10233 as_Register($src2$$reg));
10234 %}
10235
10236 ins_pipe(lmul_reg_reg);
10237 %}
10238
10239 // Combined Integer Multiply & Add/Sub
10240
10241 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10242 match(Set dst (AddI src3 (MulI src1 src2)));
10243
10244 ins_cost(INSN_COST * 3);
10245 format %{ "madd $dst, $src1, $src2, $src3" %}
10246
10247 ins_encode %{
10248 __ maddw(as_Register($dst$$reg),
10249 as_Register($src1$$reg),
10250 as_Register($src2$$reg),
10251 as_Register($src3$$reg));
10252 %}
10253
10254 ins_pipe(imac_reg_reg);
10255 %}
10256
10257 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10258 match(Set dst (SubI src3 (MulI src1 src2)));
10259
10260 ins_cost(INSN_COST * 3);
10261 format %{ "msub $dst, $src1, $src2, $src3" %}
10262
10263 ins_encode %{
10264 __ msubw(as_Register($dst$$reg),
10265 as_Register($src1$$reg),
10266 as_Register($src2$$reg),
10267 as_Register($src3$$reg));
10268 %}
10269
10270 ins_pipe(imac_reg_reg);
10271 %}
10272
10273 // Combined Integer Multiply & Neg
10274
10275 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10276 match(Set dst (MulI (SubI zero src1) src2));
10277
10278 ins_cost(INSN_COST * 3);
10279 format %{ "mneg $dst, $src1, $src2" %}
10280
10281 ins_encode %{
10282 __ mnegw(as_Register($dst$$reg),
10283 as_Register($src1$$reg),
10284 as_Register($src2$$reg));
10285 %}
10286
10287 ins_pipe(imac_reg_reg);
10288 %}
10289
10290 // Combined Long Multiply & Add/Sub
10291
10292 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10293 match(Set dst (AddL src3 (MulL src1 src2)));
10294
10295 ins_cost(INSN_COST * 5);
10296 format %{ "madd $dst, $src1, $src2, $src3" %}
10297
10298 ins_encode %{
10299 __ madd(as_Register($dst$$reg),
10300 as_Register($src1$$reg),
10301 as_Register($src2$$reg),
10302 as_Register($src3$$reg));
10303 %}
10304
10305 ins_pipe(lmac_reg_reg);
10306 %}
10307
10308 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10309 match(Set dst (SubL src3 (MulL src1 src2)));
10310
10311 ins_cost(INSN_COST * 5);
10312 format %{ "msub $dst, $src1, $src2, $src3" %}
10313
10314 ins_encode %{
10315 __ msub(as_Register($dst$$reg),
10316 as_Register($src1$$reg),
10317 as_Register($src2$$reg),
10318 as_Register($src3$$reg));
10319 %}
10320
10321 ins_pipe(lmac_reg_reg);
10322 %}
10323
10324 // Combined Long Multiply & Neg
10325
10326 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10327 match(Set dst (MulL (SubL zero src1) src2));
10328
10329 ins_cost(INSN_COST * 5);
10330 format %{ "mneg $dst, $src1, $src2" %}
10331
10332 ins_encode %{
10333 __ mneg(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 as_Register($src2$$reg));
10336 %}
10337
10338 ins_pipe(lmac_reg_reg);
10339 %}
10340
10341 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10342
10343 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10344 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10345
10346 ins_cost(INSN_COST * 3);
10347 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10348
10349 ins_encode %{
10350 __ smaddl(as_Register($dst$$reg),
10351 as_Register($src1$$reg),
10352 as_Register($src2$$reg),
10353 as_Register($src3$$reg));
10354 %}
10355
10356 ins_pipe(imac_reg_reg);
10357 %}
10358
10359 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10360 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10361
10362 ins_cost(INSN_COST * 3);
10363 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10364
10365 ins_encode %{
10366 __ smsubl(as_Register($dst$$reg),
10367 as_Register($src1$$reg),
10368 as_Register($src2$$reg),
10369 as_Register($src3$$reg));
10370 %}
10371
10372 ins_pipe(imac_reg_reg);
10373 %}
10374
10375 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10376 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10377
10378 ins_cost(INSN_COST * 3);
10379 format %{ "smnegl $dst, $src1, $src2" %}
10380
10381 ins_encode %{
10382 __ smnegl(as_Register($dst$$reg),
10383 as_Register($src1$$reg),
10384 as_Register($src2$$reg));
10385 %}
10386
10387 ins_pipe(imac_reg_reg);
10388 %}
10389
10390 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10391
10392 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10393 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10394
10395 ins_cost(INSN_COST * 5);
10396 format %{ "mulw rscratch1, $src1, $src2\n\t"
10397 "maddw $dst, $src3, $src4, rscratch1" %}
10398
10399 ins_encode %{
10400 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10401 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10402
10403 ins_pipe(imac_reg_reg);
10404 %}
10405
10406 // Integer Divide
10407
10408 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10409 match(Set dst (DivI src1 src2));
10410
10411 ins_cost(INSN_COST * 19);
10412 format %{ "sdivw $dst, $src1, $src2" %}
10413
10414 ins_encode(aarch64_enc_divw(dst, src1, src2));
10415 ins_pipe(idiv_reg_reg);
10416 %}
10417
10418 // Long Divide
10419
10420 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10421 match(Set dst (DivL src1 src2));
10422
10423 ins_cost(INSN_COST * 35);
10424 format %{ "sdiv $dst, $src1, $src2" %}
10425
10426 ins_encode(aarch64_enc_div(dst, src1, src2));
10427 ins_pipe(ldiv_reg_reg);
10428 %}
10429
10430 // Integer Remainder
10431
10432 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10433 match(Set dst (ModI src1 src2));
10434
10435 ins_cost(INSN_COST * 22);
10436 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10437 "msubw $dst, rscratch1, $src2, $src1" %}
10438
10439 ins_encode(aarch64_enc_modw(dst, src1, src2));
10440 ins_pipe(idiv_reg_reg);
10441 %}
10442
10443 // Long Remainder
10444
10445 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10446 match(Set dst (ModL src1 src2));
10447
10448 ins_cost(INSN_COST * 38);
10449 format %{ "sdiv rscratch1, $src1, $src2\n"
10450 "msub $dst, rscratch1, $src2, $src1" %}
10451
10452 ins_encode(aarch64_enc_mod(dst, src1, src2));
10453 ins_pipe(ldiv_reg_reg);
10454 %}
10455
10456 // Unsigned Integer Divide
10457
10458 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10459 match(Set dst (UDivI src1 src2));
10460
10461 ins_cost(INSN_COST * 19);
10462 format %{ "udivw $dst, $src1, $src2" %}
10463
10464 ins_encode %{
10465 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10466 %}
10467
10468 ins_pipe(idiv_reg_reg);
10469 %}
10470
10471 // Unsigned Long Divide
10472
10473 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10474 match(Set dst (UDivL src1 src2));
10475
10476 ins_cost(INSN_COST * 35);
10477 format %{ "udiv $dst, $src1, $src2" %}
10478
10479 ins_encode %{
10480 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10481 %}
10482
10483 ins_pipe(ldiv_reg_reg);
10484 %}
10485
10486 // Unsigned Integer Remainder
10487
10488 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10489 match(Set dst (UModI src1 src2));
10490
10491 ins_cost(INSN_COST * 22);
10492 format %{ "udivw rscratch1, $src1, $src2\n\t"
10493 "msubw $dst, rscratch1, $src2, $src1" %}
10494
10495 ins_encode %{
10496 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10497 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10498 %}
10499
10500 ins_pipe(idiv_reg_reg);
10501 %}
10502
10503 // Unsigned Long Remainder
10504
10505 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10506 match(Set dst (UModL src1 src2));
10507
10508 ins_cost(INSN_COST * 38);
10509 format %{ "udiv rscratch1, $src1, $src2\n"
10510 "msub $dst, rscratch1, $src2, $src1" %}
10511
10512 ins_encode %{
10513 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10514 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10515 %}
10516
10517 ins_pipe(ldiv_reg_reg);
10518 %}
10519
10520 // Integer Shifts
10521
10522 // Shift Left Register
10523 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10524 match(Set dst (LShiftI src1 src2));
10525
10526 ins_cost(INSN_COST * 2);
10527 format %{ "lslvw $dst, $src1, $src2" %}
10528
10529 ins_encode %{
10530 __ lslvw(as_Register($dst$$reg),
10531 as_Register($src1$$reg),
10532 as_Register($src2$$reg));
10533 %}
10534
10535 ins_pipe(ialu_reg_reg_vshift);
10536 %}
10537
10538 // Shift Left Immediate
10539 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10540 match(Set dst (LShiftI src1 src2));
10541
10542 ins_cost(INSN_COST);
10543 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10544
10545 ins_encode %{
10546 __ lslw(as_Register($dst$$reg),
10547 as_Register($src1$$reg),
10548 $src2$$constant & 0x1f);
10549 %}
10550
10551 ins_pipe(ialu_reg_shift);
10552 %}
10553
10554 // Shift Right Logical Register
10555 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10556 match(Set dst (URShiftI src1 src2));
10557
10558 ins_cost(INSN_COST * 2);
10559 format %{ "lsrvw $dst, $src1, $src2" %}
10560
10561 ins_encode %{
10562 __ lsrvw(as_Register($dst$$reg),
10563 as_Register($src1$$reg),
10564 as_Register($src2$$reg));
10565 %}
10566
10567 ins_pipe(ialu_reg_reg_vshift);
10568 %}
10569
10570 // Shift Right Logical Immediate
10571 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10572 match(Set dst (URShiftI src1 src2));
10573
10574 ins_cost(INSN_COST);
10575 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10576
10577 ins_encode %{
10578 __ lsrw(as_Register($dst$$reg),
10579 as_Register($src1$$reg),
10580 $src2$$constant & 0x1f);
10581 %}
10582
10583 ins_pipe(ialu_reg_shift);
10584 %}
10585
10586 // Shift Right Arithmetic Register
10587 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10588 match(Set dst (RShiftI src1 src2));
10589
10590 ins_cost(INSN_COST * 2);
10591 format %{ "asrvw $dst, $src1, $src2" %}
10592
10593 ins_encode %{
10594 __ asrvw(as_Register($dst$$reg),
10595 as_Register($src1$$reg),
10596 as_Register($src2$$reg));
10597 %}
10598
10599 ins_pipe(ialu_reg_reg_vshift);
10600 %}
10601
10602 // Shift Right Arithmetic Immediate
10603 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10604 match(Set dst (RShiftI src1 src2));
10605
10606 ins_cost(INSN_COST);
10607 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10608
10609 ins_encode %{
10610 __ asrw(as_Register($dst$$reg),
10611 as_Register($src1$$reg),
10612 $src2$$constant & 0x1f);
10613 %}
10614
10615 ins_pipe(ialu_reg_shift);
10616 %}
10617
10618 // Combined Int Mask and Right Shift (using UBFM)
10619 // TODO
10620
10621 // Long Shifts
10622
10623 // Shift Left Register
10624 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10625 match(Set dst (LShiftL src1 src2));
10626
10627 ins_cost(INSN_COST * 2);
10628 format %{ "lslv $dst, $src1, $src2" %}
10629
10630 ins_encode %{
10631 __ lslv(as_Register($dst$$reg),
10632 as_Register($src1$$reg),
10633 as_Register($src2$$reg));
10634 %}
10635
10636 ins_pipe(ialu_reg_reg_vshift);
10637 %}
10638
10639 // Shift Left Immediate
10640 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10641 match(Set dst (LShiftL src1 src2));
10642
10643 ins_cost(INSN_COST);
10644 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10645
10646 ins_encode %{
10647 __ lsl(as_Register($dst$$reg),
10648 as_Register($src1$$reg),
10649 $src2$$constant & 0x3f);
10650 %}
10651
10652 ins_pipe(ialu_reg_shift);
10653 %}
10654
10655 // Shift Right Logical Register
10656 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10657 match(Set dst (URShiftL src1 src2));
10658
10659 ins_cost(INSN_COST * 2);
10660 format %{ "lsrv $dst, $src1, $src2" %}
10661
10662 ins_encode %{
10663 __ lsrv(as_Register($dst$$reg),
10664 as_Register($src1$$reg),
10665 as_Register($src2$$reg));
10666 %}
10667
10668 ins_pipe(ialu_reg_reg_vshift);
10669 %}
10670
10671 // Shift Right Logical Immediate
10672 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10673 match(Set dst (URShiftL src1 src2));
10674
10675 ins_cost(INSN_COST);
10676 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10677
10678 ins_encode %{
10679 __ lsr(as_Register($dst$$reg),
10680 as_Register($src1$$reg),
10681 $src2$$constant & 0x3f);
10682 %}
10683
10684 ins_pipe(ialu_reg_shift);
10685 %}
10686
10687 // A special-case pattern for card table stores.
10688 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10689 match(Set dst (URShiftL (CastP2X src1) src2));
10690
10691 ins_cost(INSN_COST);
10692 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10693
10694 ins_encode %{
10695 __ lsr(as_Register($dst$$reg),
10696 as_Register($src1$$reg),
10697 $src2$$constant & 0x3f);
10698 %}
10699
10700 ins_pipe(ialu_reg_shift);
10701 %}
10702
10703 // Shift Right Arithmetic Register
10704 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10705 match(Set dst (RShiftL src1 src2));
10706
10707 ins_cost(INSN_COST * 2);
10708 format %{ "asrv $dst, $src1, $src2" %}
10709
10710 ins_encode %{
10711 __ asrv(as_Register($dst$$reg),
10712 as_Register($src1$$reg),
10713 as_Register($src2$$reg));
10714 %}
10715
10716 ins_pipe(ialu_reg_reg_vshift);
10717 %}
10718
10719 // Shift Right Arithmetic Immediate
10720 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10721 match(Set dst (RShiftL src1 src2));
10722
10723 ins_cost(INSN_COST);
10724 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10725
10726 ins_encode %{
10727 __ asr(as_Register($dst$$reg),
10728 as_Register($src1$$reg),
10729 $src2$$constant & 0x3f);
10730 %}
10731
10732 ins_pipe(ialu_reg_shift);
10733 %}
10734
10735 // BEGIN This section of the file is automatically generated. Do not edit --------------
10736 // This section is generated from aarch64_ad.m4
10737
10738 // This pattern is automatically generated from aarch64_ad.m4
10739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10740 instruct regL_not_reg(iRegLNoSp dst,
10741 iRegL src1, immL_M1 m1,
10742 rFlagsReg cr) %{
10743 match(Set dst (XorL src1 m1));
10744 ins_cost(INSN_COST);
10745 format %{ "eon $dst, $src1, zr" %}
10746
10747 ins_encode %{
10748 __ eon(as_Register($dst$$reg),
10749 as_Register($src1$$reg),
10750 zr,
10751 Assembler::LSL, 0);
10752 %}
10753
10754 ins_pipe(ialu_reg);
10755 %}
10756
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 instruct regI_not_reg(iRegINoSp dst,
10760 iRegIorL2I src1, immI_M1 m1,
10761 rFlagsReg cr) %{
10762 match(Set dst (XorI src1 m1));
10763 ins_cost(INSN_COST);
10764 format %{ "eonw $dst, $src1, zr" %}
10765
10766 ins_encode %{
10767 __ eonw(as_Register($dst$$reg),
10768 as_Register($src1$$reg),
10769 zr,
10770 Assembler::LSL, 0);
10771 %}
10772
10773 ins_pipe(ialu_reg);
10774 %}
10775
10776 // This pattern is automatically generated from aarch64_ad.m4
10777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10778 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10779 immI0 zero, iRegIorL2I src1, immI src2) %{
10780 match(Set dst (SubI zero (URShiftI src1 src2)));
10781
10782 ins_cost(1.9 * INSN_COST);
10783 format %{ "negw $dst, $src1, LSR $src2" %}
10784
10785 ins_encode %{
10786 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10787 Assembler::LSR, $src2$$constant & 0x1f);
10788 %}
10789
10790 ins_pipe(ialu_reg_shift);
10791 %}
10792
10793 // This pattern is automatically generated from aarch64_ad.m4
10794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10795 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10796 immI0 zero, iRegIorL2I src1, immI src2) %{
10797 match(Set dst (SubI zero (RShiftI src1 src2)));
10798
10799 ins_cost(1.9 * INSN_COST);
10800 format %{ "negw $dst, $src1, ASR $src2" %}
10801
10802 ins_encode %{
10803 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10804 Assembler::ASR, $src2$$constant & 0x1f);
10805 %}
10806
10807 ins_pipe(ialu_reg_shift);
10808 %}
10809
10810 // This pattern is automatically generated from aarch64_ad.m4
10811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10812 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10813 immI0 zero, iRegIorL2I src1, immI src2) %{
10814 match(Set dst (SubI zero (LShiftI src1 src2)));
10815
10816 ins_cost(1.9 * INSN_COST);
10817 format %{ "negw $dst, $src1, LSL $src2" %}
10818
10819 ins_encode %{
10820 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10821 Assembler::LSL, $src2$$constant & 0x1f);
10822 %}
10823
10824 ins_pipe(ialu_reg_shift);
10825 %}
10826
10827 // This pattern is automatically generated from aarch64_ad.m4
10828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10829 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10830 immL0 zero, iRegL src1, immI src2) %{
10831 match(Set dst (SubL zero (URShiftL src1 src2)));
10832
10833 ins_cost(1.9 * INSN_COST);
10834 format %{ "neg $dst, $src1, LSR $src2" %}
10835
10836 ins_encode %{
10837 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10838 Assembler::LSR, $src2$$constant & 0x3f);
10839 %}
10840
10841 ins_pipe(ialu_reg_shift);
10842 %}
10843
10844 // This pattern is automatically generated from aarch64_ad.m4
10845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10846 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10847 immL0 zero, iRegL src1, immI src2) %{
10848 match(Set dst (SubL zero (RShiftL src1 src2)));
10849
10850 ins_cost(1.9 * INSN_COST);
10851 format %{ "neg $dst, $src1, ASR $src2" %}
10852
10853 ins_encode %{
10854 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10855 Assembler::ASR, $src2$$constant & 0x3f);
10856 %}
10857
10858 ins_pipe(ialu_reg_shift);
10859 %}
10860
10861 // This pattern is automatically generated from aarch64_ad.m4
10862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10863 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10864 immL0 zero, iRegL src1, immI src2) %{
10865 match(Set dst (SubL zero (LShiftL src1 src2)));
10866
10867 ins_cost(1.9 * INSN_COST);
10868 format %{ "neg $dst, $src1, LSL $src2" %}
10869
10870 ins_encode %{
10871 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10872 Assembler::LSL, $src2$$constant & 0x3f);
10873 %}
10874
10875 ins_pipe(ialu_reg_shift);
10876 %}
10877
10878 // This pattern is automatically generated from aarch64_ad.m4
10879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10880 instruct AndI_reg_not_reg(iRegINoSp dst,
10881 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10882 match(Set dst (AndI src1 (XorI src2 m1)));
10883 ins_cost(INSN_COST);
10884 format %{ "bicw $dst, $src1, $src2" %}
10885
10886 ins_encode %{
10887 __ bicw(as_Register($dst$$reg),
10888 as_Register($src1$$reg),
10889 as_Register($src2$$reg),
10890 Assembler::LSL, 0);
10891 %}
10892
10893 ins_pipe(ialu_reg_reg);
10894 %}
10895
10896 // This pattern is automatically generated from aarch64_ad.m4
10897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10898 instruct AndL_reg_not_reg(iRegLNoSp dst,
10899 iRegL src1, iRegL src2, immL_M1 m1) %{
10900 match(Set dst (AndL src1 (XorL src2 m1)));
10901 ins_cost(INSN_COST);
10902 format %{ "bic $dst, $src1, $src2" %}
10903
10904 ins_encode %{
10905 __ bic(as_Register($dst$$reg),
10906 as_Register($src1$$reg),
10907 as_Register($src2$$reg),
10908 Assembler::LSL, 0);
10909 %}
10910
10911 ins_pipe(ialu_reg_reg);
10912 %}
10913
10914 // This pattern is automatically generated from aarch64_ad.m4
10915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10916 instruct OrI_reg_not_reg(iRegINoSp dst,
10917 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10918 match(Set dst (OrI src1 (XorI src2 m1)));
10919 ins_cost(INSN_COST);
10920 format %{ "ornw $dst, $src1, $src2" %}
10921
10922 ins_encode %{
10923 __ ornw(as_Register($dst$$reg),
10924 as_Register($src1$$reg),
10925 as_Register($src2$$reg),
10926 Assembler::LSL, 0);
10927 %}
10928
10929 ins_pipe(ialu_reg_reg);
10930 %}
10931
10932 // This pattern is automatically generated from aarch64_ad.m4
10933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10934 instruct OrL_reg_not_reg(iRegLNoSp dst,
10935 iRegL src1, iRegL src2, immL_M1 m1) %{
10936 match(Set dst (OrL src1 (XorL src2 m1)));
10937 ins_cost(INSN_COST);
10938 format %{ "orn $dst, $src1, $src2" %}
10939
10940 ins_encode %{
10941 __ orn(as_Register($dst$$reg),
10942 as_Register($src1$$reg),
10943 as_Register($src2$$reg),
10944 Assembler::LSL, 0);
10945 %}
10946
10947 ins_pipe(ialu_reg_reg);
10948 %}
10949
10950 // This pattern is automatically generated from aarch64_ad.m4
10951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10952 instruct XorI_reg_not_reg(iRegINoSp dst,
10953 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10954 match(Set dst (XorI m1 (XorI src2 src1)));
10955 ins_cost(INSN_COST);
10956 format %{ "eonw $dst, $src1, $src2" %}
10957
10958 ins_encode %{
10959 __ eonw(as_Register($dst$$reg),
10960 as_Register($src1$$reg),
10961 as_Register($src2$$reg),
10962 Assembler::LSL, 0);
10963 %}
10964
10965 ins_pipe(ialu_reg_reg);
10966 %}
10967
10968 // This pattern is automatically generated from aarch64_ad.m4
10969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10970 instruct XorL_reg_not_reg(iRegLNoSp dst,
10971 iRegL src1, iRegL src2, immL_M1 m1) %{
10972 match(Set dst (XorL m1 (XorL src2 src1)));
10973 ins_cost(INSN_COST);
10974 format %{ "eon $dst, $src1, $src2" %}
10975
10976 ins_encode %{
10977 __ eon(as_Register($dst$$reg),
10978 as_Register($src1$$reg),
10979 as_Register($src2$$reg),
10980 Assembler::LSL, 0);
10981 %}
10982
10983 ins_pipe(ialu_reg_reg);
10984 %}
10985
10986 // This pattern is automatically generated from aarch64_ad.m4
10987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10988 // val & (-1 ^ (val >>> shift)) ==> bicw
10989 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10990 iRegIorL2I src1, iRegIorL2I src2,
10991 immI src3, immI_M1 src4) %{
10992 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10993 ins_cost(1.9 * INSN_COST);
10994 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10995
10996 ins_encode %{
10997 __ bicw(as_Register($dst$$reg),
10998 as_Register($src1$$reg),
10999 as_Register($src2$$reg),
11000 Assembler::LSR,
11001 $src3$$constant & 0x1f);
11002 %}
11003
11004 ins_pipe(ialu_reg_reg_shift);
11005 %}
11006
11007 // This pattern is automatically generated from aarch64_ad.m4
11008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11009 // val & (-1 ^ (val >>> shift)) ==> bic
11010 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11011 iRegL src1, iRegL src2,
11012 immI src3, immL_M1 src4) %{
11013 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11014 ins_cost(1.9 * INSN_COST);
11015 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11016
11017 ins_encode %{
11018 __ bic(as_Register($dst$$reg),
11019 as_Register($src1$$reg),
11020 as_Register($src2$$reg),
11021 Assembler::LSR,
11022 $src3$$constant & 0x3f);
11023 %}
11024
11025 ins_pipe(ialu_reg_reg_shift);
11026 %}
11027
11028 // This pattern is automatically generated from aarch64_ad.m4
11029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11030 // val & (-1 ^ (val >> shift)) ==> bicw
11031 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11032 iRegIorL2I src1, iRegIorL2I src2,
11033 immI src3, immI_M1 src4) %{
11034 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11035 ins_cost(1.9 * INSN_COST);
11036 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11037
11038 ins_encode %{
11039 __ bicw(as_Register($dst$$reg),
11040 as_Register($src1$$reg),
11041 as_Register($src2$$reg),
11042 Assembler::ASR,
11043 $src3$$constant & 0x1f);
11044 %}
11045
11046 ins_pipe(ialu_reg_reg_shift);
11047 %}
11048
11049 // This pattern is automatically generated from aarch64_ad.m4
11050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11051 // val & (-1 ^ (val >> shift)) ==> bic
11052 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11053 iRegL src1, iRegL src2,
11054 immI src3, immL_M1 src4) %{
11055 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11056 ins_cost(1.9 * INSN_COST);
11057 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11058
11059 ins_encode %{
11060 __ bic(as_Register($dst$$reg),
11061 as_Register($src1$$reg),
11062 as_Register($src2$$reg),
11063 Assembler::ASR,
11064 $src3$$constant & 0x3f);
11065 %}
11066
11067 ins_pipe(ialu_reg_reg_shift);
11068 %}
11069
11070 // This pattern is automatically generated from aarch64_ad.m4
11071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11072 // val & (-1 ^ (val ror shift)) ==> bicw
11073 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11074 iRegIorL2I src1, iRegIorL2I src2,
11075 immI src3, immI_M1 src4) %{
11076 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11077 ins_cost(1.9 * INSN_COST);
11078 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11079
11080 ins_encode %{
11081 __ bicw(as_Register($dst$$reg),
11082 as_Register($src1$$reg),
11083 as_Register($src2$$reg),
11084 Assembler::ROR,
11085 $src3$$constant & 0x1f);
11086 %}
11087
11088 ins_pipe(ialu_reg_reg_shift);
11089 %}
11090
11091 // This pattern is automatically generated from aarch64_ad.m4
11092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11093 // val & (-1 ^ (val ror shift)) ==> bic
11094 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11095 iRegL src1, iRegL src2,
11096 immI src3, immL_M1 src4) %{
11097 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11098 ins_cost(1.9 * INSN_COST);
11099 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11100
11101 ins_encode %{
11102 __ bic(as_Register($dst$$reg),
11103 as_Register($src1$$reg),
11104 as_Register($src2$$reg),
11105 Assembler::ROR,
11106 $src3$$constant & 0x3f);
11107 %}
11108
11109 ins_pipe(ialu_reg_reg_shift);
11110 %}
11111
11112 // This pattern is automatically generated from aarch64_ad.m4
11113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11114 // val & (-1 ^ (val << shift)) ==> bicw
11115 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11116 iRegIorL2I src1, iRegIorL2I src2,
11117 immI src3, immI_M1 src4) %{
11118 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11119 ins_cost(1.9 * INSN_COST);
11120 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11121
11122 ins_encode %{
11123 __ bicw(as_Register($dst$$reg),
11124 as_Register($src1$$reg),
11125 as_Register($src2$$reg),
11126 Assembler::LSL,
11127 $src3$$constant & 0x1f);
11128 %}
11129
11130 ins_pipe(ialu_reg_reg_shift);
11131 %}
11132
11133 // This pattern is automatically generated from aarch64_ad.m4
11134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11135 // val & (-1 ^ (val << shift)) ==> bic
11136 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11137 iRegL src1, iRegL src2,
11138 immI src3, immL_M1 src4) %{
11139 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11140 ins_cost(1.9 * INSN_COST);
11141 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11142
11143 ins_encode %{
11144 __ bic(as_Register($dst$$reg),
11145 as_Register($src1$$reg),
11146 as_Register($src2$$reg),
11147 Assembler::LSL,
11148 $src3$$constant & 0x3f);
11149 %}
11150
11151 ins_pipe(ialu_reg_reg_shift);
11152 %}
11153
11154 // This pattern is automatically generated from aarch64_ad.m4
11155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11156 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11157 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11158 iRegIorL2I src1, iRegIorL2I src2,
11159 immI src3, immI_M1 src4) %{
11160 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11161 ins_cost(1.9 * INSN_COST);
11162 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11163
11164 ins_encode %{
11165 __ eonw(as_Register($dst$$reg),
11166 as_Register($src1$$reg),
11167 as_Register($src2$$reg),
11168 Assembler::LSR,
11169 $src3$$constant & 0x1f);
11170 %}
11171
11172 ins_pipe(ialu_reg_reg_shift);
11173 %}
11174
11175 // This pattern is automatically generated from aarch64_ad.m4
11176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11177 // val ^ (-1 ^ (val >>> shift)) ==> eon
11178 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11179 iRegL src1, iRegL src2,
11180 immI src3, immL_M1 src4) %{
11181 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11182 ins_cost(1.9 * INSN_COST);
11183 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11184
11185 ins_encode %{
11186 __ eon(as_Register($dst$$reg),
11187 as_Register($src1$$reg),
11188 as_Register($src2$$reg),
11189 Assembler::LSR,
11190 $src3$$constant & 0x3f);
11191 %}
11192
11193 ins_pipe(ialu_reg_reg_shift);
11194 %}
11195
11196 // This pattern is automatically generated from aarch64_ad.m4
11197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11198 // val ^ (-1 ^ (val >> shift)) ==> eonw
11199 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11200 iRegIorL2I src1, iRegIorL2I src2,
11201 immI src3, immI_M1 src4) %{
11202 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11203 ins_cost(1.9 * INSN_COST);
11204 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11205
11206 ins_encode %{
11207 __ eonw(as_Register($dst$$reg),
11208 as_Register($src1$$reg),
11209 as_Register($src2$$reg),
11210 Assembler::ASR,
11211 $src3$$constant & 0x1f);
11212 %}
11213
11214 ins_pipe(ialu_reg_reg_shift);
11215 %}
11216
11217 // This pattern is automatically generated from aarch64_ad.m4
11218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11219 // val ^ (-1 ^ (val >> shift)) ==> eon
11220 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11221 iRegL src1, iRegL src2,
11222 immI src3, immL_M1 src4) %{
11223 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11224 ins_cost(1.9 * INSN_COST);
11225 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11226
11227 ins_encode %{
11228 __ eon(as_Register($dst$$reg),
11229 as_Register($src1$$reg),
11230 as_Register($src2$$reg),
11231 Assembler::ASR,
11232 $src3$$constant & 0x3f);
11233 %}
11234
11235 ins_pipe(ialu_reg_reg_shift);
11236 %}
11237
11238 // This pattern is automatically generated from aarch64_ad.m4
11239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11240 // val ^ (-1 ^ (val ror shift)) ==> eonw
11241 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11242 iRegIorL2I src1, iRegIorL2I src2,
11243 immI src3, immI_M1 src4) %{
11244 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11245 ins_cost(1.9 * INSN_COST);
11246 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11247
11248 ins_encode %{
11249 __ eonw(as_Register($dst$$reg),
11250 as_Register($src1$$reg),
11251 as_Register($src2$$reg),
11252 Assembler::ROR,
11253 $src3$$constant & 0x1f);
11254 %}
11255
11256 ins_pipe(ialu_reg_reg_shift);
11257 %}
11258
11259 // This pattern is automatically generated from aarch64_ad.m4
11260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11261 // val ^ (-1 ^ (val ror shift)) ==> eon
11262 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11263 iRegL src1, iRegL src2,
11264 immI src3, immL_M1 src4) %{
11265 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11266 ins_cost(1.9 * INSN_COST);
11267 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11268
11269 ins_encode %{
11270 __ eon(as_Register($dst$$reg),
11271 as_Register($src1$$reg),
11272 as_Register($src2$$reg),
11273 Assembler::ROR,
11274 $src3$$constant & 0x3f);
11275 %}
11276
11277 ins_pipe(ialu_reg_reg_shift);
11278 %}
11279
11280 // This pattern is automatically generated from aarch64_ad.m4
11281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11282 // val ^ (-1 ^ (val << shift)) ==> eonw
11283 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11284 iRegIorL2I src1, iRegIorL2I src2,
11285 immI src3, immI_M1 src4) %{
11286 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11287 ins_cost(1.9 * INSN_COST);
11288 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11289
11290 ins_encode %{
11291 __ eonw(as_Register($dst$$reg),
11292 as_Register($src1$$reg),
11293 as_Register($src2$$reg),
11294 Assembler::LSL,
11295 $src3$$constant & 0x1f);
11296 %}
11297
11298 ins_pipe(ialu_reg_reg_shift);
11299 %}
11300
11301 // This pattern is automatically generated from aarch64_ad.m4
11302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11303 // val ^ (-1 ^ (val << shift)) ==> eon
11304 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11305 iRegL src1, iRegL src2,
11306 immI src3, immL_M1 src4) %{
11307 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11308 ins_cost(1.9 * INSN_COST);
11309 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11310
11311 ins_encode %{
11312 __ eon(as_Register($dst$$reg),
11313 as_Register($src1$$reg),
11314 as_Register($src2$$reg),
11315 Assembler::LSL,
11316 $src3$$constant & 0x3f);
11317 %}
11318
11319 ins_pipe(ialu_reg_reg_shift);
11320 %}
11321
11322 // This pattern is automatically generated from aarch64_ad.m4
11323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11324 // val | (-1 ^ (val >>> shift)) ==> ornw
11325 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11326 iRegIorL2I src1, iRegIorL2I src2,
11327 immI src3, immI_M1 src4) %{
11328 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11329 ins_cost(1.9 * INSN_COST);
11330 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11331
11332 ins_encode %{
11333 __ ornw(as_Register($dst$$reg),
11334 as_Register($src1$$reg),
11335 as_Register($src2$$reg),
11336 Assembler::LSR,
11337 $src3$$constant & 0x1f);
11338 %}
11339
11340 ins_pipe(ialu_reg_reg_shift);
11341 %}
11342
11343 // This pattern is automatically generated from aarch64_ad.m4
11344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11345 // val | (-1 ^ (val >>> shift)) ==> orn
11346 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11347 iRegL src1, iRegL src2,
11348 immI src3, immL_M1 src4) %{
11349 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11350 ins_cost(1.9 * INSN_COST);
11351 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11352
11353 ins_encode %{
11354 __ orn(as_Register($dst$$reg),
11355 as_Register($src1$$reg),
11356 as_Register($src2$$reg),
11357 Assembler::LSR,
11358 $src3$$constant & 0x3f);
11359 %}
11360
11361 ins_pipe(ialu_reg_reg_shift);
11362 %}
11363
11364 // This pattern is automatically generated from aarch64_ad.m4
11365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11366 // val | (-1 ^ (val >> shift)) ==> ornw
11367 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11368 iRegIorL2I src1, iRegIorL2I src2,
11369 immI src3, immI_M1 src4) %{
11370 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11371 ins_cost(1.9 * INSN_COST);
11372 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11373
11374 ins_encode %{
11375 __ ornw(as_Register($dst$$reg),
11376 as_Register($src1$$reg),
11377 as_Register($src2$$reg),
11378 Assembler::ASR,
11379 $src3$$constant & 0x1f);
11380 %}
11381
11382 ins_pipe(ialu_reg_reg_shift);
11383 %}
11384
11385 // This pattern is automatically generated from aarch64_ad.m4
11386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11387 // val | (-1 ^ (val >> shift)) ==> orn
11388 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11389 iRegL src1, iRegL src2,
11390 immI src3, immL_M1 src4) %{
11391 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11392 ins_cost(1.9 * INSN_COST);
11393 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11394
11395 ins_encode %{
11396 __ orn(as_Register($dst$$reg),
11397 as_Register($src1$$reg),
11398 as_Register($src2$$reg),
11399 Assembler::ASR,
11400 $src3$$constant & 0x3f);
11401 %}
11402
11403 ins_pipe(ialu_reg_reg_shift);
11404 %}
11405
11406 // This pattern is automatically generated from aarch64_ad.m4
11407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11408 // val | (-1 ^ (val ror shift)) ==> ornw
11409 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11410 iRegIorL2I src1, iRegIorL2I src2,
11411 immI src3, immI_M1 src4) %{
11412 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11413 ins_cost(1.9 * INSN_COST);
11414 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11415
11416 ins_encode %{
11417 __ ornw(as_Register($dst$$reg),
11418 as_Register($src1$$reg),
11419 as_Register($src2$$reg),
11420 Assembler::ROR,
11421 $src3$$constant & 0x1f);
11422 %}
11423
11424 ins_pipe(ialu_reg_reg_shift);
11425 %}
11426
11427 // This pattern is automatically generated from aarch64_ad.m4
11428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11429 // val | (-1 ^ (val ror shift)) ==> orn
11430 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11431 iRegL src1, iRegL src2,
11432 immI src3, immL_M1 src4) %{
11433 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11434 ins_cost(1.9 * INSN_COST);
11435 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11436
11437 ins_encode %{
11438 __ orn(as_Register($dst$$reg),
11439 as_Register($src1$$reg),
11440 as_Register($src2$$reg),
11441 Assembler::ROR,
11442 $src3$$constant & 0x3f);
11443 %}
11444
11445 ins_pipe(ialu_reg_reg_shift);
11446 %}
11447
11448 // This pattern is automatically generated from aarch64_ad.m4
11449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11450 // val | (-1 ^ (val << shift)) ==> ornw
11451 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11452 iRegIorL2I src1, iRegIorL2I src2,
11453 immI src3, immI_M1 src4) %{
11454 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11455 ins_cost(1.9 * INSN_COST);
11456 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11457
11458 ins_encode %{
11459 __ ornw(as_Register($dst$$reg),
11460 as_Register($src1$$reg),
11461 as_Register($src2$$reg),
11462 Assembler::LSL,
11463 $src3$$constant & 0x1f);
11464 %}
11465
11466 ins_pipe(ialu_reg_reg_shift);
11467 %}
11468
11469 // This pattern is automatically generated from aarch64_ad.m4
11470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11471 // val | (-1 ^ (val << shift)) ==> orn
11472 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11473 iRegL src1, iRegL src2,
11474 immI src3, immL_M1 src4) %{
11475 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11476 ins_cost(1.9 * INSN_COST);
11477 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11478
11479 ins_encode %{
11480 __ orn(as_Register($dst$$reg),
11481 as_Register($src1$$reg),
11482 as_Register($src2$$reg),
11483 Assembler::LSL,
11484 $src3$$constant & 0x3f);
11485 %}
11486
11487 ins_pipe(ialu_reg_reg_shift);
11488 %}
11489
11490 // This pattern is automatically generated from aarch64_ad.m4
11491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11492 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11493 iRegIorL2I src1, iRegIorL2I src2,
11494 immI src3) %{
11495 match(Set dst (AndI src1 (URShiftI src2 src3)));
11496
11497 ins_cost(1.9 * INSN_COST);
11498 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11499
11500 ins_encode %{
11501 __ andw(as_Register($dst$$reg),
11502 as_Register($src1$$reg),
11503 as_Register($src2$$reg),
11504 Assembler::LSR,
11505 $src3$$constant & 0x1f);
11506 %}
11507
11508 ins_pipe(ialu_reg_reg_shift);
11509 %}
11510
11511 // This pattern is automatically generated from aarch64_ad.m4
11512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11513 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11514 iRegL src1, iRegL src2,
11515 immI src3) %{
11516 match(Set dst (AndL src1 (URShiftL src2 src3)));
11517
11518 ins_cost(1.9 * INSN_COST);
11519 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11520
11521 ins_encode %{
11522 __ andr(as_Register($dst$$reg),
11523 as_Register($src1$$reg),
11524 as_Register($src2$$reg),
11525 Assembler::LSR,
11526 $src3$$constant & 0x3f);
11527 %}
11528
11529 ins_pipe(ialu_reg_reg_shift);
11530 %}
11531
11532 // This pattern is automatically generated from aarch64_ad.m4
11533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11534 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11535 iRegIorL2I src1, iRegIorL2I src2,
11536 immI src3) %{
11537 match(Set dst (AndI src1 (RShiftI src2 src3)));
11538
11539 ins_cost(1.9 * INSN_COST);
11540 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11541
11542 ins_encode %{
11543 __ andw(as_Register($dst$$reg),
11544 as_Register($src1$$reg),
11545 as_Register($src2$$reg),
11546 Assembler::ASR,
11547 $src3$$constant & 0x1f);
11548 %}
11549
11550 ins_pipe(ialu_reg_reg_shift);
11551 %}
11552
11553 // This pattern is automatically generated from aarch64_ad.m4
11554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11555 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11556 iRegL src1, iRegL src2,
11557 immI src3) %{
11558 match(Set dst (AndL src1 (RShiftL src2 src3)));
11559
11560 ins_cost(1.9 * INSN_COST);
11561 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11562
11563 ins_encode %{
11564 __ andr(as_Register($dst$$reg),
11565 as_Register($src1$$reg),
11566 as_Register($src2$$reg),
11567 Assembler::ASR,
11568 $src3$$constant & 0x3f);
11569 %}
11570
11571 ins_pipe(ialu_reg_reg_shift);
11572 %}
11573
11574 // This pattern is automatically generated from aarch64_ad.m4
11575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11576 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11577 iRegIorL2I src1, iRegIorL2I src2,
11578 immI src3) %{
11579 match(Set dst (AndI src1 (LShiftI src2 src3)));
11580
11581 ins_cost(1.9 * INSN_COST);
11582 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11583
11584 ins_encode %{
11585 __ andw(as_Register($dst$$reg),
11586 as_Register($src1$$reg),
11587 as_Register($src2$$reg),
11588 Assembler::LSL,
11589 $src3$$constant & 0x1f);
11590 %}
11591
11592 ins_pipe(ialu_reg_reg_shift);
11593 %}
11594
11595 // This pattern is automatically generated from aarch64_ad.m4
11596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11597 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11598 iRegL src1, iRegL src2,
11599 immI src3) %{
11600 match(Set dst (AndL src1 (LShiftL src2 src3)));
11601
11602 ins_cost(1.9 * INSN_COST);
11603 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11604
11605 ins_encode %{
11606 __ andr(as_Register($dst$$reg),
11607 as_Register($src1$$reg),
11608 as_Register($src2$$reg),
11609 Assembler::LSL,
11610 $src3$$constant & 0x3f);
11611 %}
11612
11613 ins_pipe(ialu_reg_reg_shift);
11614 %}
11615
11616 // This pattern is automatically generated from aarch64_ad.m4
11617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11618 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11619 iRegIorL2I src1, iRegIorL2I src2,
11620 immI src3) %{
11621 match(Set dst (AndI src1 (RotateRight src2 src3)));
11622
11623 ins_cost(1.9 * INSN_COST);
11624 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11625
11626 ins_encode %{
11627 __ andw(as_Register($dst$$reg),
11628 as_Register($src1$$reg),
11629 as_Register($src2$$reg),
11630 Assembler::ROR,
11631 $src3$$constant & 0x1f);
11632 %}
11633
11634 ins_pipe(ialu_reg_reg_shift);
11635 %}
11636
11637 // This pattern is automatically generated from aarch64_ad.m4
11638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11639 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11640 iRegL src1, iRegL src2,
11641 immI src3) %{
11642 match(Set dst (AndL src1 (RotateRight src2 src3)));
11643
11644 ins_cost(1.9 * INSN_COST);
11645 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11646
11647 ins_encode %{
11648 __ andr(as_Register($dst$$reg),
11649 as_Register($src1$$reg),
11650 as_Register($src2$$reg),
11651 Assembler::ROR,
11652 $src3$$constant & 0x3f);
11653 %}
11654
11655 ins_pipe(ialu_reg_reg_shift);
11656 %}
11657
11658 // This pattern is automatically generated from aarch64_ad.m4
11659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11660 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11661 iRegIorL2I src1, iRegIorL2I src2,
11662 immI src3) %{
11663 match(Set dst (XorI src1 (URShiftI src2 src3)));
11664
11665 ins_cost(1.9 * INSN_COST);
11666 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11667
11668 ins_encode %{
11669 __ eorw(as_Register($dst$$reg),
11670 as_Register($src1$$reg),
11671 as_Register($src2$$reg),
11672 Assembler::LSR,
11673 $src3$$constant & 0x1f);
11674 %}
11675
11676 ins_pipe(ialu_reg_reg_shift);
11677 %}
11678
11679 // This pattern is automatically generated from aarch64_ad.m4
11680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11681 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11682 iRegL src1, iRegL src2,
11683 immI src3) %{
11684 match(Set dst (XorL src1 (URShiftL src2 src3)));
11685
11686 ins_cost(1.9 * INSN_COST);
11687 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11688
11689 ins_encode %{
11690 __ eor(as_Register($dst$$reg),
11691 as_Register($src1$$reg),
11692 as_Register($src2$$reg),
11693 Assembler::LSR,
11694 $src3$$constant & 0x3f);
11695 %}
11696
11697 ins_pipe(ialu_reg_reg_shift);
11698 %}
11699
11700 // This pattern is automatically generated from aarch64_ad.m4
11701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11702 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11703 iRegIorL2I src1, iRegIorL2I src2,
11704 immI src3) %{
11705 match(Set dst (XorI src1 (RShiftI src2 src3)));
11706
11707 ins_cost(1.9 * INSN_COST);
11708 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11709
11710 ins_encode %{
11711 __ eorw(as_Register($dst$$reg),
11712 as_Register($src1$$reg),
11713 as_Register($src2$$reg),
11714 Assembler::ASR,
11715 $src3$$constant & 0x1f);
11716 %}
11717
11718 ins_pipe(ialu_reg_reg_shift);
11719 %}
11720
11721 // This pattern is automatically generated from aarch64_ad.m4
11722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11723 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11724 iRegL src1, iRegL src2,
11725 immI src3) %{
11726 match(Set dst (XorL src1 (RShiftL src2 src3)));
11727
11728 ins_cost(1.9 * INSN_COST);
11729 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11730
11731 ins_encode %{
11732 __ eor(as_Register($dst$$reg),
11733 as_Register($src1$$reg),
11734 as_Register($src2$$reg),
11735 Assembler::ASR,
11736 $src3$$constant & 0x3f);
11737 %}
11738
11739 ins_pipe(ialu_reg_reg_shift);
11740 %}
11741
11742 // This pattern is automatically generated from aarch64_ad.m4
11743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11744 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11745 iRegIorL2I src1, iRegIorL2I src2,
11746 immI src3) %{
11747 match(Set dst (XorI src1 (LShiftI src2 src3)));
11748
11749 ins_cost(1.9 * INSN_COST);
11750 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11751
11752 ins_encode %{
11753 __ eorw(as_Register($dst$$reg),
11754 as_Register($src1$$reg),
11755 as_Register($src2$$reg),
11756 Assembler::LSL,
11757 $src3$$constant & 0x1f);
11758 %}
11759
11760 ins_pipe(ialu_reg_reg_shift);
11761 %}
11762
11763 // This pattern is automatically generated from aarch64_ad.m4
11764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11765 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11766 iRegL src1, iRegL src2,
11767 immI src3) %{
11768 match(Set dst (XorL src1 (LShiftL src2 src3)));
11769
11770 ins_cost(1.9 * INSN_COST);
11771 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11772
11773 ins_encode %{
11774 __ eor(as_Register($dst$$reg),
11775 as_Register($src1$$reg),
11776 as_Register($src2$$reg),
11777 Assembler::LSL,
11778 $src3$$constant & 0x3f);
11779 %}
11780
11781 ins_pipe(ialu_reg_reg_shift);
11782 %}
11783
11784 // This pattern is automatically generated from aarch64_ad.m4
11785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11786 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11787 iRegIorL2I src1, iRegIorL2I src2,
11788 immI src3) %{
11789 match(Set dst (XorI src1 (RotateRight src2 src3)));
11790
11791 ins_cost(1.9 * INSN_COST);
11792 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11793
11794 ins_encode %{
11795 __ eorw(as_Register($dst$$reg),
11796 as_Register($src1$$reg),
11797 as_Register($src2$$reg),
11798 Assembler::ROR,
11799 $src3$$constant & 0x1f);
11800 %}
11801
11802 ins_pipe(ialu_reg_reg_shift);
11803 %}
11804
11805 // This pattern is automatically generated from aarch64_ad.m4
11806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11807 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11808 iRegL src1, iRegL src2,
11809 immI src3) %{
11810 match(Set dst (XorL src1 (RotateRight src2 src3)));
11811
11812 ins_cost(1.9 * INSN_COST);
11813 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11814
11815 ins_encode %{
11816 __ eor(as_Register($dst$$reg),
11817 as_Register($src1$$reg),
11818 as_Register($src2$$reg),
11819 Assembler::ROR,
11820 $src3$$constant & 0x3f);
11821 %}
11822
11823 ins_pipe(ialu_reg_reg_shift);
11824 %}
11825
11826 // This pattern is automatically generated from aarch64_ad.m4
11827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11828 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11829 iRegIorL2I src1, iRegIorL2I src2,
11830 immI src3) %{
11831 match(Set dst (OrI src1 (URShiftI src2 src3)));
11832
11833 ins_cost(1.9 * INSN_COST);
11834 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11835
11836 ins_encode %{
11837 __ orrw(as_Register($dst$$reg),
11838 as_Register($src1$$reg),
11839 as_Register($src2$$reg),
11840 Assembler::LSR,
11841 $src3$$constant & 0x1f);
11842 %}
11843
11844 ins_pipe(ialu_reg_reg_shift);
11845 %}
11846
11847 // This pattern is automatically generated from aarch64_ad.m4
11848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11849 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11850 iRegL src1, iRegL src2,
11851 immI src3) %{
11852 match(Set dst (OrL src1 (URShiftL src2 src3)));
11853
11854 ins_cost(1.9 * INSN_COST);
11855 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11856
11857 ins_encode %{
11858 __ orr(as_Register($dst$$reg),
11859 as_Register($src1$$reg),
11860 as_Register($src2$$reg),
11861 Assembler::LSR,
11862 $src3$$constant & 0x3f);
11863 %}
11864
11865 ins_pipe(ialu_reg_reg_shift);
11866 %}
11867
11868 // This pattern is automatically generated from aarch64_ad.m4
11869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11870 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11871 iRegIorL2I src1, iRegIorL2I src2,
11872 immI src3) %{
11873 match(Set dst (OrI src1 (RShiftI src2 src3)));
11874
11875 ins_cost(1.9 * INSN_COST);
11876 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11877
11878 ins_encode %{
11879 __ orrw(as_Register($dst$$reg),
11880 as_Register($src1$$reg),
11881 as_Register($src2$$reg),
11882 Assembler::ASR,
11883 $src3$$constant & 0x1f);
11884 %}
11885
11886 ins_pipe(ialu_reg_reg_shift);
11887 %}
11888
11889 // This pattern is automatically generated from aarch64_ad.m4
11890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11891 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11892 iRegL src1, iRegL src2,
11893 immI src3) %{
11894 match(Set dst (OrL src1 (RShiftL src2 src3)));
11895
11896 ins_cost(1.9 * INSN_COST);
11897 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11898
11899 ins_encode %{
11900 __ orr(as_Register($dst$$reg),
11901 as_Register($src1$$reg),
11902 as_Register($src2$$reg),
11903 Assembler::ASR,
11904 $src3$$constant & 0x3f);
11905 %}
11906
11907 ins_pipe(ialu_reg_reg_shift);
11908 %}
11909
11910 // This pattern is automatically generated from aarch64_ad.m4
11911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11912 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11913 iRegIorL2I src1, iRegIorL2I src2,
11914 immI src3) %{
11915 match(Set dst (OrI src1 (LShiftI src2 src3)));
11916
11917 ins_cost(1.9 * INSN_COST);
11918 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11919
11920 ins_encode %{
11921 __ orrw(as_Register($dst$$reg),
11922 as_Register($src1$$reg),
11923 as_Register($src2$$reg),
11924 Assembler::LSL,
11925 $src3$$constant & 0x1f);
11926 %}
11927
11928 ins_pipe(ialu_reg_reg_shift);
11929 %}
11930
11931 // This pattern is automatically generated from aarch64_ad.m4
11932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11933 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11934 iRegL src1, iRegL src2,
11935 immI src3) %{
11936 match(Set dst (OrL src1 (LShiftL src2 src3)));
11937
11938 ins_cost(1.9 * INSN_COST);
11939 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11940
11941 ins_encode %{
11942 __ orr(as_Register($dst$$reg),
11943 as_Register($src1$$reg),
11944 as_Register($src2$$reg),
11945 Assembler::LSL,
11946 $src3$$constant & 0x3f);
11947 %}
11948
11949 ins_pipe(ialu_reg_reg_shift);
11950 %}
11951
11952 // This pattern is automatically generated from aarch64_ad.m4
11953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11954 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11955 iRegIorL2I src1, iRegIorL2I src2,
11956 immI src3) %{
11957 match(Set dst (OrI src1 (RotateRight src2 src3)));
11958
11959 ins_cost(1.9 * INSN_COST);
11960 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11961
11962 ins_encode %{
11963 __ orrw(as_Register($dst$$reg),
11964 as_Register($src1$$reg),
11965 as_Register($src2$$reg),
11966 Assembler::ROR,
11967 $src3$$constant & 0x1f);
11968 %}
11969
11970 ins_pipe(ialu_reg_reg_shift);
11971 %}
11972
11973 // This pattern is automatically generated from aarch64_ad.m4
11974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11975 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11976 iRegL src1, iRegL src2,
11977 immI src3) %{
11978 match(Set dst (OrL src1 (RotateRight src2 src3)));
11979
11980 ins_cost(1.9 * INSN_COST);
11981 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11982
11983 ins_encode %{
11984 __ orr(as_Register($dst$$reg),
11985 as_Register($src1$$reg),
11986 as_Register($src2$$reg),
11987 Assembler::ROR,
11988 $src3$$constant & 0x3f);
11989 %}
11990
11991 ins_pipe(ialu_reg_reg_shift);
11992 %}
11993
11994 // This pattern is automatically generated from aarch64_ad.m4
11995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11996 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11997 iRegIorL2I src1, iRegIorL2I src2,
11998 immI src3) %{
11999 match(Set dst (AddI src1 (URShiftI src2 src3)));
12000
12001 ins_cost(1.9 * INSN_COST);
12002 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12003
12004 ins_encode %{
12005 __ addw(as_Register($dst$$reg),
12006 as_Register($src1$$reg),
12007 as_Register($src2$$reg),
12008 Assembler::LSR,
12009 $src3$$constant & 0x1f);
12010 %}
12011
12012 ins_pipe(ialu_reg_reg_shift);
12013 %}
12014
12015 // This pattern is automatically generated from aarch64_ad.m4
12016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12017 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12018 iRegL src1, iRegL src2,
12019 immI src3) %{
12020 match(Set dst (AddL src1 (URShiftL src2 src3)));
12021
12022 ins_cost(1.9 * INSN_COST);
12023 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12024
12025 ins_encode %{
12026 __ add(as_Register($dst$$reg),
12027 as_Register($src1$$reg),
12028 as_Register($src2$$reg),
12029 Assembler::LSR,
12030 $src3$$constant & 0x3f);
12031 %}
12032
12033 ins_pipe(ialu_reg_reg_shift);
12034 %}
12035
12036 // This pattern is automatically generated from aarch64_ad.m4
12037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12038 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12039 iRegIorL2I src1, iRegIorL2I src2,
12040 immI src3) %{
12041 match(Set dst (AddI src1 (RShiftI src2 src3)));
12042
12043 ins_cost(1.9 * INSN_COST);
12044 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12045
12046 ins_encode %{
12047 __ addw(as_Register($dst$$reg),
12048 as_Register($src1$$reg),
12049 as_Register($src2$$reg),
12050 Assembler::ASR,
12051 $src3$$constant & 0x1f);
12052 %}
12053
12054 ins_pipe(ialu_reg_reg_shift);
12055 %}
12056
12057 // This pattern is automatically generated from aarch64_ad.m4
12058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12059 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12060 iRegL src1, iRegL src2,
12061 immI src3) %{
12062 match(Set dst (AddL src1 (RShiftL src2 src3)));
12063
12064 ins_cost(1.9 * INSN_COST);
12065 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12066
12067 ins_encode %{
12068 __ add(as_Register($dst$$reg),
12069 as_Register($src1$$reg),
12070 as_Register($src2$$reg),
12071 Assembler::ASR,
12072 $src3$$constant & 0x3f);
12073 %}
12074
12075 ins_pipe(ialu_reg_reg_shift);
12076 %}
12077
12078 // This pattern is automatically generated from aarch64_ad.m4
12079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12080 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12081 iRegIorL2I src1, iRegIorL2I src2,
12082 immI src3) %{
12083 match(Set dst (AddI src1 (LShiftI src2 src3)));
12084
12085 ins_cost(1.9 * INSN_COST);
12086 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12087
12088 ins_encode %{
12089 __ addw(as_Register($dst$$reg),
12090 as_Register($src1$$reg),
12091 as_Register($src2$$reg),
12092 Assembler::LSL,
12093 $src3$$constant & 0x1f);
12094 %}
12095
12096 ins_pipe(ialu_reg_reg_shift);
12097 %}
12098
12099 // This pattern is automatically generated from aarch64_ad.m4
12100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12101 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12102 iRegL src1, iRegL src2,
12103 immI src3) %{
12104 match(Set dst (AddL src1 (LShiftL src2 src3)));
12105
12106 ins_cost(1.9 * INSN_COST);
12107 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12108
12109 ins_encode %{
12110 __ add(as_Register($dst$$reg),
12111 as_Register($src1$$reg),
12112 as_Register($src2$$reg),
12113 Assembler::LSL,
12114 $src3$$constant & 0x3f);
12115 %}
12116
12117 ins_pipe(ialu_reg_reg_shift);
12118 %}
12119
12120 // This pattern is automatically generated from aarch64_ad.m4
12121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12122 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12123 iRegIorL2I src1, iRegIorL2I src2,
12124 immI src3) %{
12125 match(Set dst (SubI src1 (URShiftI src2 src3)));
12126
12127 ins_cost(1.9 * INSN_COST);
12128 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12129
12130 ins_encode %{
12131 __ subw(as_Register($dst$$reg),
12132 as_Register($src1$$reg),
12133 as_Register($src2$$reg),
12134 Assembler::LSR,
12135 $src3$$constant & 0x1f);
12136 %}
12137
12138 ins_pipe(ialu_reg_reg_shift);
12139 %}
12140
12141 // This pattern is automatically generated from aarch64_ad.m4
12142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12143 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12144 iRegL src1, iRegL src2,
12145 immI src3) %{
12146 match(Set dst (SubL src1 (URShiftL src2 src3)));
12147
12148 ins_cost(1.9 * INSN_COST);
12149 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12150
12151 ins_encode %{
12152 __ sub(as_Register($dst$$reg),
12153 as_Register($src1$$reg),
12154 as_Register($src2$$reg),
12155 Assembler::LSR,
12156 $src3$$constant & 0x3f);
12157 %}
12158
12159 ins_pipe(ialu_reg_reg_shift);
12160 %}
12161
12162 // This pattern is automatically generated from aarch64_ad.m4
12163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12164 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12165 iRegIorL2I src1, iRegIorL2I src2,
12166 immI src3) %{
12167 match(Set dst (SubI src1 (RShiftI src2 src3)));
12168
12169 ins_cost(1.9 * INSN_COST);
12170 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12171
12172 ins_encode %{
12173 __ subw(as_Register($dst$$reg),
12174 as_Register($src1$$reg),
12175 as_Register($src2$$reg),
12176 Assembler::ASR,
12177 $src3$$constant & 0x1f);
12178 %}
12179
12180 ins_pipe(ialu_reg_reg_shift);
12181 %}
12182
12183 // This pattern is automatically generated from aarch64_ad.m4
12184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12185 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12186 iRegL src1, iRegL src2,
12187 immI src3) %{
12188 match(Set dst (SubL src1 (RShiftL src2 src3)));
12189
12190 ins_cost(1.9 * INSN_COST);
12191 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12192
12193 ins_encode %{
12194 __ sub(as_Register($dst$$reg),
12195 as_Register($src1$$reg),
12196 as_Register($src2$$reg),
12197 Assembler::ASR,
12198 $src3$$constant & 0x3f);
12199 %}
12200
12201 ins_pipe(ialu_reg_reg_shift);
12202 %}
12203
12204 // This pattern is automatically generated from aarch64_ad.m4
12205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12206 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12207 iRegIorL2I src1, iRegIorL2I src2,
12208 immI src3) %{
12209 match(Set dst (SubI src1 (LShiftI src2 src3)));
12210
12211 ins_cost(1.9 * INSN_COST);
12212 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12213
12214 ins_encode %{
12215 __ subw(as_Register($dst$$reg),
12216 as_Register($src1$$reg),
12217 as_Register($src2$$reg),
12218 Assembler::LSL,
12219 $src3$$constant & 0x1f);
12220 %}
12221
12222 ins_pipe(ialu_reg_reg_shift);
12223 %}
12224
12225 // This pattern is automatically generated from aarch64_ad.m4
12226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12227 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12228 iRegL src1, iRegL src2,
12229 immI src3) %{
12230 match(Set dst (SubL src1 (LShiftL src2 src3)));
12231
12232 ins_cost(1.9 * INSN_COST);
12233 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12234
12235 ins_encode %{
12236 __ sub(as_Register($dst$$reg),
12237 as_Register($src1$$reg),
12238 as_Register($src2$$reg),
12239 Assembler::LSL,
12240 $src3$$constant & 0x3f);
12241 %}
12242
12243 ins_pipe(ialu_reg_reg_shift);
12244 %}
12245
12246 // This pattern is automatically generated from aarch64_ad.m4
12247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12248
12249 // Shift Left followed by Shift Right.
12250 // This idiom is used by the compiler for the i2b bytecode etc.
12251 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12252 %{
12253 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12254 ins_cost(INSN_COST * 2);
12255 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12256 ins_encode %{
12257 int lshift = $lshift_count$$constant & 63;
12258 int rshift = $rshift_count$$constant & 63;
12259 int s = 63 - lshift;
12260 int r = (rshift - lshift) & 63;
12261 __ sbfm(as_Register($dst$$reg),
12262 as_Register($src$$reg),
12263 r, s);
12264 %}
12265
12266 ins_pipe(ialu_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271
12272 // Shift Left followed by Shift Right.
12273 // This idiom is used by the compiler for the i2b bytecode etc.
12274 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12275 %{
12276 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12277 ins_cost(INSN_COST * 2);
12278 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12279 ins_encode %{
12280 int lshift = $lshift_count$$constant & 31;
12281 int rshift = $rshift_count$$constant & 31;
12282 int s = 31 - lshift;
12283 int r = (rshift - lshift) & 31;
12284 __ sbfmw(as_Register($dst$$reg),
12285 as_Register($src$$reg),
12286 r, s);
12287 %}
12288
12289 ins_pipe(ialu_reg_shift);
12290 %}
12291
12292 // This pattern is automatically generated from aarch64_ad.m4
12293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12294
12295 // Shift Left followed by Shift Right.
12296 // This idiom is used by the compiler for the i2b bytecode etc.
12297 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12298 %{
12299 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12300 ins_cost(INSN_COST * 2);
12301 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12302 ins_encode %{
12303 int lshift = $lshift_count$$constant & 63;
12304 int rshift = $rshift_count$$constant & 63;
12305 int s = 63 - lshift;
12306 int r = (rshift - lshift) & 63;
12307 __ ubfm(as_Register($dst$$reg),
12308 as_Register($src$$reg),
12309 r, s);
12310 %}
12311
12312 ins_pipe(ialu_reg_shift);
12313 %}
12314
12315 // This pattern is automatically generated from aarch64_ad.m4
12316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12317
12318 // Shift Left followed by Shift Right.
12319 // This idiom is used by the compiler for the i2b bytecode etc.
12320 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12321 %{
12322 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12323 ins_cost(INSN_COST * 2);
12324 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12325 ins_encode %{
12326 int lshift = $lshift_count$$constant & 31;
12327 int rshift = $rshift_count$$constant & 31;
12328 int s = 31 - lshift;
12329 int r = (rshift - lshift) & 31;
12330 __ ubfmw(as_Register($dst$$reg),
12331 as_Register($src$$reg),
12332 r, s);
12333 %}
12334
12335 ins_pipe(ialu_reg_shift);
12336 %}
12337
12338 // Bitfield extract with shift & mask
12339
12340 // This pattern is automatically generated from aarch64_ad.m4
12341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12342 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12343 %{
12344 match(Set dst (AndI (URShiftI src rshift) mask));
12345 // Make sure we are not going to exceed what ubfxw can do.
12346 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12347
12348 ins_cost(INSN_COST);
12349 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12350 ins_encode %{
12351 int rshift = $rshift$$constant & 31;
12352 intptr_t mask = $mask$$constant;
12353 int width = exact_log2(mask+1);
12354 __ ubfxw(as_Register($dst$$reg),
12355 as_Register($src$$reg), rshift, width);
12356 %}
12357 ins_pipe(ialu_reg_shift);
12358 %}
12359
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12363 %{
12364 match(Set dst (AndL (URShiftL src rshift) mask));
12365 // Make sure we are not going to exceed what ubfx can do.
12366 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12367
12368 ins_cost(INSN_COST);
12369 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12370 ins_encode %{
12371 int rshift = $rshift$$constant & 63;
12372 intptr_t mask = $mask$$constant;
12373 int width = exact_log2_long(mask+1);
12374 __ ubfx(as_Register($dst$$reg),
12375 as_Register($src$$reg), rshift, width);
12376 %}
12377 ins_pipe(ialu_reg_shift);
12378 %}
12379
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383
12384 // We can use ubfx when extending an And with a mask when we know mask
12385 // is positive. We know that because immI_bitmask guarantees it.
12386 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12387 %{
12388 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12389 // Make sure we are not going to exceed what ubfxw can do.
12390 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12391
12392 ins_cost(INSN_COST * 2);
12393 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12394 ins_encode %{
12395 int rshift = $rshift$$constant & 31;
12396 intptr_t mask = $mask$$constant;
12397 int width = exact_log2(mask+1);
12398 __ ubfx(as_Register($dst$$reg),
12399 as_Register($src$$reg), rshift, width);
12400 %}
12401 ins_pipe(ialu_reg_shift);
12402 %}
12403
12404
12405 // This pattern is automatically generated from aarch64_ad.m4
12406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12407
12408 // We can use ubfiz when masking by a positive number and then left shifting the result.
12409 // We know that the mask is positive because immI_bitmask guarantees it.
12410 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12411 %{
12412 match(Set dst (LShiftI (AndI src mask) lshift));
12413 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12414
12415 ins_cost(INSN_COST);
12416 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12417 ins_encode %{
12418 int lshift = $lshift$$constant & 31;
12419 intptr_t mask = $mask$$constant;
12420 int width = exact_log2(mask+1);
12421 __ ubfizw(as_Register($dst$$reg),
12422 as_Register($src$$reg), lshift, width);
12423 %}
12424 ins_pipe(ialu_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429
12430 // We can use ubfiz when masking by a positive number and then left shifting the result.
12431 // We know that the mask is positive because immL_bitmask guarantees it.
12432 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12433 %{
12434 match(Set dst (LShiftL (AndL src mask) lshift));
12435 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12436
12437 ins_cost(INSN_COST);
12438 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12439 ins_encode %{
12440 int lshift = $lshift$$constant & 63;
12441 intptr_t mask = $mask$$constant;
12442 int width = exact_log2_long(mask+1);
12443 __ ubfiz(as_Register($dst$$reg),
12444 as_Register($src$$reg), lshift, width);
12445 %}
12446 ins_pipe(ialu_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451
12452 // We can use ubfiz when masking by a positive number and then left shifting the result.
12453 // We know that the mask is positive because immI_bitmask guarantees it.
12454 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12455 %{
12456 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12457 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12458
12459 ins_cost(INSN_COST);
12460 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12461 ins_encode %{
12462 int lshift = $lshift$$constant & 31;
12463 intptr_t mask = $mask$$constant;
12464 int width = exact_log2(mask+1);
12465 __ ubfizw(as_Register($dst$$reg),
12466 as_Register($src$$reg), lshift, width);
12467 %}
12468 ins_pipe(ialu_reg_shift);
12469 %}
12470
12471 // This pattern is automatically generated from aarch64_ad.m4
12472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12473
12474 // We can use ubfiz when masking by a positive number and then left shifting the result.
12475 // We know that the mask is positive because immL_bitmask guarantees it.
12476 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12477 %{
12478 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12479 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12480
12481 ins_cost(INSN_COST);
12482 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12483 ins_encode %{
12484 int lshift = $lshift$$constant & 63;
12485 intptr_t mask = $mask$$constant;
12486 int width = exact_log2_long(mask+1);
12487 __ ubfiz(as_Register($dst$$reg),
12488 as_Register($src$$reg), lshift, width);
12489 %}
12490 ins_pipe(ialu_reg_shift);
12491 %}
12492
12493
12494 // This pattern is automatically generated from aarch64_ad.m4
12495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12496
12497 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12498 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12499 %{
12500 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12501 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12502
12503 ins_cost(INSN_COST);
12504 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12505 ins_encode %{
12506 int lshift = $lshift$$constant & 63;
12507 intptr_t mask = $mask$$constant;
12508 int width = exact_log2(mask+1);
12509 __ ubfiz(as_Register($dst$$reg),
12510 as_Register($src$$reg), lshift, width);
12511 %}
12512 ins_pipe(ialu_reg_shift);
12513 %}
12514
12515 // This pattern is automatically generated from aarch64_ad.m4
12516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12517
12518 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12519 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12520 %{
12521 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12522 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12523
12524 ins_cost(INSN_COST);
12525 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12526 ins_encode %{
12527 int lshift = $lshift$$constant & 31;
12528 intptr_t mask = $mask$$constant;
12529 int width = exact_log2(mask+1);
12530 __ ubfiz(as_Register($dst$$reg),
12531 as_Register($src$$reg), lshift, width);
12532 %}
12533 ins_pipe(ialu_reg_shift);
12534 %}
12535
12536 // This pattern is automatically generated from aarch64_ad.m4
12537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12538
12539 // Can skip int2long conversions after AND with small bitmask
12540 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12541 %{
12542 match(Set dst (ConvI2L (AndI src msk)));
12543 ins_cost(INSN_COST);
12544 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12545 ins_encode %{
12546 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12547 %}
12548 ins_pipe(ialu_reg_shift);
12549 %}
12550
12551
12552 // Rotations
12553
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12557 %{
12558 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12559 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12560
12561 ins_cost(INSN_COST);
12562 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12563
12564 ins_encode %{
12565 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12566 $rshift$$constant & 63);
12567 %}
12568 ins_pipe(ialu_reg_reg_extr);
12569 %}
12570
12571
12572 // This pattern is automatically generated from aarch64_ad.m4
12573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12574 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12575 %{
12576 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12577 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12578
12579 ins_cost(INSN_COST);
12580 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12581
12582 ins_encode %{
12583 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12584 $rshift$$constant & 31);
12585 %}
12586 ins_pipe(ialu_reg_reg_extr);
12587 %}
12588
12589
12590 // This pattern is automatically generated from aarch64_ad.m4
12591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12592 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12593 %{
12594 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12595 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12596
12597 ins_cost(INSN_COST);
12598 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12599
12600 ins_encode %{
12601 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12602 $rshift$$constant & 63);
12603 %}
12604 ins_pipe(ialu_reg_reg_extr);
12605 %}
12606
12607
12608 // This pattern is automatically generated from aarch64_ad.m4
12609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12610 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12611 %{
12612 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12613 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12614
12615 ins_cost(INSN_COST);
12616 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12617
12618 ins_encode %{
12619 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12620 $rshift$$constant & 31);
12621 %}
12622 ins_pipe(ialu_reg_reg_extr);
12623 %}
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12628 %{
12629 match(Set dst (RotateRight src shift));
12630
12631 ins_cost(INSN_COST);
12632 format %{ "ror $dst, $src, $shift" %}
12633
12634 ins_encode %{
12635 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12636 $shift$$constant & 0x1f);
12637 %}
12638 ins_pipe(ialu_reg_reg_vshift);
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12644 %{
12645 match(Set dst (RotateRight src shift));
12646
12647 ins_cost(INSN_COST);
12648 format %{ "ror $dst, $src, $shift" %}
12649
12650 ins_encode %{
12651 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12652 $shift$$constant & 0x3f);
12653 %}
12654 ins_pipe(ialu_reg_reg_vshift);
12655 %}
12656
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12660 %{
12661 match(Set dst (RotateRight src shift));
12662
12663 ins_cost(INSN_COST);
12664 format %{ "ror $dst, $src, $shift" %}
12665
12666 ins_encode %{
12667 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12668 %}
12669 ins_pipe(ialu_reg_reg_vshift);
12670 %}
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12675 %{
12676 match(Set dst (RotateRight src shift));
12677
12678 ins_cost(INSN_COST);
12679 format %{ "ror $dst, $src, $shift" %}
12680
12681 ins_encode %{
12682 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12683 %}
12684 ins_pipe(ialu_reg_reg_vshift);
12685 %}
12686
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12690 %{
12691 match(Set dst (RotateLeft src shift));
12692
12693 ins_cost(INSN_COST);
12694 format %{ "rol $dst, $src, $shift" %}
12695
12696 ins_encode %{
12697 __ subw(rscratch1, zr, as_Register($shift$$reg));
12698 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12699 %}
12700 ins_pipe(ialu_reg_reg_vshift);
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12706 %{
12707 match(Set dst (RotateLeft src shift));
12708
12709 ins_cost(INSN_COST);
12710 format %{ "rol $dst, $src, $shift" %}
12711
12712 ins_encode %{
12713 __ subw(rscratch1, zr, as_Register($shift$$reg));
12714 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12715 %}
12716 ins_pipe(ialu_reg_reg_vshift);
12717 %}
12718
12719
12720 // Add/subtract (extended)
12721
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12725 %{
12726 match(Set dst (AddL src1 (ConvI2L src2)));
12727 ins_cost(INSN_COST);
12728 format %{ "add $dst, $src1, $src2, sxtw" %}
12729
12730 ins_encode %{
12731 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12732 as_Register($src2$$reg), ext::sxtw);
12733 %}
12734 ins_pipe(ialu_reg_reg);
12735 %}
12736
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12740 %{
12741 match(Set dst (SubL src1 (ConvI2L src2)));
12742 ins_cost(INSN_COST);
12743 format %{ "sub $dst, $src1, $src2, sxtw" %}
12744
12745 ins_encode %{
12746 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12747 as_Register($src2$$reg), ext::sxtw);
12748 %}
12749 ins_pipe(ialu_reg_reg);
12750 %}
12751
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12755 %{
12756 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12757 ins_cost(INSN_COST);
12758 format %{ "add $dst, $src1, $src2, sxth" %}
12759
12760 ins_encode %{
12761 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12762 as_Register($src2$$reg), ext::sxth);
12763 %}
12764 ins_pipe(ialu_reg_reg);
12765 %}
12766
12767 // This pattern is automatically generated from aarch64_ad.m4
12768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12769 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12770 %{
12771 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12772 ins_cost(INSN_COST);
12773 format %{ "add $dst, $src1, $src2, sxtb" %}
12774
12775 ins_encode %{
12776 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12777 as_Register($src2$$reg), ext::sxtb);
12778 %}
12779 ins_pipe(ialu_reg_reg);
12780 %}
12781
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12785 %{
12786 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12787 ins_cost(INSN_COST);
12788 format %{ "add $dst, $src1, $src2, uxtb" %}
12789
12790 ins_encode %{
12791 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12792 as_Register($src2$$reg), ext::uxtb);
12793 %}
12794 ins_pipe(ialu_reg_reg);
12795 %}
12796
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12800 %{
12801 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12802 ins_cost(INSN_COST);
12803 format %{ "add $dst, $src1, $src2, sxth" %}
12804
12805 ins_encode %{
12806 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12807 as_Register($src2$$reg), ext::sxth);
12808 %}
12809 ins_pipe(ialu_reg_reg);
12810 %}
12811
12812 // This pattern is automatically generated from aarch64_ad.m4
12813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12814 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12815 %{
12816 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12817 ins_cost(INSN_COST);
12818 format %{ "add $dst, $src1, $src2, sxtw" %}
12819
12820 ins_encode %{
12821 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12822 as_Register($src2$$reg), ext::sxtw);
12823 %}
12824 ins_pipe(ialu_reg_reg);
12825 %}
12826
12827 // This pattern is automatically generated from aarch64_ad.m4
12828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12829 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12830 %{
12831 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12832 ins_cost(INSN_COST);
12833 format %{ "add $dst, $src1, $src2, sxtb" %}
12834
12835 ins_encode %{
12836 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12837 as_Register($src2$$reg), ext::sxtb);
12838 %}
12839 ins_pipe(ialu_reg_reg);
12840 %}
12841
12842 // This pattern is automatically generated from aarch64_ad.m4
12843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12844 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12845 %{
12846 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12847 ins_cost(INSN_COST);
12848 format %{ "add $dst, $src1, $src2, uxtb" %}
12849
12850 ins_encode %{
12851 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12852 as_Register($src2$$reg), ext::uxtb);
12853 %}
12854 ins_pipe(ialu_reg_reg);
12855 %}
12856
12857 // This pattern is automatically generated from aarch64_ad.m4
12858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12859 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12860 %{
12861 match(Set dst (AddI src1 (AndI src2 mask)));
12862 ins_cost(INSN_COST);
12863 format %{ "addw $dst, $src1, $src2, uxtb" %}
12864
12865 ins_encode %{
12866 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12867 as_Register($src2$$reg), ext::uxtb);
12868 %}
12869 ins_pipe(ialu_reg_reg);
12870 %}
12871
12872 // This pattern is automatically generated from aarch64_ad.m4
12873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12874 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12875 %{
12876 match(Set dst (AddI src1 (AndI src2 mask)));
12877 ins_cost(INSN_COST);
12878 format %{ "addw $dst, $src1, $src2, uxth" %}
12879
12880 ins_encode %{
12881 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12882 as_Register($src2$$reg), ext::uxth);
12883 %}
12884 ins_pipe(ialu_reg_reg);
12885 %}
12886
12887 // This pattern is automatically generated from aarch64_ad.m4
12888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12889 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12890 %{
12891 match(Set dst (AddL src1 (AndL src2 mask)));
12892 ins_cost(INSN_COST);
12893 format %{ "add $dst, $src1, $src2, uxtb" %}
12894
12895 ins_encode %{
12896 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12897 as_Register($src2$$reg), ext::uxtb);
12898 %}
12899 ins_pipe(ialu_reg_reg);
12900 %}
12901
12902 // This pattern is automatically generated from aarch64_ad.m4
12903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12904 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12905 %{
12906 match(Set dst (AddL src1 (AndL src2 mask)));
12907 ins_cost(INSN_COST);
12908 format %{ "add $dst, $src1, $src2, uxth" %}
12909
12910 ins_encode %{
12911 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12912 as_Register($src2$$reg), ext::uxth);
12913 %}
12914 ins_pipe(ialu_reg_reg);
12915 %}
12916
12917 // This pattern is automatically generated from aarch64_ad.m4
12918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12919 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12920 %{
12921 match(Set dst (AddL src1 (AndL src2 mask)));
12922 ins_cost(INSN_COST);
12923 format %{ "add $dst, $src1, $src2, uxtw" %}
12924
12925 ins_encode %{
12926 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12927 as_Register($src2$$reg), ext::uxtw);
12928 %}
12929 ins_pipe(ialu_reg_reg);
12930 %}
12931
12932 // This pattern is automatically generated from aarch64_ad.m4
12933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12934 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12935 %{
12936 match(Set dst (SubI src1 (AndI src2 mask)));
12937 ins_cost(INSN_COST);
12938 format %{ "subw $dst, $src1, $src2, uxtb" %}
12939
12940 ins_encode %{
12941 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12942 as_Register($src2$$reg), ext::uxtb);
12943 %}
12944 ins_pipe(ialu_reg_reg);
12945 %}
12946
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12950 %{
12951 match(Set dst (SubI src1 (AndI src2 mask)));
12952 ins_cost(INSN_COST);
12953 format %{ "subw $dst, $src1, $src2, uxth" %}
12954
12955 ins_encode %{
12956 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12957 as_Register($src2$$reg), ext::uxth);
12958 %}
12959 ins_pipe(ialu_reg_reg);
12960 %}
12961
12962 // This pattern is automatically generated from aarch64_ad.m4
12963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12964 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12965 %{
12966 match(Set dst (SubL src1 (AndL src2 mask)));
12967 ins_cost(INSN_COST);
12968 format %{ "sub $dst, $src1, $src2, uxtb" %}
12969
12970 ins_encode %{
12971 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12972 as_Register($src2$$reg), ext::uxtb);
12973 %}
12974 ins_pipe(ialu_reg_reg);
12975 %}
12976
12977 // This pattern is automatically generated from aarch64_ad.m4
12978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12979 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12980 %{
12981 match(Set dst (SubL src1 (AndL src2 mask)));
12982 ins_cost(INSN_COST);
12983 format %{ "sub $dst, $src1, $src2, uxth" %}
12984
12985 ins_encode %{
12986 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12987 as_Register($src2$$reg), ext::uxth);
12988 %}
12989 ins_pipe(ialu_reg_reg);
12990 %}
12991
12992 // This pattern is automatically generated from aarch64_ad.m4
12993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12994 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12995 %{
12996 match(Set dst (SubL src1 (AndL src2 mask)));
12997 ins_cost(INSN_COST);
12998 format %{ "sub $dst, $src1, $src2, uxtw" %}
12999
13000 ins_encode %{
13001 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13002 as_Register($src2$$reg), ext::uxtw);
13003 %}
13004 ins_pipe(ialu_reg_reg);
13005 %}
13006
13007
13008 // This pattern is automatically generated from aarch64_ad.m4
13009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13010 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13011 %{
13012 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13013 ins_cost(1.9 * INSN_COST);
13014 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13015
13016 ins_encode %{
13017 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13018 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13019 %}
13020 ins_pipe(ialu_reg_reg_shift);
13021 %}
13022
13023 // This pattern is automatically generated from aarch64_ad.m4
13024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13025 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13026 %{
13027 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13028 ins_cost(1.9 * INSN_COST);
13029 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13030
13031 ins_encode %{
13032 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13033 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13034 %}
13035 ins_pipe(ialu_reg_reg_shift);
13036 %}
13037
13038 // This pattern is automatically generated from aarch64_ad.m4
13039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13040 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13041 %{
13042 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13043 ins_cost(1.9 * INSN_COST);
13044 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13045
13046 ins_encode %{
13047 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13048 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13049 %}
13050 ins_pipe(ialu_reg_reg_shift);
13051 %}
13052
13053 // This pattern is automatically generated from aarch64_ad.m4
13054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13055 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13056 %{
13057 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13058 ins_cost(1.9 * INSN_COST);
13059 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13060
13061 ins_encode %{
13062 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13063 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13064 %}
13065 ins_pipe(ialu_reg_reg_shift);
13066 %}
13067
13068 // This pattern is automatically generated from aarch64_ad.m4
13069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13070 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13071 %{
13072 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13073 ins_cost(1.9 * INSN_COST);
13074 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13075
13076 ins_encode %{
13077 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13078 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13079 %}
13080 ins_pipe(ialu_reg_reg_shift);
13081 %}
13082
13083 // This pattern is automatically generated from aarch64_ad.m4
13084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13085 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13086 %{
13087 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13088 ins_cost(1.9 * INSN_COST);
13089 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13090
13091 ins_encode %{
13092 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13093 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13094 %}
13095 ins_pipe(ialu_reg_reg_shift);
13096 %}
13097
13098 // This pattern is automatically generated from aarch64_ad.m4
13099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13100 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13101 %{
13102 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13103 ins_cost(1.9 * INSN_COST);
13104 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13105
13106 ins_encode %{
13107 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13108 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13109 %}
13110 ins_pipe(ialu_reg_reg_shift);
13111 %}
13112
13113 // This pattern is automatically generated from aarch64_ad.m4
13114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13115 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13116 %{
13117 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13118 ins_cost(1.9 * INSN_COST);
13119 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13120
13121 ins_encode %{
13122 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13123 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13124 %}
13125 ins_pipe(ialu_reg_reg_shift);
13126 %}
13127
13128 // This pattern is automatically generated from aarch64_ad.m4
13129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13130 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13131 %{
13132 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13133 ins_cost(1.9 * INSN_COST);
13134 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13135
13136 ins_encode %{
13137 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13138 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13139 %}
13140 ins_pipe(ialu_reg_reg_shift);
13141 %}
13142
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13146 %{
13147 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13148 ins_cost(1.9 * INSN_COST);
13149 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13150
13151 ins_encode %{
13152 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13153 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13154 %}
13155 ins_pipe(ialu_reg_reg_shift);
13156 %}
13157
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13161 %{
13162 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13163 ins_cost(1.9 * INSN_COST);
13164 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13165
13166 ins_encode %{
13167 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13168 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13169 %}
13170 ins_pipe(ialu_reg_reg_shift);
13171 %}
13172
13173 // This pattern is automatically generated from aarch64_ad.m4
13174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13175 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13176 %{
13177 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13178 ins_cost(1.9 * INSN_COST);
13179 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13180
13181 ins_encode %{
13182 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13183 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13184 %}
13185 ins_pipe(ialu_reg_reg_shift);
13186 %}
13187
13188 // This pattern is automatically generated from aarch64_ad.m4
13189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13190 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13191 %{
13192 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13193 ins_cost(1.9 * INSN_COST);
13194 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13195
13196 ins_encode %{
13197 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13198 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13199 %}
13200 ins_pipe(ialu_reg_reg_shift);
13201 %}
13202
13203 // This pattern is automatically generated from aarch64_ad.m4
13204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13205 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13206 %{
13207 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13208 ins_cost(1.9 * INSN_COST);
13209 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13210
13211 ins_encode %{
13212 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13213 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13214 %}
13215 ins_pipe(ialu_reg_reg_shift);
13216 %}
13217
13218 // This pattern is automatically generated from aarch64_ad.m4
13219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13220 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13221 %{
13222 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13223 ins_cost(1.9 * INSN_COST);
13224 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13225
13226 ins_encode %{
13227 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13228 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13229 %}
13230 ins_pipe(ialu_reg_reg_shift);
13231 %}
13232
13233 // This pattern is automatically generated from aarch64_ad.m4
13234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13235 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13236 %{
13237 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13238 ins_cost(1.9 * INSN_COST);
13239 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13240
13241 ins_encode %{
13242 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13243 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13244 %}
13245 ins_pipe(ialu_reg_reg_shift);
13246 %}
13247
13248 // This pattern is automatically generated from aarch64_ad.m4
13249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13250 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13251 %{
13252 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13253 ins_cost(1.9 * INSN_COST);
13254 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13255
13256 ins_encode %{
13257 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13258 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13259 %}
13260 ins_pipe(ialu_reg_reg_shift);
13261 %}
13262
13263 // This pattern is automatically generated from aarch64_ad.m4
13264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13265 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13266 %{
13267 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13268 ins_cost(1.9 * INSN_COST);
13269 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13270
13271 ins_encode %{
13272 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13273 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13274 %}
13275 ins_pipe(ialu_reg_reg_shift);
13276 %}
13277
13278 // This pattern is automatically generated from aarch64_ad.m4
13279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13280 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13281 %{
13282 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13283 ins_cost(1.9 * INSN_COST);
13284 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13285
13286 ins_encode %{
13287 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13288 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13289 %}
13290 ins_pipe(ialu_reg_reg_shift);
13291 %}
13292
13293 // This pattern is automatically generated from aarch64_ad.m4
13294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13295 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13296 %{
13297 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13298 ins_cost(1.9 * INSN_COST);
13299 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13300
13301 ins_encode %{
13302 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13303 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13304 %}
13305 ins_pipe(ialu_reg_reg_shift);
13306 %}
13307
13308 // This pattern is automatically generated from aarch64_ad.m4
13309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13310 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13311 %{
13312 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13313 ins_cost(1.9 * INSN_COST);
13314 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13315
13316 ins_encode %{
13317 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13318 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13319 %}
13320 ins_pipe(ialu_reg_reg_shift);
13321 %}
13322
13323 // This pattern is automatically generated from aarch64_ad.m4
13324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13325 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13326 %{
13327 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13328 ins_cost(1.9 * INSN_COST);
13329 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13330
13331 ins_encode %{
13332 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13333 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13334 %}
13335 ins_pipe(ialu_reg_reg_shift);
13336 %}
13337
13338 // This pattern is automatically generated from aarch64_ad.m4
13339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13340 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13341 %{
13342 effect(DEF dst, USE src1, USE src2, USE cr);
13343 ins_cost(INSN_COST * 2);
13344 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13345
13346 ins_encode %{
13347 __ cselw($dst$$Register,
13348 $src1$$Register,
13349 $src2$$Register,
13350 Assembler::LT);
13351 %}
13352 ins_pipe(icond_reg_reg);
13353 %}
13354
13355 // This pattern is automatically generated from aarch64_ad.m4
13356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13357 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13358 %{
13359 effect(DEF dst, USE src1, USE src2, USE cr);
13360 ins_cost(INSN_COST * 2);
13361 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13362
13363 ins_encode %{
13364 __ cselw($dst$$Register,
13365 $src1$$Register,
13366 $src2$$Register,
13367 Assembler::GT);
13368 %}
13369 ins_pipe(icond_reg_reg);
13370 %}
13371
13372 // This pattern is automatically generated from aarch64_ad.m4
13373 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13374 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13375 %{
13376 effect(DEF dst, USE src1, USE cr);
13377 ins_cost(INSN_COST * 2);
13378 format %{ "cselw $dst, $src1, zr lt\t" %}
13379
13380 ins_encode %{
13381 __ cselw($dst$$Register,
13382 $src1$$Register,
13383 zr,
13384 Assembler::LT);
13385 %}
13386 ins_pipe(icond_reg);
13387 %}
13388
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13392 %{
13393 effect(DEF dst, USE src1, USE cr);
13394 ins_cost(INSN_COST * 2);
13395 format %{ "cselw $dst, $src1, zr gt\t" %}
13396
13397 ins_encode %{
13398 __ cselw($dst$$Register,
13399 $src1$$Register,
13400 zr,
13401 Assembler::GT);
13402 %}
13403 ins_pipe(icond_reg);
13404 %}
13405
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13409 %{
13410 effect(DEF dst, USE src1, USE cr);
13411 ins_cost(INSN_COST * 2);
13412 format %{ "csincw $dst, $src1, zr le\t" %}
13413
13414 ins_encode %{
13415 __ csincw($dst$$Register,
13416 $src1$$Register,
13417 zr,
13418 Assembler::LE);
13419 %}
13420 ins_pipe(icond_reg);
13421 %}
13422
13423 // This pattern is automatically generated from aarch64_ad.m4
13424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13425 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13426 %{
13427 effect(DEF dst, USE src1, USE cr);
13428 ins_cost(INSN_COST * 2);
13429 format %{ "csincw $dst, $src1, zr gt\t" %}
13430
13431 ins_encode %{
13432 __ csincw($dst$$Register,
13433 $src1$$Register,
13434 zr,
13435 Assembler::GT);
13436 %}
13437 ins_pipe(icond_reg);
13438 %}
13439
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13443 %{
13444 effect(DEF dst, USE src1, USE cr);
13445 ins_cost(INSN_COST * 2);
13446 format %{ "csinvw $dst, $src1, zr lt\t" %}
13447
13448 ins_encode %{
13449 __ csinvw($dst$$Register,
13450 $src1$$Register,
13451 zr,
13452 Assembler::LT);
13453 %}
13454 ins_pipe(icond_reg);
13455 %}
13456
13457 // This pattern is automatically generated from aarch64_ad.m4
13458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13459 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13460 %{
13461 effect(DEF dst, USE src1, USE cr);
13462 ins_cost(INSN_COST * 2);
13463 format %{ "csinvw $dst, $src1, zr ge\t" %}
13464
13465 ins_encode %{
13466 __ csinvw($dst$$Register,
13467 $src1$$Register,
13468 zr,
13469 Assembler::GE);
13470 %}
13471 ins_pipe(icond_reg);
13472 %}
13473
13474 // This pattern is automatically generated from aarch64_ad.m4
13475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13476 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13477 %{
13478 match(Set dst (MinI src imm));
13479 ins_cost(INSN_COST * 3);
13480 expand %{
13481 rFlagsReg cr;
13482 compI_reg_imm0(cr, src);
13483 cmovI_reg_imm0_lt(dst, src, cr);
13484 %}
13485 %}
13486
13487 // This pattern is automatically generated from aarch64_ad.m4
13488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13489 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13490 %{
13491 match(Set dst (MinI imm src));
13492 ins_cost(INSN_COST * 3);
13493 expand %{
13494 rFlagsReg cr;
13495 compI_reg_imm0(cr, src);
13496 cmovI_reg_imm0_lt(dst, src, cr);
13497 %}
13498 %}
13499
13500 // This pattern is automatically generated from aarch64_ad.m4
13501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13502 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13503 %{
13504 match(Set dst (MinI src imm));
13505 ins_cost(INSN_COST * 3);
13506 expand %{
13507 rFlagsReg cr;
13508 compI_reg_imm0(cr, src);
13509 cmovI_reg_imm1_le(dst, src, cr);
13510 %}
13511 %}
13512
13513 // This pattern is automatically generated from aarch64_ad.m4
13514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13515 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13516 %{
13517 match(Set dst (MinI imm src));
13518 ins_cost(INSN_COST * 3);
13519 expand %{
13520 rFlagsReg cr;
13521 compI_reg_imm0(cr, src);
13522 cmovI_reg_imm1_le(dst, src, cr);
13523 %}
13524 %}
13525
13526 // This pattern is automatically generated from aarch64_ad.m4
13527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13528 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13529 %{
13530 match(Set dst (MinI src imm));
13531 ins_cost(INSN_COST * 3);
13532 expand %{
13533 rFlagsReg cr;
13534 compI_reg_imm0(cr, src);
13535 cmovI_reg_immM1_lt(dst, src, cr);
13536 %}
13537 %}
13538
13539 // This pattern is automatically generated from aarch64_ad.m4
13540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13541 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13542 %{
13543 match(Set dst (MinI imm src));
13544 ins_cost(INSN_COST * 3);
13545 expand %{
13546 rFlagsReg cr;
13547 compI_reg_imm0(cr, src);
13548 cmovI_reg_immM1_lt(dst, src, cr);
13549 %}
13550 %}
13551
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13555 %{
13556 match(Set dst (MaxI src imm));
13557 ins_cost(INSN_COST * 3);
13558 expand %{
13559 rFlagsReg cr;
13560 compI_reg_imm0(cr, src);
13561 cmovI_reg_imm0_gt(dst, src, cr);
13562 %}
13563 %}
13564
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13568 %{
13569 match(Set dst (MaxI imm src));
13570 ins_cost(INSN_COST * 3);
13571 expand %{
13572 rFlagsReg cr;
13573 compI_reg_imm0(cr, src);
13574 cmovI_reg_imm0_gt(dst, src, cr);
13575 %}
13576 %}
13577
13578 // This pattern is automatically generated from aarch64_ad.m4
13579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13580 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13581 %{
13582 match(Set dst (MaxI src imm));
13583 ins_cost(INSN_COST * 3);
13584 expand %{
13585 rFlagsReg cr;
13586 compI_reg_imm0(cr, src);
13587 cmovI_reg_imm1_gt(dst, src, cr);
13588 %}
13589 %}
13590
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13594 %{
13595 match(Set dst (MaxI imm src));
13596 ins_cost(INSN_COST * 3);
13597 expand %{
13598 rFlagsReg cr;
13599 compI_reg_imm0(cr, src);
13600 cmovI_reg_imm1_gt(dst, src, cr);
13601 %}
13602 %}
13603
13604 // This pattern is automatically generated from aarch64_ad.m4
13605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13606 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13607 %{
13608 match(Set dst (MaxI src imm));
13609 ins_cost(INSN_COST * 3);
13610 expand %{
13611 rFlagsReg cr;
13612 compI_reg_imm0(cr, src);
13613 cmovI_reg_immM1_ge(dst, src, cr);
13614 %}
13615 %}
13616
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13620 %{
13621 match(Set dst (MaxI imm src));
13622 ins_cost(INSN_COST * 3);
13623 expand %{
13624 rFlagsReg cr;
13625 compI_reg_imm0(cr, src);
13626 cmovI_reg_immM1_ge(dst, src, cr);
13627 %}
13628 %}
13629
13630 // This pattern is automatically generated from aarch64_ad.m4
13631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13632 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13633 %{
13634 match(Set dst (ReverseI src));
13635 ins_cost(INSN_COST);
13636 format %{ "rbitw $dst, $src" %}
13637 ins_encode %{
13638 __ rbitw($dst$$Register, $src$$Register);
13639 %}
13640 ins_pipe(ialu_reg);
13641 %}
13642
13643 // This pattern is automatically generated from aarch64_ad.m4
13644 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13645 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13646 %{
13647 match(Set dst (ReverseL src));
13648 ins_cost(INSN_COST);
13649 format %{ "rbit $dst, $src" %}
13650 ins_encode %{
13651 __ rbit($dst$$Register, $src$$Register);
13652 %}
13653 ins_pipe(ialu_reg);
13654 %}
13655
13656
13657 // END This section of the file is automatically generated. Do not edit --------------
13658
13659
13660 // ============================================================================
13661 // Floating Point Arithmetic Instructions
13662
13663 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13664 match(Set dst (AddF src1 src2));
13665
13666 ins_cost(INSN_COST * 5);
13667 format %{ "fadds $dst, $src1, $src2" %}
13668
13669 ins_encode %{
13670 __ fadds(as_FloatRegister($dst$$reg),
13671 as_FloatRegister($src1$$reg),
13672 as_FloatRegister($src2$$reg));
13673 %}
13674
13675 ins_pipe(fp_dop_reg_reg_s);
13676 %}
13677
13678 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13679 match(Set dst (AddD src1 src2));
13680
13681 ins_cost(INSN_COST * 5);
13682 format %{ "faddd $dst, $src1, $src2" %}
13683
13684 ins_encode %{
13685 __ faddd(as_FloatRegister($dst$$reg),
13686 as_FloatRegister($src1$$reg),
13687 as_FloatRegister($src2$$reg));
13688 %}
13689
13690 ins_pipe(fp_dop_reg_reg_d);
13691 %}
13692
13693 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13694 match(Set dst (SubF src1 src2));
13695
13696 ins_cost(INSN_COST * 5);
13697 format %{ "fsubs $dst, $src1, $src2" %}
13698
13699 ins_encode %{
13700 __ fsubs(as_FloatRegister($dst$$reg),
13701 as_FloatRegister($src1$$reg),
13702 as_FloatRegister($src2$$reg));
13703 %}
13704
13705 ins_pipe(fp_dop_reg_reg_s);
13706 %}
13707
13708 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13709 match(Set dst (SubD src1 src2));
13710
13711 ins_cost(INSN_COST * 5);
13712 format %{ "fsubd $dst, $src1, $src2" %}
13713
13714 ins_encode %{
13715 __ fsubd(as_FloatRegister($dst$$reg),
13716 as_FloatRegister($src1$$reg),
13717 as_FloatRegister($src2$$reg));
13718 %}
13719
13720 ins_pipe(fp_dop_reg_reg_d);
13721 %}
13722
13723 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13724 match(Set dst (MulF src1 src2));
13725
13726 ins_cost(INSN_COST * 6);
13727 format %{ "fmuls $dst, $src1, $src2" %}
13728
13729 ins_encode %{
13730 __ fmuls(as_FloatRegister($dst$$reg),
13731 as_FloatRegister($src1$$reg),
13732 as_FloatRegister($src2$$reg));
13733 %}
13734
13735 ins_pipe(fp_dop_reg_reg_s);
13736 %}
13737
13738 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13739 match(Set dst (MulD src1 src2));
13740
13741 ins_cost(INSN_COST * 6);
13742 format %{ "fmuld $dst, $src1, $src2" %}
13743
13744 ins_encode %{
13745 __ fmuld(as_FloatRegister($dst$$reg),
13746 as_FloatRegister($src1$$reg),
13747 as_FloatRegister($src2$$reg));
13748 %}
13749
13750 ins_pipe(fp_dop_reg_reg_d);
13751 %}
13752
13753 // src1 * src2 + src3
13754 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13755 match(Set dst (FmaF src3 (Binary src1 src2)));
13756
13757 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13758
13759 ins_encode %{
13760 assert(UseFMA, "Needs FMA instructions support.");
13761 __ fmadds(as_FloatRegister($dst$$reg),
13762 as_FloatRegister($src1$$reg),
13763 as_FloatRegister($src2$$reg),
13764 as_FloatRegister($src3$$reg));
13765 %}
13766
13767 ins_pipe(pipe_class_default);
13768 %}
13769
13770 // src1 * src2 + src3
13771 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13772 match(Set dst (FmaD src3 (Binary src1 src2)));
13773
13774 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13775
13776 ins_encode %{
13777 assert(UseFMA, "Needs FMA instructions support.");
13778 __ fmaddd(as_FloatRegister($dst$$reg),
13779 as_FloatRegister($src1$$reg),
13780 as_FloatRegister($src2$$reg),
13781 as_FloatRegister($src3$$reg));
13782 %}
13783
13784 ins_pipe(pipe_class_default);
13785 %}
13786
13787 // src1 * (-src2) + src3
13788 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13789 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13790 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13791
13792 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13793
13794 ins_encode %{
13795 assert(UseFMA, "Needs FMA instructions support.");
13796 __ fmsubs(as_FloatRegister($dst$$reg),
13797 as_FloatRegister($src1$$reg),
13798 as_FloatRegister($src2$$reg),
13799 as_FloatRegister($src3$$reg));
13800 %}
13801
13802 ins_pipe(pipe_class_default);
13803 %}
13804
13805 // src1 * (-src2) + src3
13806 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13807 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13808 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13809
13810 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13811
13812 ins_encode %{
13813 assert(UseFMA, "Needs FMA instructions support.");
13814 __ fmsubd(as_FloatRegister($dst$$reg),
13815 as_FloatRegister($src1$$reg),
13816 as_FloatRegister($src2$$reg),
13817 as_FloatRegister($src3$$reg));
13818 %}
13819
13820 ins_pipe(pipe_class_default);
13821 %}
13822
13823 // src1 * (-src2) - src3
13824 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13825 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13826 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13827
13828 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13829
13830 ins_encode %{
13831 assert(UseFMA, "Needs FMA instructions support.");
13832 __ fnmadds(as_FloatRegister($dst$$reg),
13833 as_FloatRegister($src1$$reg),
13834 as_FloatRegister($src2$$reg),
13835 as_FloatRegister($src3$$reg));
13836 %}
13837
13838 ins_pipe(pipe_class_default);
13839 %}
13840
13841 // src1 * (-src2) - src3
13842 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13843 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13844 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13845
13846 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13847
13848 ins_encode %{
13849 assert(UseFMA, "Needs FMA instructions support.");
13850 __ fnmaddd(as_FloatRegister($dst$$reg),
13851 as_FloatRegister($src1$$reg),
13852 as_FloatRegister($src2$$reg),
13853 as_FloatRegister($src3$$reg));
13854 %}
13855
13856 ins_pipe(pipe_class_default);
13857 %}
13858
13859 // src1 * src2 - src3
13860 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13861 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13862
13863 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13864
13865 ins_encode %{
13866 assert(UseFMA, "Needs FMA instructions support.");
13867 __ fnmsubs(as_FloatRegister($dst$$reg),
13868 as_FloatRegister($src1$$reg),
13869 as_FloatRegister($src2$$reg),
13870 as_FloatRegister($src3$$reg));
13871 %}
13872
13873 ins_pipe(pipe_class_default);
13874 %}
13875
13876 // src1 * src2 - src3
13877 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13878 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13879
13880 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13881
13882 ins_encode %{
13883 assert(UseFMA, "Needs FMA instructions support.");
13884 // n.b. insn name should be fnmsubd
13885 __ fnmsub(as_FloatRegister($dst$$reg),
13886 as_FloatRegister($src1$$reg),
13887 as_FloatRegister($src2$$reg),
13888 as_FloatRegister($src3$$reg));
13889 %}
13890
13891 ins_pipe(pipe_class_default);
13892 %}
13893
13894
13895 // Math.max(FF)F
13896 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13897 match(Set dst (MaxF src1 src2));
13898
13899 format %{ "fmaxs $dst, $src1, $src2" %}
13900 ins_encode %{
13901 __ fmaxs(as_FloatRegister($dst$$reg),
13902 as_FloatRegister($src1$$reg),
13903 as_FloatRegister($src2$$reg));
13904 %}
13905
13906 ins_pipe(fp_dop_reg_reg_s);
13907 %}
13908
13909 // Math.min(FF)F
13910 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13911 match(Set dst (MinF src1 src2));
13912
13913 format %{ "fmins $dst, $src1, $src2" %}
13914 ins_encode %{
13915 __ fmins(as_FloatRegister($dst$$reg),
13916 as_FloatRegister($src1$$reg),
13917 as_FloatRegister($src2$$reg));
13918 %}
13919
13920 ins_pipe(fp_dop_reg_reg_s);
13921 %}
13922
13923 // Math.max(DD)D
13924 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13925 match(Set dst (MaxD src1 src2));
13926
13927 format %{ "fmaxd $dst, $src1, $src2" %}
13928 ins_encode %{
13929 __ fmaxd(as_FloatRegister($dst$$reg),
13930 as_FloatRegister($src1$$reg),
13931 as_FloatRegister($src2$$reg));
13932 %}
13933
13934 ins_pipe(fp_dop_reg_reg_d);
13935 %}
13936
13937 // Math.min(DD)D
13938 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13939 match(Set dst (MinD src1 src2));
13940
13941 format %{ "fmind $dst, $src1, $src2" %}
13942 ins_encode %{
13943 __ fmind(as_FloatRegister($dst$$reg),
13944 as_FloatRegister($src1$$reg),
13945 as_FloatRegister($src2$$reg));
13946 %}
13947
13948 ins_pipe(fp_dop_reg_reg_d);
13949 %}
13950
13951
13952 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13953 match(Set dst (DivF src1 src2));
13954
13955 ins_cost(INSN_COST * 18);
13956 format %{ "fdivs $dst, $src1, $src2" %}
13957
13958 ins_encode %{
13959 __ fdivs(as_FloatRegister($dst$$reg),
13960 as_FloatRegister($src1$$reg),
13961 as_FloatRegister($src2$$reg));
13962 %}
13963
13964 ins_pipe(fp_div_s);
13965 %}
13966
13967 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13968 match(Set dst (DivD src1 src2));
13969
13970 ins_cost(INSN_COST * 32);
13971 format %{ "fdivd $dst, $src1, $src2" %}
13972
13973 ins_encode %{
13974 __ fdivd(as_FloatRegister($dst$$reg),
13975 as_FloatRegister($src1$$reg),
13976 as_FloatRegister($src2$$reg));
13977 %}
13978
13979 ins_pipe(fp_div_d);
13980 %}
13981
13982 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13983 match(Set dst (NegF src));
13984
13985 ins_cost(INSN_COST * 3);
13986 format %{ "fneg $dst, $src" %}
13987
13988 ins_encode %{
13989 __ fnegs(as_FloatRegister($dst$$reg),
13990 as_FloatRegister($src$$reg));
13991 %}
13992
13993 ins_pipe(fp_uop_s);
13994 %}
13995
13996 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13997 match(Set dst (NegD src));
13998
13999 ins_cost(INSN_COST * 3);
14000 format %{ "fnegd $dst, $src" %}
14001
14002 ins_encode %{
14003 __ fnegd(as_FloatRegister($dst$$reg),
14004 as_FloatRegister($src$$reg));
14005 %}
14006
14007 ins_pipe(fp_uop_d);
14008 %}
14009
14010 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14011 %{
14012 match(Set dst (AbsI src));
14013
14014 effect(KILL cr);
14015 ins_cost(INSN_COST * 2);
14016 format %{ "cmpw $src, zr\n\t"
14017 "cnegw $dst, $src, Assembler::LT\t# int abs"
14018 %}
14019
14020 ins_encode %{
14021 __ cmpw(as_Register($src$$reg), zr);
14022 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14023 %}
14024 ins_pipe(pipe_class_default);
14025 %}
14026
14027 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14028 %{
14029 match(Set dst (AbsL src));
14030
14031 effect(KILL cr);
14032 ins_cost(INSN_COST * 2);
14033 format %{ "cmp $src, zr\n\t"
14034 "cneg $dst, $src, Assembler::LT\t# long abs"
14035 %}
14036
14037 ins_encode %{
14038 __ cmp(as_Register($src$$reg), zr);
14039 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14040 %}
14041 ins_pipe(pipe_class_default);
14042 %}
14043
14044 instruct absF_reg(vRegF dst, vRegF src) %{
14045 match(Set dst (AbsF src));
14046
14047 ins_cost(INSN_COST * 3);
14048 format %{ "fabss $dst, $src" %}
14049 ins_encode %{
14050 __ fabss(as_FloatRegister($dst$$reg),
14051 as_FloatRegister($src$$reg));
14052 %}
14053
14054 ins_pipe(fp_uop_s);
14055 %}
14056
14057 instruct absD_reg(vRegD dst, vRegD src) %{
14058 match(Set dst (AbsD src));
14059
14060 ins_cost(INSN_COST * 3);
14061 format %{ "fabsd $dst, $src" %}
14062 ins_encode %{
14063 __ fabsd(as_FloatRegister($dst$$reg),
14064 as_FloatRegister($src$$reg));
14065 %}
14066
14067 ins_pipe(fp_uop_d);
14068 %}
14069
14070 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14071 match(Set dst (AbsF (SubF src1 src2)));
14072
14073 ins_cost(INSN_COST * 3);
14074 format %{ "fabds $dst, $src1, $src2" %}
14075 ins_encode %{
14076 __ fabds(as_FloatRegister($dst$$reg),
14077 as_FloatRegister($src1$$reg),
14078 as_FloatRegister($src2$$reg));
14079 %}
14080
14081 ins_pipe(fp_uop_s);
14082 %}
14083
14084 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14085 match(Set dst (AbsD (SubD src1 src2)));
14086
14087 ins_cost(INSN_COST * 3);
14088 format %{ "fabdd $dst, $src1, $src2" %}
14089 ins_encode %{
14090 __ fabdd(as_FloatRegister($dst$$reg),
14091 as_FloatRegister($src1$$reg),
14092 as_FloatRegister($src2$$reg));
14093 %}
14094
14095 ins_pipe(fp_uop_d);
14096 %}
14097
14098 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14099 match(Set dst (SqrtD src));
14100
14101 ins_cost(INSN_COST * 50);
14102 format %{ "fsqrtd $dst, $src" %}
14103 ins_encode %{
14104 __ fsqrtd(as_FloatRegister($dst$$reg),
14105 as_FloatRegister($src$$reg));
14106 %}
14107
14108 ins_pipe(fp_div_s);
14109 %}
14110
14111 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14112 match(Set dst (SqrtF src));
14113
14114 ins_cost(INSN_COST * 50);
14115 format %{ "fsqrts $dst, $src" %}
14116 ins_encode %{
14117 __ fsqrts(as_FloatRegister($dst$$reg),
14118 as_FloatRegister($src$$reg));
14119 %}
14120
14121 ins_pipe(fp_div_d);
14122 %}
14123
14124 // Math.rint, floor, ceil
14125 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14126 match(Set dst (RoundDoubleMode src rmode));
14127 format %{ "frint $dst, $src, $rmode" %}
14128 ins_encode %{
14129 switch ($rmode$$constant) {
14130 case RoundDoubleModeNode::rmode_rint:
14131 __ frintnd(as_FloatRegister($dst$$reg),
14132 as_FloatRegister($src$$reg));
14133 break;
14134 case RoundDoubleModeNode::rmode_floor:
14135 __ frintmd(as_FloatRegister($dst$$reg),
14136 as_FloatRegister($src$$reg));
14137 break;
14138 case RoundDoubleModeNode::rmode_ceil:
14139 __ frintpd(as_FloatRegister($dst$$reg),
14140 as_FloatRegister($src$$reg));
14141 break;
14142 }
14143 %}
14144 ins_pipe(fp_uop_d);
14145 %}
14146
14147 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14148 match(Set dst (CopySignD src1 (Binary src2 zero)));
14149 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14150 format %{ "CopySignD $dst $src1 $src2" %}
14151 ins_encode %{
14152 FloatRegister dst = as_FloatRegister($dst$$reg),
14153 src1 = as_FloatRegister($src1$$reg),
14154 src2 = as_FloatRegister($src2$$reg),
14155 zero = as_FloatRegister($zero$$reg);
14156 __ fnegd(dst, zero);
14157 __ bsl(dst, __ T8B, src2, src1);
14158 %}
14159 ins_pipe(fp_uop_d);
14160 %}
14161
14162 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14163 match(Set dst (CopySignF src1 src2));
14164 effect(TEMP_DEF dst, USE src1, USE src2);
14165 format %{ "CopySignF $dst $src1 $src2" %}
14166 ins_encode %{
14167 FloatRegister dst = as_FloatRegister($dst$$reg),
14168 src1 = as_FloatRegister($src1$$reg),
14169 src2 = as_FloatRegister($src2$$reg);
14170 __ movi(dst, __ T2S, 0x80, 24);
14171 __ bsl(dst, __ T8B, src2, src1);
14172 %}
14173 ins_pipe(fp_uop_d);
14174 %}
14175
14176 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14177 match(Set dst (SignumD src (Binary zero one)));
14178 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14179 format %{ "signumD $dst, $src" %}
14180 ins_encode %{
14181 FloatRegister src = as_FloatRegister($src$$reg),
14182 dst = as_FloatRegister($dst$$reg),
14183 zero = as_FloatRegister($zero$$reg),
14184 one = as_FloatRegister($one$$reg);
14185 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14186 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14187 // Bit selection instruction gets bit from "one" for each enabled bit in
14188 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14189 // NaN the whole "src" will be copied because "dst" is zero. For all other
14190 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14191 // from "src", and all other bits are copied from 1.0.
14192 __ bsl(dst, __ T8B, one, src);
14193 %}
14194 ins_pipe(fp_uop_d);
14195 %}
14196
14197 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14198 match(Set dst (SignumF src (Binary zero one)));
14199 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14200 format %{ "signumF $dst, $src" %}
14201 ins_encode %{
14202 FloatRegister src = as_FloatRegister($src$$reg),
14203 dst = as_FloatRegister($dst$$reg),
14204 zero = as_FloatRegister($zero$$reg),
14205 one = as_FloatRegister($one$$reg);
14206 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14207 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14208 // Bit selection instruction gets bit from "one" for each enabled bit in
14209 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14210 // NaN the whole "src" will be copied because "dst" is zero. For all other
14211 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14212 // from "src", and all other bits are copied from 1.0.
14213 __ bsl(dst, __ T8B, one, src);
14214 %}
14215 ins_pipe(fp_uop_d);
14216 %}
14217
14218 instruct onspinwait() %{
14219 match(OnSpinWait);
14220 ins_cost(INSN_COST);
14221
14222 format %{ "onspinwait" %}
14223
14224 ins_encode %{
14225 __ spin_wait();
14226 %}
14227 ins_pipe(pipe_class_empty);
14228 %}
14229
14230 // ============================================================================
14231 // Logical Instructions
14232
14233 // Integer Logical Instructions
14234
14235 // And Instructions
14236
14237
14238 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14239 match(Set dst (AndI src1 src2));
14240
14241 format %{ "andw $dst, $src1, $src2\t# int" %}
14242
14243 ins_cost(INSN_COST);
14244 ins_encode %{
14245 __ andw(as_Register($dst$$reg),
14246 as_Register($src1$$reg),
14247 as_Register($src2$$reg));
14248 %}
14249
14250 ins_pipe(ialu_reg_reg);
14251 %}
14252
14253 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14254 match(Set dst (AndI src1 src2));
14255
14256 format %{ "andsw $dst, $src1, $src2\t# int" %}
14257
14258 ins_cost(INSN_COST);
14259 ins_encode %{
14260 __ andw(as_Register($dst$$reg),
14261 as_Register($src1$$reg),
14262 (uint64_t)($src2$$constant));
14263 %}
14264
14265 ins_pipe(ialu_reg_imm);
14266 %}
14267
14268 // Or Instructions
14269
14270 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14271 match(Set dst (OrI src1 src2));
14272
14273 format %{ "orrw $dst, $src1, $src2\t# int" %}
14274
14275 ins_cost(INSN_COST);
14276 ins_encode %{
14277 __ orrw(as_Register($dst$$reg),
14278 as_Register($src1$$reg),
14279 as_Register($src2$$reg));
14280 %}
14281
14282 ins_pipe(ialu_reg_reg);
14283 %}
14284
14285 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14286 match(Set dst (OrI src1 src2));
14287
14288 format %{ "orrw $dst, $src1, $src2\t# int" %}
14289
14290 ins_cost(INSN_COST);
14291 ins_encode %{
14292 __ orrw(as_Register($dst$$reg),
14293 as_Register($src1$$reg),
14294 (uint64_t)($src2$$constant));
14295 %}
14296
14297 ins_pipe(ialu_reg_imm);
14298 %}
14299
14300 // Xor Instructions
14301
14302 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14303 match(Set dst (XorI src1 src2));
14304
14305 format %{ "eorw $dst, $src1, $src2\t# int" %}
14306
14307 ins_cost(INSN_COST);
14308 ins_encode %{
14309 __ eorw(as_Register($dst$$reg),
14310 as_Register($src1$$reg),
14311 as_Register($src2$$reg));
14312 %}
14313
14314 ins_pipe(ialu_reg_reg);
14315 %}
14316
14317 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14318 match(Set dst (XorI src1 src2));
14319
14320 format %{ "eorw $dst, $src1, $src2\t# int" %}
14321
14322 ins_cost(INSN_COST);
14323 ins_encode %{
14324 __ eorw(as_Register($dst$$reg),
14325 as_Register($src1$$reg),
14326 (uint64_t)($src2$$constant));
14327 %}
14328
14329 ins_pipe(ialu_reg_imm);
14330 %}
14331
14332 // Long Logical Instructions
14333 // TODO
14334
14335 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14336 match(Set dst (AndL src1 src2));
14337
14338 format %{ "and $dst, $src1, $src2\t# int" %}
14339
14340 ins_cost(INSN_COST);
14341 ins_encode %{
14342 __ andr(as_Register($dst$$reg),
14343 as_Register($src1$$reg),
14344 as_Register($src2$$reg));
14345 %}
14346
14347 ins_pipe(ialu_reg_reg);
14348 %}
14349
14350 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14351 match(Set dst (AndL src1 src2));
14352
14353 format %{ "and $dst, $src1, $src2\t# int" %}
14354
14355 ins_cost(INSN_COST);
14356 ins_encode %{
14357 __ andr(as_Register($dst$$reg),
14358 as_Register($src1$$reg),
14359 (uint64_t)($src2$$constant));
14360 %}
14361
14362 ins_pipe(ialu_reg_imm);
14363 %}
14364
14365 // Or Instructions
14366
14367 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14368 match(Set dst (OrL src1 src2));
14369
14370 format %{ "orr $dst, $src1, $src2\t# int" %}
14371
14372 ins_cost(INSN_COST);
14373 ins_encode %{
14374 __ orr(as_Register($dst$$reg),
14375 as_Register($src1$$reg),
14376 as_Register($src2$$reg));
14377 %}
14378
14379 ins_pipe(ialu_reg_reg);
14380 %}
14381
14382 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14383 match(Set dst (OrL src1 src2));
14384
14385 format %{ "orr $dst, $src1, $src2\t# int" %}
14386
14387 ins_cost(INSN_COST);
14388 ins_encode %{
14389 __ orr(as_Register($dst$$reg),
14390 as_Register($src1$$reg),
14391 (uint64_t)($src2$$constant));
14392 %}
14393
14394 ins_pipe(ialu_reg_imm);
14395 %}
14396
14397 // Xor Instructions
14398
14399 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14400 match(Set dst (XorL src1 src2));
14401
14402 format %{ "eor $dst, $src1, $src2\t# int" %}
14403
14404 ins_cost(INSN_COST);
14405 ins_encode %{
14406 __ eor(as_Register($dst$$reg),
14407 as_Register($src1$$reg),
14408 as_Register($src2$$reg));
14409 %}
14410
14411 ins_pipe(ialu_reg_reg);
14412 %}
14413
14414 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14415 match(Set dst (XorL src1 src2));
14416
14417 ins_cost(INSN_COST);
14418 format %{ "eor $dst, $src1, $src2\t# int" %}
14419
14420 ins_encode %{
14421 __ eor(as_Register($dst$$reg),
14422 as_Register($src1$$reg),
14423 (uint64_t)($src2$$constant));
14424 %}
14425
14426 ins_pipe(ialu_reg_imm);
14427 %}
14428
14429 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14430 %{
14431 match(Set dst (ConvI2L src));
14432
14433 ins_cost(INSN_COST);
14434 format %{ "sxtw $dst, $src\t# i2l" %}
14435 ins_encode %{
14436 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14437 %}
14438 ins_pipe(ialu_reg_shift);
14439 %}
14440
14441 // this pattern occurs in bigmath arithmetic
14442 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14443 %{
14444 match(Set dst (AndL (ConvI2L src) mask));
14445
14446 ins_cost(INSN_COST);
14447 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14448 ins_encode %{
14449 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14450 %}
14451
14452 ins_pipe(ialu_reg_shift);
14453 %}
14454
14455 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14456 match(Set dst (ConvL2I src));
14457
14458 ins_cost(INSN_COST);
14459 format %{ "movw $dst, $src \t// l2i" %}
14460
14461 ins_encode %{
14462 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14463 %}
14464
14465 ins_pipe(ialu_reg);
14466 %}
14467
14468 instruct convD2F_reg(vRegF dst, vRegD src) %{
14469 match(Set dst (ConvD2F src));
14470
14471 ins_cost(INSN_COST * 5);
14472 format %{ "fcvtd $dst, $src \t// d2f" %}
14473
14474 ins_encode %{
14475 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14476 %}
14477
14478 ins_pipe(fp_d2f);
14479 %}
14480
14481 instruct convF2D_reg(vRegD dst, vRegF src) %{
14482 match(Set dst (ConvF2D src));
14483
14484 ins_cost(INSN_COST * 5);
14485 format %{ "fcvts $dst, $src \t// f2d" %}
14486
14487 ins_encode %{
14488 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14489 %}
14490
14491 ins_pipe(fp_f2d);
14492 %}
14493
14494 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14495 match(Set dst (ConvF2I src));
14496
14497 ins_cost(INSN_COST * 5);
14498 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14499
14500 ins_encode %{
14501 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14502 %}
14503
14504 ins_pipe(fp_f2i);
14505 %}
14506
14507 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14508 match(Set dst (ConvF2L src));
14509
14510 ins_cost(INSN_COST * 5);
14511 format %{ "fcvtzs $dst, $src \t// f2l" %}
14512
14513 ins_encode %{
14514 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14515 %}
14516
14517 ins_pipe(fp_f2l);
14518 %}
14519
14520 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14521 match(Set dst (ConvF2HF src));
14522 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14523 "smov $dst, $tmp\t# move result from $tmp to $dst"
14524 %}
14525 effect(TEMP tmp);
14526 ins_encode %{
14527 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14528 %}
14529 ins_pipe(pipe_slow);
14530 %}
14531
14532 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14533 match(Set dst (ConvHF2F src));
14534 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14535 "fcvt $dst, $tmp\t# convert half to single precision"
14536 %}
14537 effect(TEMP tmp);
14538 ins_encode %{
14539 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14540 %}
14541 ins_pipe(pipe_slow);
14542 %}
14543
14544 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14545 match(Set dst (ConvI2F src));
14546
14547 ins_cost(INSN_COST * 5);
14548 format %{ "scvtfws $dst, $src \t// i2f" %}
14549
14550 ins_encode %{
14551 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14552 %}
14553
14554 ins_pipe(fp_i2f);
14555 %}
14556
14557 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14558 match(Set dst (ConvL2F src));
14559
14560 ins_cost(INSN_COST * 5);
14561 format %{ "scvtfs $dst, $src \t// l2f" %}
14562
14563 ins_encode %{
14564 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14565 %}
14566
14567 ins_pipe(fp_l2f);
14568 %}
14569
14570 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14571 match(Set dst (ConvD2I src));
14572
14573 ins_cost(INSN_COST * 5);
14574 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14575
14576 ins_encode %{
14577 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14578 %}
14579
14580 ins_pipe(fp_d2i);
14581 %}
14582
14583 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14584 match(Set dst (ConvD2L src));
14585
14586 ins_cost(INSN_COST * 5);
14587 format %{ "fcvtzd $dst, $src \t// d2l" %}
14588
14589 ins_encode %{
14590 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14591 %}
14592
14593 ins_pipe(fp_d2l);
14594 %}
14595
14596 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14597 match(Set dst (ConvI2D src));
14598
14599 ins_cost(INSN_COST * 5);
14600 format %{ "scvtfwd $dst, $src \t// i2d" %}
14601
14602 ins_encode %{
14603 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14604 %}
14605
14606 ins_pipe(fp_i2d);
14607 %}
14608
14609 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14610 match(Set dst (ConvL2D src));
14611
14612 ins_cost(INSN_COST * 5);
14613 format %{ "scvtfd $dst, $src \t// l2d" %}
14614
14615 ins_encode %{
14616 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14617 %}
14618
14619 ins_pipe(fp_l2d);
14620 %}
14621
14622 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14623 %{
14624 match(Set dst (RoundD src));
14625 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14626 format %{ "java_round_double $dst,$src"%}
14627 ins_encode %{
14628 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14629 as_FloatRegister($ftmp$$reg));
14630 %}
14631 ins_pipe(pipe_slow);
14632 %}
14633
14634 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14635 %{
14636 match(Set dst (RoundF src));
14637 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14638 format %{ "java_round_float $dst,$src"%}
14639 ins_encode %{
14640 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14641 as_FloatRegister($ftmp$$reg));
14642 %}
14643 ins_pipe(pipe_slow);
14644 %}
14645
14646 // stack <-> reg and reg <-> reg shuffles with no conversion
14647
14648 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14649
14650 match(Set dst (MoveF2I src));
14651
14652 effect(DEF dst, USE src);
14653
14654 ins_cost(4 * INSN_COST);
14655
14656 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14657
14658 ins_encode %{
14659 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14660 %}
14661
14662 ins_pipe(iload_reg_reg);
14663
14664 %}
14665
14666 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14667
14668 match(Set dst (MoveI2F src));
14669
14670 effect(DEF dst, USE src);
14671
14672 ins_cost(4 * INSN_COST);
14673
14674 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14675
14676 ins_encode %{
14677 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14678 %}
14679
14680 ins_pipe(pipe_class_memory);
14681
14682 %}
14683
14684 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14685
14686 match(Set dst (MoveD2L src));
14687
14688 effect(DEF dst, USE src);
14689
14690 ins_cost(4 * INSN_COST);
14691
14692 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14693
14694 ins_encode %{
14695 __ ldr($dst$$Register, Address(sp, $src$$disp));
14696 %}
14697
14698 ins_pipe(iload_reg_reg);
14699
14700 %}
14701
14702 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14703
14704 match(Set dst (MoveL2D src));
14705
14706 effect(DEF dst, USE src);
14707
14708 ins_cost(4 * INSN_COST);
14709
14710 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14711
14712 ins_encode %{
14713 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14714 %}
14715
14716 ins_pipe(pipe_class_memory);
14717
14718 %}
14719
14720 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14721
14722 match(Set dst (MoveF2I src));
14723
14724 effect(DEF dst, USE src);
14725
14726 ins_cost(INSN_COST);
14727
14728 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14729
14730 ins_encode %{
14731 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14732 %}
14733
14734 ins_pipe(pipe_class_memory);
14735
14736 %}
14737
14738 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14739
14740 match(Set dst (MoveI2F src));
14741
14742 effect(DEF dst, USE src);
14743
14744 ins_cost(INSN_COST);
14745
14746 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14747
14748 ins_encode %{
14749 __ strw($src$$Register, Address(sp, $dst$$disp));
14750 %}
14751
14752 ins_pipe(istore_reg_reg);
14753
14754 %}
14755
14756 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14757
14758 match(Set dst (MoveD2L src));
14759
14760 effect(DEF dst, USE src);
14761
14762 ins_cost(INSN_COST);
14763
14764 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14765
14766 ins_encode %{
14767 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14768 %}
14769
14770 ins_pipe(pipe_class_memory);
14771
14772 %}
14773
14774 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14775
14776 match(Set dst (MoveL2D src));
14777
14778 effect(DEF dst, USE src);
14779
14780 ins_cost(INSN_COST);
14781
14782 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14783
14784 ins_encode %{
14785 __ str($src$$Register, Address(sp, $dst$$disp));
14786 %}
14787
14788 ins_pipe(istore_reg_reg);
14789
14790 %}
14791
14792 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14793
14794 match(Set dst (MoveF2I src));
14795
14796 effect(DEF dst, USE src);
14797
14798 ins_cost(INSN_COST);
14799
14800 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14801
14802 ins_encode %{
14803 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14804 %}
14805
14806 ins_pipe(fp_f2i);
14807
14808 %}
14809
14810 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14811
14812 match(Set dst (MoveI2F src));
14813
14814 effect(DEF dst, USE src);
14815
14816 ins_cost(INSN_COST);
14817
14818 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14819
14820 ins_encode %{
14821 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14822 %}
14823
14824 ins_pipe(fp_i2f);
14825
14826 %}
14827
14828 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14829
14830 match(Set dst (MoveD2L src));
14831
14832 effect(DEF dst, USE src);
14833
14834 ins_cost(INSN_COST);
14835
14836 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14837
14838 ins_encode %{
14839 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14840 %}
14841
14842 ins_pipe(fp_d2l);
14843
14844 %}
14845
14846 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14847
14848 match(Set dst (MoveL2D src));
14849
14850 effect(DEF dst, USE src);
14851
14852 ins_cost(INSN_COST);
14853
14854 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14855
14856 ins_encode %{
14857 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14858 %}
14859
14860 ins_pipe(fp_l2d);
14861
14862 %}
14863
14864 // ============================================================================
14865 // clearing of an array
14866
14867 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14868 %{
14869 match(Set dummy (ClearArray (Binary cnt base) zero));
14870 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14871
14872 ins_cost(4 * INSN_COST);
14873 format %{ "ClearArray $cnt, $base" %}
14874
14875 ins_encode %{
14876 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14877 if (tpc == nullptr) {
14878 ciEnv::current()->record_failure("CodeCache is full");
14879 return;
14880 }
14881 %}
14882
14883 ins_pipe(pipe_class_memory);
14884 %}
14885
14886 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14887 %{
14888 predicate(((ClearArrayNode*)n)->word_copy_only());
14889 match(Set dummy (ClearArray (Binary cnt base) val));
14890 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14891
14892 ins_cost(4 * INSN_COST);
14893 format %{ "ClearArray $cnt, $base, $val" %}
14894
14895 ins_encode %{
14896 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14897 %}
14898
14899 ins_pipe(pipe_class_memory);
14900 %}
14901
14902 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14903 %{
14904 predicate((uint64_t)n->in(2)->get_long()
14905 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14906 && !((ClearArrayNode*)n)->word_copy_only());
14907 match(Set dummy (ClearArray cnt base));
14908 effect(TEMP temp, USE_KILL base, KILL cr);
14909
14910 ins_cost(4 * INSN_COST);
14911 format %{ "ClearArray $cnt, $base" %}
14912
14913 ins_encode %{
14914 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14915 if (tpc == nullptr) {
14916 ciEnv::current()->record_failure("CodeCache is full");
14917 return;
14918 }
14919 %}
14920
14921 ins_pipe(pipe_class_memory);
14922 %}
14923
14924 // ============================================================================
14925 // Overflow Math Instructions
14926
14927 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14928 %{
14929 match(Set cr (OverflowAddI op1 op2));
14930
14931 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14932 ins_cost(INSN_COST);
14933 ins_encode %{
14934 __ cmnw($op1$$Register, $op2$$Register);
14935 %}
14936
14937 ins_pipe(icmp_reg_reg);
14938 %}
14939
14940 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14941 %{
14942 match(Set cr (OverflowAddI op1 op2));
14943
14944 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14945 ins_cost(INSN_COST);
14946 ins_encode %{
14947 __ cmnw($op1$$Register, $op2$$constant);
14948 %}
14949
14950 ins_pipe(icmp_reg_imm);
14951 %}
14952
14953 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14954 %{
14955 match(Set cr (OverflowAddL op1 op2));
14956
14957 format %{ "cmn $op1, $op2\t# overflow check long" %}
14958 ins_cost(INSN_COST);
14959 ins_encode %{
14960 __ cmn($op1$$Register, $op2$$Register);
14961 %}
14962
14963 ins_pipe(icmp_reg_reg);
14964 %}
14965
14966 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14967 %{
14968 match(Set cr (OverflowAddL op1 op2));
14969
14970 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14971 ins_cost(INSN_COST);
14972 ins_encode %{
14973 __ adds(zr, $op1$$Register, $op2$$constant);
14974 %}
14975
14976 ins_pipe(icmp_reg_imm);
14977 %}
14978
14979 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14980 %{
14981 match(Set cr (OverflowSubI op1 op2));
14982
14983 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14984 ins_cost(INSN_COST);
14985 ins_encode %{
14986 __ cmpw($op1$$Register, $op2$$Register);
14987 %}
14988
14989 ins_pipe(icmp_reg_reg);
14990 %}
14991
14992 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14993 %{
14994 match(Set cr (OverflowSubI op1 op2));
14995
14996 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14997 ins_cost(INSN_COST);
14998 ins_encode %{
14999 __ cmpw($op1$$Register, $op2$$constant);
15000 %}
15001
15002 ins_pipe(icmp_reg_imm);
15003 %}
15004
15005 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15006 %{
15007 match(Set cr (OverflowSubL op1 op2));
15008
15009 format %{ "cmp $op1, $op2\t# overflow check long" %}
15010 ins_cost(INSN_COST);
15011 ins_encode %{
15012 __ cmp($op1$$Register, $op2$$Register);
15013 %}
15014
15015 ins_pipe(icmp_reg_reg);
15016 %}
15017
15018 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15019 %{
15020 match(Set cr (OverflowSubL op1 op2));
15021
15022 format %{ "cmp $op1, $op2\t# overflow check long" %}
15023 ins_cost(INSN_COST);
15024 ins_encode %{
15025 __ subs(zr, $op1$$Register, $op2$$constant);
15026 %}
15027
15028 ins_pipe(icmp_reg_imm);
15029 %}
15030
15031 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15032 %{
15033 match(Set cr (OverflowSubI zero op1));
15034
15035 format %{ "cmpw zr, $op1\t# overflow check int" %}
15036 ins_cost(INSN_COST);
15037 ins_encode %{
15038 __ cmpw(zr, $op1$$Register);
15039 %}
15040
15041 ins_pipe(icmp_reg_imm);
15042 %}
15043
15044 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15045 %{
15046 match(Set cr (OverflowSubL zero op1));
15047
15048 format %{ "cmp zr, $op1\t# overflow check long" %}
15049 ins_cost(INSN_COST);
15050 ins_encode %{
15051 __ cmp(zr, $op1$$Register);
15052 %}
15053
15054 ins_pipe(icmp_reg_imm);
15055 %}
15056
15057 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15058 %{
15059 match(Set cr (OverflowMulI op1 op2));
15060
15061 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15062 "cmp rscratch1, rscratch1, sxtw\n\t"
15063 "movw rscratch1, #0x80000000\n\t"
15064 "cselw rscratch1, rscratch1, zr, NE\n\t"
15065 "cmpw rscratch1, #1" %}
15066 ins_cost(5 * INSN_COST);
15067 ins_encode %{
15068 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15069 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15070 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15071 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15072 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15073 %}
15074
15075 ins_pipe(pipe_slow);
15076 %}
15077
15078 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15079 %{
15080 match(If cmp (OverflowMulI op1 op2));
15081 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15082 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15083 effect(USE labl, KILL cr);
15084
15085 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15086 "cmp rscratch1, rscratch1, sxtw\n\t"
15087 "b$cmp $labl" %}
15088 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15089 ins_encode %{
15090 Label* L = $labl$$label;
15091 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15092 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15093 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15094 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15095 %}
15096
15097 ins_pipe(pipe_serial);
15098 %}
15099
15100 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15101 %{
15102 match(Set cr (OverflowMulL op1 op2));
15103
15104 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15105 "smulh rscratch2, $op1, $op2\n\t"
15106 "cmp rscratch2, rscratch1, ASR #63\n\t"
15107 "movw rscratch1, #0x80000000\n\t"
15108 "cselw rscratch1, rscratch1, zr, NE\n\t"
15109 "cmpw rscratch1, #1" %}
15110 ins_cost(6 * INSN_COST);
15111 ins_encode %{
15112 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15113 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15114 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15115 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15116 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15117 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15118 %}
15119
15120 ins_pipe(pipe_slow);
15121 %}
15122
15123 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15124 %{
15125 match(If cmp (OverflowMulL op1 op2));
15126 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15127 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15128 effect(USE labl, KILL cr);
15129
15130 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15131 "smulh rscratch2, $op1, $op2\n\t"
15132 "cmp rscratch2, rscratch1, ASR #63\n\t"
15133 "b$cmp $labl" %}
15134 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15135 ins_encode %{
15136 Label* L = $labl$$label;
15137 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15138 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15139 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15140 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15141 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15142 %}
15143
15144 ins_pipe(pipe_serial);
15145 %}
15146
15147 // ============================================================================
15148 // Compare Instructions
15149
15150 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15151 %{
15152 match(Set cr (CmpI op1 op2));
15153
15154 effect(DEF cr, USE op1, USE op2);
15155
15156 ins_cost(INSN_COST);
15157 format %{ "cmpw $op1, $op2" %}
15158
15159 ins_encode(aarch64_enc_cmpw(op1, op2));
15160
15161 ins_pipe(icmp_reg_reg);
15162 %}
15163
15164 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15165 %{
15166 match(Set cr (CmpI op1 zero));
15167
15168 effect(DEF cr, USE op1);
15169
15170 ins_cost(INSN_COST);
15171 format %{ "cmpw $op1, 0" %}
15172
15173 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15174
15175 ins_pipe(icmp_reg_imm);
15176 %}
15177
15178 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15179 %{
15180 match(Set cr (CmpI op1 op2));
15181
15182 effect(DEF cr, USE op1);
15183
15184 ins_cost(INSN_COST);
15185 format %{ "cmpw $op1, $op2" %}
15186
15187 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15188
15189 ins_pipe(icmp_reg_imm);
15190 %}
15191
15192 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15193 %{
15194 match(Set cr (CmpI op1 op2));
15195
15196 effect(DEF cr, USE op1);
15197
15198 ins_cost(INSN_COST * 2);
15199 format %{ "cmpw $op1, $op2" %}
15200
15201 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15202
15203 ins_pipe(icmp_reg_imm);
15204 %}
15205
15206 // Unsigned compare Instructions; really, same as signed compare
15207 // except it should only be used to feed an If or a CMovI which takes a
15208 // cmpOpU.
15209
15210 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15211 %{
15212 match(Set cr (CmpU op1 op2));
15213
15214 effect(DEF cr, USE op1, USE op2);
15215
15216 ins_cost(INSN_COST);
15217 format %{ "cmpw $op1, $op2\t# unsigned" %}
15218
15219 ins_encode(aarch64_enc_cmpw(op1, op2));
15220
15221 ins_pipe(icmp_reg_reg);
15222 %}
15223
15224 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15225 %{
15226 match(Set cr (CmpU op1 zero));
15227
15228 effect(DEF cr, USE op1);
15229
15230 ins_cost(INSN_COST);
15231 format %{ "cmpw $op1, #0\t# unsigned" %}
15232
15233 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15234
15235 ins_pipe(icmp_reg_imm);
15236 %}
15237
15238 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15239 %{
15240 match(Set cr (CmpU op1 op2));
15241
15242 effect(DEF cr, USE op1);
15243
15244 ins_cost(INSN_COST);
15245 format %{ "cmpw $op1, $op2\t# unsigned" %}
15246
15247 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15248
15249 ins_pipe(icmp_reg_imm);
15250 %}
15251
15252 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15253 %{
15254 match(Set cr (CmpU op1 op2));
15255
15256 effect(DEF cr, USE op1);
15257
15258 ins_cost(INSN_COST * 2);
15259 format %{ "cmpw $op1, $op2\t# unsigned" %}
15260
15261 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15262
15263 ins_pipe(icmp_reg_imm);
15264 %}
15265
15266 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15267 %{
15268 match(Set cr (CmpL op1 op2));
15269
15270 effect(DEF cr, USE op1, USE op2);
15271
15272 ins_cost(INSN_COST);
15273 format %{ "cmp $op1, $op2" %}
15274
15275 ins_encode(aarch64_enc_cmp(op1, op2));
15276
15277 ins_pipe(icmp_reg_reg);
15278 %}
15279
15280 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15281 %{
15282 match(Set cr (CmpL op1 zero));
15283
15284 effect(DEF cr, USE op1);
15285
15286 ins_cost(INSN_COST);
15287 format %{ "tst $op1" %}
15288
15289 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15290
15291 ins_pipe(icmp_reg_imm);
15292 %}
15293
15294 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15295 %{
15296 match(Set cr (CmpL op1 op2));
15297
15298 effect(DEF cr, USE op1);
15299
15300 ins_cost(INSN_COST);
15301 format %{ "cmp $op1, $op2" %}
15302
15303 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15304
15305 ins_pipe(icmp_reg_imm);
15306 %}
15307
15308 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15309 %{
15310 match(Set cr (CmpL op1 op2));
15311
15312 effect(DEF cr, USE op1);
15313
15314 ins_cost(INSN_COST * 2);
15315 format %{ "cmp $op1, $op2" %}
15316
15317 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15318
15319 ins_pipe(icmp_reg_imm);
15320 %}
15321
15322 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15323 %{
15324 match(Set cr (CmpUL op1 op2));
15325
15326 effect(DEF cr, USE op1, USE op2);
15327
15328 ins_cost(INSN_COST);
15329 format %{ "cmp $op1, $op2" %}
15330
15331 ins_encode(aarch64_enc_cmp(op1, op2));
15332
15333 ins_pipe(icmp_reg_reg);
15334 %}
15335
15336 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15337 %{
15338 match(Set cr (CmpUL op1 zero));
15339
15340 effect(DEF cr, USE op1);
15341
15342 ins_cost(INSN_COST);
15343 format %{ "tst $op1" %}
15344
15345 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15346
15347 ins_pipe(icmp_reg_imm);
15348 %}
15349
15350 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15351 %{
15352 match(Set cr (CmpUL op1 op2));
15353
15354 effect(DEF cr, USE op1);
15355
15356 ins_cost(INSN_COST);
15357 format %{ "cmp $op1, $op2" %}
15358
15359 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15360
15361 ins_pipe(icmp_reg_imm);
15362 %}
15363
15364 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15365 %{
15366 match(Set cr (CmpUL op1 op2));
15367
15368 effect(DEF cr, USE op1);
15369
15370 ins_cost(INSN_COST * 2);
15371 format %{ "cmp $op1, $op2" %}
15372
15373 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15374
15375 ins_pipe(icmp_reg_imm);
15376 %}
15377
15378 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15379 %{
15380 match(Set cr (CmpP op1 op2));
15381
15382 effect(DEF cr, USE op1, USE op2);
15383
15384 ins_cost(INSN_COST);
15385 format %{ "cmp $op1, $op2\t // ptr" %}
15386
15387 ins_encode(aarch64_enc_cmpp(op1, op2));
15388
15389 ins_pipe(icmp_reg_reg);
15390 %}
15391
15392 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15393 %{
15394 match(Set cr (CmpN op1 op2));
15395
15396 effect(DEF cr, USE op1, USE op2);
15397
15398 ins_cost(INSN_COST);
15399 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15400
15401 ins_encode(aarch64_enc_cmpn(op1, op2));
15402
15403 ins_pipe(icmp_reg_reg);
15404 %}
15405
15406 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15407 %{
15408 match(Set cr (CmpP op1 zero));
15409
15410 effect(DEF cr, USE op1, USE zero);
15411
15412 ins_cost(INSN_COST);
15413 format %{ "cmp $op1, 0\t // ptr" %}
15414
15415 ins_encode(aarch64_enc_testp(op1));
15416
15417 ins_pipe(icmp_reg_imm);
15418 %}
15419
15420 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15421 %{
15422 match(Set cr (CmpN op1 zero));
15423
15424 effect(DEF cr, USE op1, USE zero);
15425
15426 ins_cost(INSN_COST);
15427 format %{ "cmp $op1, 0\t // compressed ptr" %}
15428
15429 ins_encode(aarch64_enc_testn(op1));
15430
15431 ins_pipe(icmp_reg_imm);
15432 %}
15433
15434 // FP comparisons
15435 //
15436 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15437 // using normal cmpOp. See declaration of rFlagsReg for details.
15438
15439 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15440 %{
15441 match(Set cr (CmpF src1 src2));
15442
15443 ins_cost(3 * INSN_COST);
15444 format %{ "fcmps $src1, $src2" %}
15445
15446 ins_encode %{
15447 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15448 %}
15449
15450 ins_pipe(pipe_class_compare);
15451 %}
15452
15453 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15454 %{
15455 match(Set cr (CmpF src1 src2));
15456
15457 ins_cost(3 * INSN_COST);
15458 format %{ "fcmps $src1, 0.0" %}
15459
15460 ins_encode %{
15461 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15462 %}
15463
15464 ins_pipe(pipe_class_compare);
15465 %}
15466 // FROM HERE
15467
15468 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15469 %{
15470 match(Set cr (CmpD src1 src2));
15471
15472 ins_cost(3 * INSN_COST);
15473 format %{ "fcmpd $src1, $src2" %}
15474
15475 ins_encode %{
15476 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15477 %}
15478
15479 ins_pipe(pipe_class_compare);
15480 %}
15481
15482 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15483 %{
15484 match(Set cr (CmpD src1 src2));
15485
15486 ins_cost(3 * INSN_COST);
15487 format %{ "fcmpd $src1, 0.0" %}
15488
15489 ins_encode %{
15490 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15491 %}
15492
15493 ins_pipe(pipe_class_compare);
15494 %}
15495
15496 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15497 %{
15498 match(Set dst (CmpF3 src1 src2));
15499 effect(KILL cr);
15500
15501 ins_cost(5 * INSN_COST);
15502 format %{ "fcmps $src1, $src2\n\t"
15503 "csinvw($dst, zr, zr, eq\n\t"
15504 "csnegw($dst, $dst, $dst, lt)"
15505 %}
15506
15507 ins_encode %{
15508 Label done;
15509 FloatRegister s1 = as_FloatRegister($src1$$reg);
15510 FloatRegister s2 = as_FloatRegister($src2$$reg);
15511 Register d = as_Register($dst$$reg);
15512 __ fcmps(s1, s2);
15513 // installs 0 if EQ else -1
15514 __ csinvw(d, zr, zr, Assembler::EQ);
15515 // keeps -1 if less or unordered else installs 1
15516 __ csnegw(d, d, d, Assembler::LT);
15517 __ bind(done);
15518 %}
15519
15520 ins_pipe(pipe_class_default);
15521
15522 %}
15523
15524 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15525 %{
15526 match(Set dst (CmpD3 src1 src2));
15527 effect(KILL cr);
15528
15529 ins_cost(5 * INSN_COST);
15530 format %{ "fcmpd $src1, $src2\n\t"
15531 "csinvw($dst, zr, zr, eq\n\t"
15532 "csnegw($dst, $dst, $dst, lt)"
15533 %}
15534
15535 ins_encode %{
15536 Label done;
15537 FloatRegister s1 = as_FloatRegister($src1$$reg);
15538 FloatRegister s2 = as_FloatRegister($src2$$reg);
15539 Register d = as_Register($dst$$reg);
15540 __ fcmpd(s1, s2);
15541 // installs 0 if EQ else -1
15542 __ csinvw(d, zr, zr, Assembler::EQ);
15543 // keeps -1 if less or unordered else installs 1
15544 __ csnegw(d, d, d, Assembler::LT);
15545 __ bind(done);
15546 %}
15547 ins_pipe(pipe_class_default);
15548
15549 %}
15550
15551 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15552 %{
15553 match(Set dst (CmpF3 src1 zero));
15554 effect(KILL cr);
15555
15556 ins_cost(5 * INSN_COST);
15557 format %{ "fcmps $src1, 0.0\n\t"
15558 "csinvw($dst, zr, zr, eq\n\t"
15559 "csnegw($dst, $dst, $dst, lt)"
15560 %}
15561
15562 ins_encode %{
15563 Label done;
15564 FloatRegister s1 = as_FloatRegister($src1$$reg);
15565 Register d = as_Register($dst$$reg);
15566 __ fcmps(s1, 0.0);
15567 // installs 0 if EQ else -1
15568 __ csinvw(d, zr, zr, Assembler::EQ);
15569 // keeps -1 if less or unordered else installs 1
15570 __ csnegw(d, d, d, Assembler::LT);
15571 __ bind(done);
15572 %}
15573
15574 ins_pipe(pipe_class_default);
15575
15576 %}
15577
15578 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15579 %{
15580 match(Set dst (CmpD3 src1 zero));
15581 effect(KILL cr);
15582
15583 ins_cost(5 * INSN_COST);
15584 format %{ "fcmpd $src1, 0.0\n\t"
15585 "csinvw($dst, zr, zr, eq\n\t"
15586 "csnegw($dst, $dst, $dst, lt)"
15587 %}
15588
15589 ins_encode %{
15590 Label done;
15591 FloatRegister s1 = as_FloatRegister($src1$$reg);
15592 Register d = as_Register($dst$$reg);
15593 __ fcmpd(s1, 0.0);
15594 // installs 0 if EQ else -1
15595 __ csinvw(d, zr, zr, Assembler::EQ);
15596 // keeps -1 if less or unordered else installs 1
15597 __ csnegw(d, d, d, Assembler::LT);
15598 __ bind(done);
15599 %}
15600 ins_pipe(pipe_class_default);
15601
15602 %}
15603
15604 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15605 %{
15606 match(Set dst (CmpLTMask p q));
15607 effect(KILL cr);
15608
15609 ins_cost(3 * INSN_COST);
15610
15611 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15612 "csetw $dst, lt\n\t"
15613 "subw $dst, zr, $dst"
15614 %}
15615
15616 ins_encode %{
15617 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15618 __ csetw(as_Register($dst$$reg), Assembler::LT);
15619 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15620 %}
15621
15622 ins_pipe(ialu_reg_reg);
15623 %}
15624
15625 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15626 %{
15627 match(Set dst (CmpLTMask src zero));
15628 effect(KILL cr);
15629
15630 ins_cost(INSN_COST);
15631
15632 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15633
15634 ins_encode %{
15635 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15636 %}
15637
15638 ins_pipe(ialu_reg_shift);
15639 %}
15640
15641 // ============================================================================
15642 // Max and Min
15643
15644 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15645
15646 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15647 %{
15648 effect(DEF cr, USE src);
15649 ins_cost(INSN_COST);
15650 format %{ "cmpw $src, 0" %}
15651
15652 ins_encode %{
15653 __ cmpw($src$$Register, 0);
15654 %}
15655 ins_pipe(icmp_reg_imm);
15656 %}
15657
15658 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15659 %{
15660 match(Set dst (MinI src1 src2));
15661 ins_cost(INSN_COST * 3);
15662
15663 expand %{
15664 rFlagsReg cr;
15665 compI_reg_reg(cr, src1, src2);
15666 cmovI_reg_reg_lt(dst, src1, src2, cr);
15667 %}
15668 %}
15669
15670 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15671 %{
15672 match(Set dst (MaxI src1 src2));
15673 ins_cost(INSN_COST * 3);
15674
15675 expand %{
15676 rFlagsReg cr;
15677 compI_reg_reg(cr, src1, src2);
15678 cmovI_reg_reg_gt(dst, src1, src2, cr);
15679 %}
15680 %}
15681
15682
15683 // ============================================================================
15684 // Branch Instructions
15685
15686 // Direct Branch.
15687 instruct branch(label lbl)
15688 %{
15689 match(Goto);
15690
15691 effect(USE lbl);
15692
15693 ins_cost(BRANCH_COST);
15694 format %{ "b $lbl" %}
15695
15696 ins_encode(aarch64_enc_b(lbl));
15697
15698 ins_pipe(pipe_branch);
15699 %}
15700
15701 // Conditional Near Branch
15702 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15703 %{
15704 // Same match rule as `branchConFar'.
15705 match(If cmp cr);
15706
15707 effect(USE lbl);
15708
15709 ins_cost(BRANCH_COST);
15710 // If set to 1 this indicates that the current instruction is a
15711 // short variant of a long branch. This avoids using this
15712 // instruction in first-pass matching. It will then only be used in
15713 // the `Shorten_branches' pass.
15714 // ins_short_branch(1);
15715 format %{ "b$cmp $lbl" %}
15716
15717 ins_encode(aarch64_enc_br_con(cmp, lbl));
15718
15719 ins_pipe(pipe_branch_cond);
15720 %}
15721
15722 // Conditional Near Branch Unsigned
15723 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15724 %{
15725 // Same match rule as `branchConFar'.
15726 match(If cmp cr);
15727
15728 effect(USE lbl);
15729
15730 ins_cost(BRANCH_COST);
15731 // If set to 1 this indicates that the current instruction is a
15732 // short variant of a long branch. This avoids using this
15733 // instruction in first-pass matching. It will then only be used in
15734 // the `Shorten_branches' pass.
15735 // ins_short_branch(1);
15736 format %{ "b$cmp $lbl\t# unsigned" %}
15737
15738 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15739
15740 ins_pipe(pipe_branch_cond);
15741 %}
15742
15743 // Make use of CBZ and CBNZ. These instructions, as well as being
15744 // shorter than (cmp; branch), have the additional benefit of not
15745 // killing the flags.
15746
15747 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15748 match(If cmp (CmpI op1 op2));
15749 effect(USE labl);
15750
15751 ins_cost(BRANCH_COST);
15752 format %{ "cbw$cmp $op1, $labl" %}
15753 ins_encode %{
15754 Label* L = $labl$$label;
15755 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15756 if (cond == Assembler::EQ)
15757 __ cbzw($op1$$Register, *L);
15758 else
15759 __ cbnzw($op1$$Register, *L);
15760 %}
15761 ins_pipe(pipe_cmp_branch);
15762 %}
15763
15764 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15765 match(If cmp (CmpL op1 op2));
15766 effect(USE labl);
15767
15768 ins_cost(BRANCH_COST);
15769 format %{ "cb$cmp $op1, $labl" %}
15770 ins_encode %{
15771 Label* L = $labl$$label;
15772 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15773 if (cond == Assembler::EQ)
15774 __ cbz($op1$$Register, *L);
15775 else
15776 __ cbnz($op1$$Register, *L);
15777 %}
15778 ins_pipe(pipe_cmp_branch);
15779 %}
15780
15781 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15782 match(If cmp (CmpP op1 op2));
15783 effect(USE labl);
15784
15785 ins_cost(BRANCH_COST);
15786 format %{ "cb$cmp $op1, $labl" %}
15787 ins_encode %{
15788 Label* L = $labl$$label;
15789 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15790 if (cond == Assembler::EQ)
15791 __ cbz($op1$$Register, *L);
15792 else
15793 __ cbnz($op1$$Register, *L);
15794 %}
15795 ins_pipe(pipe_cmp_branch);
15796 %}
15797
15798 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15799 match(If cmp (CmpN op1 op2));
15800 effect(USE labl);
15801
15802 ins_cost(BRANCH_COST);
15803 format %{ "cbw$cmp $op1, $labl" %}
15804 ins_encode %{
15805 Label* L = $labl$$label;
15806 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15807 if (cond == Assembler::EQ)
15808 __ cbzw($op1$$Register, *L);
15809 else
15810 __ cbnzw($op1$$Register, *L);
15811 %}
15812 ins_pipe(pipe_cmp_branch);
15813 %}
15814
15815 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15816 match(If cmp (CmpP (DecodeN oop) zero));
15817 effect(USE labl);
15818
15819 ins_cost(BRANCH_COST);
15820 format %{ "cb$cmp $oop, $labl" %}
15821 ins_encode %{
15822 Label* L = $labl$$label;
15823 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15824 if (cond == Assembler::EQ)
15825 __ cbzw($oop$$Register, *L);
15826 else
15827 __ cbnzw($oop$$Register, *L);
15828 %}
15829 ins_pipe(pipe_cmp_branch);
15830 %}
15831
15832 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15833 match(If cmp (CmpU op1 op2));
15834 effect(USE labl);
15835
15836 ins_cost(BRANCH_COST);
15837 format %{ "cbw$cmp $op1, $labl" %}
15838 ins_encode %{
15839 Label* L = $labl$$label;
15840 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15841 if (cond == Assembler::EQ || cond == Assembler::LS) {
15842 __ cbzw($op1$$Register, *L);
15843 } else {
15844 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15845 __ cbnzw($op1$$Register, *L);
15846 }
15847 %}
15848 ins_pipe(pipe_cmp_branch);
15849 %}
15850
15851 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15852 match(If cmp (CmpUL op1 op2));
15853 effect(USE labl);
15854
15855 ins_cost(BRANCH_COST);
15856 format %{ "cb$cmp $op1, $labl" %}
15857 ins_encode %{
15858 Label* L = $labl$$label;
15859 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15860 if (cond == Assembler::EQ || cond == Assembler::LS) {
15861 __ cbz($op1$$Register, *L);
15862 } else {
15863 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15864 __ cbnz($op1$$Register, *L);
15865 }
15866 %}
15867 ins_pipe(pipe_cmp_branch);
15868 %}
15869
15870 // Test bit and Branch
15871
15872 // Patterns for short (< 32KiB) variants
15873 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15874 match(If cmp (CmpL op1 op2));
15875 effect(USE labl);
15876
15877 ins_cost(BRANCH_COST);
15878 format %{ "cb$cmp $op1, $labl # long" %}
15879 ins_encode %{
15880 Label* L = $labl$$label;
15881 Assembler::Condition cond =
15882 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15883 __ tbr(cond, $op1$$Register, 63, *L);
15884 %}
15885 ins_pipe(pipe_cmp_branch);
15886 ins_short_branch(1);
15887 %}
15888
15889 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15890 match(If cmp (CmpI op1 op2));
15891 effect(USE labl);
15892
15893 ins_cost(BRANCH_COST);
15894 format %{ "cb$cmp $op1, $labl # int" %}
15895 ins_encode %{
15896 Label* L = $labl$$label;
15897 Assembler::Condition cond =
15898 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15899 __ tbr(cond, $op1$$Register, 31, *L);
15900 %}
15901 ins_pipe(pipe_cmp_branch);
15902 ins_short_branch(1);
15903 %}
15904
15905 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15906 match(If cmp (CmpL (AndL op1 op2) op3));
15907 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15908 effect(USE labl);
15909
15910 ins_cost(BRANCH_COST);
15911 format %{ "tb$cmp $op1, $op2, $labl" %}
15912 ins_encode %{
15913 Label* L = $labl$$label;
15914 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15915 int bit = exact_log2_long($op2$$constant);
15916 __ tbr(cond, $op1$$Register, bit, *L);
15917 %}
15918 ins_pipe(pipe_cmp_branch);
15919 ins_short_branch(1);
15920 %}
15921
15922 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15923 match(If cmp (CmpI (AndI op1 op2) op3));
15924 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15925 effect(USE labl);
15926
15927 ins_cost(BRANCH_COST);
15928 format %{ "tb$cmp $op1, $op2, $labl" %}
15929 ins_encode %{
15930 Label* L = $labl$$label;
15931 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15932 int bit = exact_log2((juint)$op2$$constant);
15933 __ tbr(cond, $op1$$Register, bit, *L);
15934 %}
15935 ins_pipe(pipe_cmp_branch);
15936 ins_short_branch(1);
15937 %}
15938
15939 // And far variants
15940 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15941 match(If cmp (CmpL op1 op2));
15942 effect(USE labl);
15943
15944 ins_cost(BRANCH_COST);
15945 format %{ "cb$cmp $op1, $labl # long" %}
15946 ins_encode %{
15947 Label* L = $labl$$label;
15948 Assembler::Condition cond =
15949 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15950 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15951 %}
15952 ins_pipe(pipe_cmp_branch);
15953 %}
15954
15955 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15956 match(If cmp (CmpI op1 op2));
15957 effect(USE labl);
15958
15959 ins_cost(BRANCH_COST);
15960 format %{ "cb$cmp $op1, $labl # int" %}
15961 ins_encode %{
15962 Label* L = $labl$$label;
15963 Assembler::Condition cond =
15964 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15965 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15966 %}
15967 ins_pipe(pipe_cmp_branch);
15968 %}
15969
15970 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15971 match(If cmp (CmpL (AndL op1 op2) op3));
15972 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15973 effect(USE labl);
15974
15975 ins_cost(BRANCH_COST);
15976 format %{ "tb$cmp $op1, $op2, $labl" %}
15977 ins_encode %{
15978 Label* L = $labl$$label;
15979 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15980 int bit = exact_log2_long($op2$$constant);
15981 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15982 %}
15983 ins_pipe(pipe_cmp_branch);
15984 %}
15985
15986 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15987 match(If cmp (CmpI (AndI op1 op2) op3));
15988 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15989 effect(USE labl);
15990
15991 ins_cost(BRANCH_COST);
15992 format %{ "tb$cmp $op1, $op2, $labl" %}
15993 ins_encode %{
15994 Label* L = $labl$$label;
15995 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15996 int bit = exact_log2((juint)$op2$$constant);
15997 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15998 %}
15999 ins_pipe(pipe_cmp_branch);
16000 %}
16001
16002 // Test bits
16003
16004 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16005 match(Set cr (CmpL (AndL op1 op2) op3));
16006 predicate(Assembler::operand_valid_for_logical_immediate
16007 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16008
16009 ins_cost(INSN_COST);
16010 format %{ "tst $op1, $op2 # long" %}
16011 ins_encode %{
16012 __ tst($op1$$Register, $op2$$constant);
16013 %}
16014 ins_pipe(ialu_reg_reg);
16015 %}
16016
16017 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16018 match(Set cr (CmpI (AndI op1 op2) op3));
16019 predicate(Assembler::operand_valid_for_logical_immediate
16020 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16021
16022 ins_cost(INSN_COST);
16023 format %{ "tst $op1, $op2 # int" %}
16024 ins_encode %{
16025 __ tstw($op1$$Register, $op2$$constant);
16026 %}
16027 ins_pipe(ialu_reg_reg);
16028 %}
16029
16030 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16031 match(Set cr (CmpL (AndL op1 op2) op3));
16032
16033 ins_cost(INSN_COST);
16034 format %{ "tst $op1, $op2 # long" %}
16035 ins_encode %{
16036 __ tst($op1$$Register, $op2$$Register);
16037 %}
16038 ins_pipe(ialu_reg_reg);
16039 %}
16040
16041 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16042 match(Set cr (CmpI (AndI op1 op2) op3));
16043
16044 ins_cost(INSN_COST);
16045 format %{ "tstw $op1, $op2 # int" %}
16046 ins_encode %{
16047 __ tstw($op1$$Register, $op2$$Register);
16048 %}
16049 ins_pipe(ialu_reg_reg);
16050 %}
16051
16052
16053 // Conditional Far Branch
16054 // Conditional Far Branch Unsigned
16055 // TODO: fixme
16056
16057 // counted loop end branch near
16058 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16059 %{
16060 match(CountedLoopEnd cmp cr);
16061
16062 effect(USE lbl);
16063
16064 ins_cost(BRANCH_COST);
16065 // short variant.
16066 // ins_short_branch(1);
16067 format %{ "b$cmp $lbl \t// counted loop end" %}
16068
16069 ins_encode(aarch64_enc_br_con(cmp, lbl));
16070
16071 ins_pipe(pipe_branch);
16072 %}
16073
16074 // counted loop end branch far
16075 // TODO: fixme
16076
16077 // ============================================================================
16078 // inlined locking and unlocking
16079
16080 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16081 %{
16082 predicate(LockingMode != LM_LIGHTWEIGHT);
16083 match(Set cr (FastLock object box));
16084 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16085
16086 ins_cost(5 * INSN_COST);
16087 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16088
16089 ins_encode %{
16090 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16091 %}
16092
16093 ins_pipe(pipe_serial);
16094 %}
16095
16096 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16097 %{
16098 predicate(LockingMode != LM_LIGHTWEIGHT);
16099 match(Set cr (FastUnlock object box));
16100 effect(TEMP tmp, TEMP tmp2);
16101
16102 ins_cost(5 * INSN_COST);
16103 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16104
16105 ins_encode %{
16106 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16107 %}
16108
16109 ins_pipe(pipe_serial);
16110 %}
16111
16112 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16113 %{
16114 predicate(LockingMode == LM_LIGHTWEIGHT);
16115 match(Set cr (FastLock object box));
16116 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16117
16118 ins_cost(5 * INSN_COST);
16119 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16120
16121 ins_encode %{
16122 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16123 %}
16124
16125 ins_pipe(pipe_serial);
16126 %}
16127
16128 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16129 %{
16130 predicate(LockingMode == LM_LIGHTWEIGHT);
16131 match(Set cr (FastUnlock object box));
16132 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16133
16134 ins_cost(5 * INSN_COST);
16135 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16136
16137 ins_encode %{
16138 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16139 %}
16140
16141 ins_pipe(pipe_serial);
16142 %}
16143
16144 // ============================================================================
16145 // Safepoint Instructions
16146
16147 // TODO
16148 // provide a near and far version of this code
16149
16150 instruct safePoint(rFlagsReg cr, iRegP poll)
16151 %{
16152 match(SafePoint poll);
16153 effect(KILL cr);
16154
16155 format %{
16156 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16157 %}
16158 ins_encode %{
16159 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16160 %}
16161 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16162 %}
16163
16164
16165 // ============================================================================
16166 // Procedure Call/Return Instructions
16167
16168 // Call Java Static Instruction
16169
16170 instruct CallStaticJavaDirect(method meth)
16171 %{
16172 match(CallStaticJava);
16173
16174 effect(USE meth);
16175
16176 ins_cost(CALL_COST);
16177
16178 format %{ "call,static $meth \t// ==> " %}
16179
16180 ins_encode(aarch64_enc_java_static_call(meth),
16181 aarch64_enc_call_epilog);
16182
16183 ins_pipe(pipe_class_call);
16184 %}
16185
16186 // TO HERE
16187
16188 // Call Java Dynamic Instruction
16189 instruct CallDynamicJavaDirect(method meth)
16190 %{
16191 match(CallDynamicJava);
16192
16193 effect(USE meth);
16194
16195 ins_cost(CALL_COST);
16196
16197 format %{ "CALL,dynamic $meth \t// ==> " %}
16198
16199 ins_encode(aarch64_enc_java_dynamic_call(meth),
16200 aarch64_enc_call_epilog);
16201
16202 ins_pipe(pipe_class_call);
16203 %}
16204
16205 // Call Runtime Instruction
16206
16207 instruct CallRuntimeDirect(method meth)
16208 %{
16209 match(CallRuntime);
16210
16211 effect(USE meth);
16212
16213 ins_cost(CALL_COST);
16214
16215 format %{ "CALL, runtime $meth" %}
16216
16217 ins_encode( aarch64_enc_java_to_runtime(meth) );
16218
16219 ins_pipe(pipe_class_call);
16220 %}
16221
16222 // Call Runtime Instruction
16223
16224 instruct CallLeafDirect(method meth)
16225 %{
16226 match(CallLeaf);
16227
16228 effect(USE meth);
16229
16230 ins_cost(CALL_COST);
16231
16232 format %{ "CALL, runtime leaf $meth" %}
16233
16234 ins_encode( aarch64_enc_java_to_runtime(meth) );
16235
16236 ins_pipe(pipe_class_call);
16237 %}
16238
16239 // Call Runtime Instruction without safepoint and with vector arguments
16240 instruct CallLeafDirectVector(method meth)
16241 %{
16242 match(CallLeafVector);
16243
16244 effect(USE meth);
16245
16246 ins_cost(CALL_COST);
16247
16248 format %{ "CALL, runtime leaf vector $meth" %}
16249
16250 ins_encode(aarch64_enc_java_to_runtime(meth));
16251
16252 ins_pipe(pipe_class_call);
16253 %}
16254
16255 // Call Runtime Instruction
16256
16257 // entry point is null, target holds the address to call
16258 instruct CallLeafNoFPIndirect(iRegP target)
16259 %{
16260 predicate(n->as_Call()->entry_point() == nullptr);
16261
16262 match(CallLeafNoFP target);
16263
16264 ins_cost(CALL_COST);
16265
16266 format %{ "CALL, runtime leaf nofp indirect $target" %}
16267
16268 ins_encode %{
16269 __ blr($target$$Register);
16270 %}
16271
16272 ins_pipe(pipe_class_call);
16273 %}
16274
16275 instruct CallLeafNoFPDirect(method meth)
16276 %{
16277 predicate(n->as_Call()->entry_point() != nullptr);
16278
16279 match(CallLeafNoFP);
16280
16281 effect(USE meth);
16282
16283 ins_cost(CALL_COST);
16284
16285 format %{ "CALL, runtime leaf nofp $meth" %}
16286
16287 ins_encode( aarch64_enc_java_to_runtime(meth) );
16288
16289 ins_pipe(pipe_class_call);
16290 %}
16291
16292 // Tail Call; Jump from runtime stub to Java code.
16293 // Also known as an 'interprocedural jump'.
16294 // Target of jump will eventually return to caller.
16295 // TailJump below removes the return address.
16296 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16297 // emitted just above the TailCall which has reset rfp to the caller state.
16298 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16299 %{
16300 match(TailCall jump_target method_ptr);
16301
16302 ins_cost(CALL_COST);
16303
16304 format %{ "br $jump_target\t# $method_ptr holds method" %}
16305
16306 ins_encode(aarch64_enc_tail_call(jump_target));
16307
16308 ins_pipe(pipe_class_call);
16309 %}
16310
16311 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16312 %{
16313 match(TailJump jump_target ex_oop);
16314
16315 ins_cost(CALL_COST);
16316
16317 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16318
16319 ins_encode(aarch64_enc_tail_jmp(jump_target));
16320
16321 ins_pipe(pipe_class_call);
16322 %}
16323
16324 // Forward exception.
16325 instruct ForwardExceptionjmp()
16326 %{
16327 match(ForwardException);
16328 ins_cost(CALL_COST);
16329
16330 format %{ "b forward_exception_stub" %}
16331 ins_encode %{
16332 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16333 %}
16334 ins_pipe(pipe_class_call);
16335 %}
16336
16337 // Create exception oop: created by stack-crawling runtime code.
16338 // Created exception is now available to this handler, and is setup
16339 // just prior to jumping to this handler. No code emitted.
16340 // TODO check
16341 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16342 instruct CreateException(iRegP_R0 ex_oop)
16343 %{
16344 match(Set ex_oop (CreateEx));
16345
16346 format %{ " -- \t// exception oop; no code emitted" %}
16347
16348 size(0);
16349
16350 ins_encode( /*empty*/ );
16351
16352 ins_pipe(pipe_class_empty);
16353 %}
16354
16355 // Rethrow exception: The exception oop will come in the first
16356 // argument position. Then JUMP (not call) to the rethrow stub code.
16357 instruct RethrowException() %{
16358 match(Rethrow);
16359 ins_cost(CALL_COST);
16360
16361 format %{ "b rethrow_stub" %}
16362
16363 ins_encode( aarch64_enc_rethrow() );
16364
16365 ins_pipe(pipe_class_call);
16366 %}
16367
16368
16369 // Return Instruction
16370 // epilog node loads ret address into lr as part of frame pop
16371 instruct Ret()
16372 %{
16373 match(Return);
16374
16375 format %{ "ret\t// return register" %}
16376
16377 ins_encode( aarch64_enc_ret() );
16378
16379 ins_pipe(pipe_branch);
16380 %}
16381
16382 // Die now.
16383 instruct ShouldNotReachHere() %{
16384 match(Halt);
16385
16386 ins_cost(CALL_COST);
16387 format %{ "ShouldNotReachHere" %}
16388
16389 ins_encode %{
16390 if (is_reachable()) {
16391 __ stop(_halt_reason);
16392 }
16393 %}
16394
16395 ins_pipe(pipe_class_default);
16396 %}
16397
16398 // ============================================================================
16399 // Partial Subtype Check
16400 //
16401 // superklass array for an instance of the superklass. Set a hidden
16402 // internal cache on a hit (cache is checked with exposed code in
16403 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16404 // encoding ALSO sets flags.
16405
16406 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16407 %{
16408 match(Set result (PartialSubtypeCheck sub super));
16409 predicate(!UseSecondarySupersTable);
16410 effect(KILL cr, KILL temp);
16411
16412 ins_cost(20 * INSN_COST); // slightly larger than the next version
16413 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16414
16415 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16416
16417 opcode(0x1); // Force zero of result reg on hit
16418
16419 ins_pipe(pipe_class_memory);
16420 %}
16421
16422 // Two versions of partialSubtypeCheck, both used when we need to
16423 // search for a super class in the secondary supers array. The first
16424 // is used when we don't know _a priori_ the class being searched
16425 // for. The second, far more common, is used when we do know: this is
16426 // used for instanceof, checkcast, and any case where C2 can determine
16427 // it by constant propagation.
16428
16429 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16430 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16431 rFlagsReg cr)
16432 %{
16433 match(Set result (PartialSubtypeCheck sub super));
16434 predicate(UseSecondarySupersTable);
16435 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16436
16437 ins_cost(10 * INSN_COST); // slightly larger than the next version
16438 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16439
16440 ins_encode %{
16441 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16442 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16443 $vtemp$$FloatRegister,
16444 $result$$Register, /*L_success*/nullptr);
16445 %}
16446
16447 ins_pipe(pipe_class_memory);
16448 %}
16449
16450 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16451 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16452 rFlagsReg cr)
16453 %{
16454 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16455 predicate(UseSecondarySupersTable);
16456 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16457
16458 ins_cost(5 * INSN_COST); // smaller than the next version
16459 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16460
16461 ins_encode %{
16462 bool success = false;
16463 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16464 if (InlineSecondarySupersTest) {
16465 success =
16466 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16467 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16468 $vtemp$$FloatRegister,
16469 $result$$Register,
16470 super_klass_slot);
16471 } else {
16472 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16473 success = (call != nullptr);
16474 }
16475 if (!success) {
16476 ciEnv::current()->record_failure("CodeCache is full");
16477 return;
16478 }
16479 %}
16480
16481 ins_pipe(pipe_class_memory);
16482 %}
16483
16484 // Intrisics for String.compareTo()
16485
16486 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16487 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16488 %{
16489 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16490 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16491 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16492
16493 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16494 ins_encode %{
16495 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16496 __ string_compare($str1$$Register, $str2$$Register,
16497 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16498 $tmp1$$Register, $tmp2$$Register,
16499 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16500 %}
16501 ins_pipe(pipe_class_memory);
16502 %}
16503
16504 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16505 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16506 %{
16507 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16508 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16509 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16510
16511 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16512 ins_encode %{
16513 __ string_compare($str1$$Register, $str2$$Register,
16514 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16515 $tmp1$$Register, $tmp2$$Register,
16516 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16517 %}
16518 ins_pipe(pipe_class_memory);
16519 %}
16520
16521 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16522 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16523 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16524 %{
16525 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16526 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16527 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16528 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16529
16530 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16531 ins_encode %{
16532 __ string_compare($str1$$Register, $str2$$Register,
16533 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16534 $tmp1$$Register, $tmp2$$Register,
16535 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16536 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16537 %}
16538 ins_pipe(pipe_class_memory);
16539 %}
16540
16541 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16542 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16543 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16544 %{
16545 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16546 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16547 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16548 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16549
16550 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16551 ins_encode %{
16552 __ string_compare($str1$$Register, $str2$$Register,
16553 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16554 $tmp1$$Register, $tmp2$$Register,
16555 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16556 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16557 %}
16558 ins_pipe(pipe_class_memory);
16559 %}
16560
16561 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16562 // these string_compare variants as NEON register type for convenience so that the prototype of
16563 // string_compare can be shared with all variants.
16564
16565 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16566 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16567 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16568 pRegGov_P1 pgtmp2, rFlagsReg cr)
16569 %{
16570 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16571 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16572 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16573 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16574
16575 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16576 ins_encode %{
16577 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16578 __ string_compare($str1$$Register, $str2$$Register,
16579 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16580 $tmp1$$Register, $tmp2$$Register,
16581 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16582 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16583 StrIntrinsicNode::LL);
16584 %}
16585 ins_pipe(pipe_class_memory);
16586 %}
16587
16588 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16589 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16590 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16591 pRegGov_P1 pgtmp2, rFlagsReg cr)
16592 %{
16593 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16594 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16595 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16596 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16597
16598 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16599 ins_encode %{
16600 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16601 __ string_compare($str1$$Register, $str2$$Register,
16602 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16603 $tmp1$$Register, $tmp2$$Register,
16604 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16605 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16606 StrIntrinsicNode::LU);
16607 %}
16608 ins_pipe(pipe_class_memory);
16609 %}
16610
16611 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16612 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16613 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16614 pRegGov_P1 pgtmp2, rFlagsReg cr)
16615 %{
16616 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16617 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16618 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16619 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16620
16621 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16622 ins_encode %{
16623 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16624 __ string_compare($str1$$Register, $str2$$Register,
16625 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16626 $tmp1$$Register, $tmp2$$Register,
16627 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16628 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16629 StrIntrinsicNode::UL);
16630 %}
16631 ins_pipe(pipe_class_memory);
16632 %}
16633
16634 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16635 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16636 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16637 pRegGov_P1 pgtmp2, rFlagsReg cr)
16638 %{
16639 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16640 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16641 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16642 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16643
16644 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16645 ins_encode %{
16646 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16647 __ string_compare($str1$$Register, $str2$$Register,
16648 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16649 $tmp1$$Register, $tmp2$$Register,
16650 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16651 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16652 StrIntrinsicNode::UU);
16653 %}
16654 ins_pipe(pipe_class_memory);
16655 %}
16656
16657 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16658 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16659 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16660 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16661 %{
16662 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16663 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16664 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16665 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16666 TEMP vtmp0, TEMP vtmp1, KILL cr);
16667 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16668 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16669
16670 ins_encode %{
16671 __ string_indexof($str1$$Register, $str2$$Register,
16672 $cnt1$$Register, $cnt2$$Register,
16673 $tmp1$$Register, $tmp2$$Register,
16674 $tmp3$$Register, $tmp4$$Register,
16675 $tmp5$$Register, $tmp6$$Register,
16676 -1, $result$$Register, StrIntrinsicNode::UU);
16677 %}
16678 ins_pipe(pipe_class_memory);
16679 %}
16680
16681 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16682 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16683 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16684 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16685 %{
16686 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16687 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16688 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16689 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16690 TEMP vtmp0, TEMP vtmp1, KILL cr);
16691 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16692 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16693
16694 ins_encode %{
16695 __ string_indexof($str1$$Register, $str2$$Register,
16696 $cnt1$$Register, $cnt2$$Register,
16697 $tmp1$$Register, $tmp2$$Register,
16698 $tmp3$$Register, $tmp4$$Register,
16699 $tmp5$$Register, $tmp6$$Register,
16700 -1, $result$$Register, StrIntrinsicNode::LL);
16701 %}
16702 ins_pipe(pipe_class_memory);
16703 %}
16704
16705 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16706 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16707 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16708 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16709 %{
16710 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16711 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16712 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16713 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16714 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16715 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16716 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16717
16718 ins_encode %{
16719 __ string_indexof($str1$$Register, $str2$$Register,
16720 $cnt1$$Register, $cnt2$$Register,
16721 $tmp1$$Register, $tmp2$$Register,
16722 $tmp3$$Register, $tmp4$$Register,
16723 $tmp5$$Register, $tmp6$$Register,
16724 -1, $result$$Register, StrIntrinsicNode::UL);
16725 %}
16726 ins_pipe(pipe_class_memory);
16727 %}
16728
16729 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16730 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16731 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16732 %{
16733 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16734 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16735 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16736 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16737 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16738 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16739
16740 ins_encode %{
16741 int icnt2 = (int)$int_cnt2$$constant;
16742 __ string_indexof($str1$$Register, $str2$$Register,
16743 $cnt1$$Register, zr,
16744 $tmp1$$Register, $tmp2$$Register,
16745 $tmp3$$Register, $tmp4$$Register, zr, zr,
16746 icnt2, $result$$Register, StrIntrinsicNode::UU);
16747 %}
16748 ins_pipe(pipe_class_memory);
16749 %}
16750
16751 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16752 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16753 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16754 %{
16755 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16756 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16757 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16758 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16759 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16760 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16761
16762 ins_encode %{
16763 int icnt2 = (int)$int_cnt2$$constant;
16764 __ string_indexof($str1$$Register, $str2$$Register,
16765 $cnt1$$Register, zr,
16766 $tmp1$$Register, $tmp2$$Register,
16767 $tmp3$$Register, $tmp4$$Register, zr, zr,
16768 icnt2, $result$$Register, StrIntrinsicNode::LL);
16769 %}
16770 ins_pipe(pipe_class_memory);
16771 %}
16772
16773 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16774 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16775 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16776 %{
16777 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16778 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16779 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16780 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16781 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16782 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16783
16784 ins_encode %{
16785 int icnt2 = (int)$int_cnt2$$constant;
16786 __ string_indexof($str1$$Register, $str2$$Register,
16787 $cnt1$$Register, zr,
16788 $tmp1$$Register, $tmp2$$Register,
16789 $tmp3$$Register, $tmp4$$Register, zr, zr,
16790 icnt2, $result$$Register, StrIntrinsicNode::UL);
16791 %}
16792 ins_pipe(pipe_class_memory);
16793 %}
16794
16795 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16796 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16797 iRegINoSp tmp3, rFlagsReg cr)
16798 %{
16799 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16800 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16801 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16802 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16803
16804 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16805
16806 ins_encode %{
16807 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16808 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16809 $tmp3$$Register);
16810 %}
16811 ins_pipe(pipe_class_memory);
16812 %}
16813
16814 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16815 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16816 iRegINoSp tmp3, rFlagsReg cr)
16817 %{
16818 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16819 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16820 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16821 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16822
16823 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16824
16825 ins_encode %{
16826 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16827 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16828 $tmp3$$Register);
16829 %}
16830 ins_pipe(pipe_class_memory);
16831 %}
16832
16833 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16834 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16835 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16836 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16837 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16838 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16839 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16840 ins_encode %{
16841 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16842 $result$$Register, $ztmp1$$FloatRegister,
16843 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16844 $ptmp$$PRegister, true /* isL */);
16845 %}
16846 ins_pipe(pipe_class_memory);
16847 %}
16848
16849 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16850 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16851 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16852 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16853 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16854 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16855 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16856 ins_encode %{
16857 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16858 $result$$Register, $ztmp1$$FloatRegister,
16859 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16860 $ptmp$$PRegister, false /* isL */);
16861 %}
16862 ins_pipe(pipe_class_memory);
16863 %}
16864
16865 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16866 iRegI_R0 result, rFlagsReg cr)
16867 %{
16868 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16869 match(Set result (StrEquals (Binary str1 str2) cnt));
16870 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16871
16872 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16873 ins_encode %{
16874 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16875 __ string_equals($str1$$Register, $str2$$Register,
16876 $result$$Register, $cnt$$Register);
16877 %}
16878 ins_pipe(pipe_class_memory);
16879 %}
16880
16881 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16882 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16883 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16884 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16885 iRegP_R10 tmp, rFlagsReg cr)
16886 %{
16887 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16888 match(Set result (AryEq ary1 ary2));
16889 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16890 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16891 TEMP vtmp6, TEMP vtmp7, KILL cr);
16892
16893 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16894 ins_encode %{
16895 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16896 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16897 $result$$Register, $tmp$$Register, 1);
16898 if (tpc == nullptr) {
16899 ciEnv::current()->record_failure("CodeCache is full");
16900 return;
16901 }
16902 %}
16903 ins_pipe(pipe_class_memory);
16904 %}
16905
16906 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16907 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16908 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16909 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16910 iRegP_R10 tmp, rFlagsReg cr)
16911 %{
16912 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16913 match(Set result (AryEq ary1 ary2));
16914 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16915 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16916 TEMP vtmp6, TEMP vtmp7, KILL cr);
16917
16918 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16919 ins_encode %{
16920 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16921 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16922 $result$$Register, $tmp$$Register, 2);
16923 if (tpc == nullptr) {
16924 ciEnv::current()->record_failure("CodeCache is full");
16925 return;
16926 }
16927 %}
16928 ins_pipe(pipe_class_memory);
16929 %}
16930
16931 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16932 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16933 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16934 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16935 %{
16936 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16937 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16938 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16939
16940 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16941 ins_encode %{
16942 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16943 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16944 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16945 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16946 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16947 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16948 (BasicType)$basic_type$$constant);
16949 if (tpc == nullptr) {
16950 ciEnv::current()->record_failure("CodeCache is full");
16951 return;
16952 }
16953 %}
16954 ins_pipe(pipe_class_memory);
16955 %}
16956
16957 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16958 %{
16959 match(Set result (CountPositives ary1 len));
16960 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16961 format %{ "count positives byte[] $ary1,$len -> $result" %}
16962 ins_encode %{
16963 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16964 if (tpc == nullptr) {
16965 ciEnv::current()->record_failure("CodeCache is full");
16966 return;
16967 }
16968 %}
16969 ins_pipe( pipe_slow );
16970 %}
16971
16972 // fast char[] to byte[] compression
16973 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16974 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16975 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16976 iRegI_R0 result, rFlagsReg cr)
16977 %{
16978 match(Set result (StrCompressedCopy src (Binary dst len)));
16979 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16980 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16981
16982 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16983 ins_encode %{
16984 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16985 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16986 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16987 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16988 %}
16989 ins_pipe(pipe_slow);
16990 %}
16991
16992 // fast byte[] to char[] inflation
16993 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16994 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16995 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16996 %{
16997 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16998 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16999 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17000 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17001
17002 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17003 ins_encode %{
17004 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17005 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17006 $vtmp2$$FloatRegister, $tmp$$Register);
17007 if (tpc == nullptr) {
17008 ciEnv::current()->record_failure("CodeCache is full");
17009 return;
17010 }
17011 %}
17012 ins_pipe(pipe_class_memory);
17013 %}
17014
17015 // encode char[] to byte[] in ISO_8859_1
17016 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17017 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17018 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17019 iRegI_R0 result, rFlagsReg cr)
17020 %{
17021 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17022 match(Set result (EncodeISOArray src (Binary dst len)));
17023 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17024 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17025
17026 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17027 ins_encode %{
17028 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17029 $result$$Register, false,
17030 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17031 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17032 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17033 %}
17034 ins_pipe(pipe_class_memory);
17035 %}
17036
17037 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17038 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17039 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17040 iRegI_R0 result, rFlagsReg cr)
17041 %{
17042 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17043 match(Set result (EncodeISOArray src (Binary dst len)));
17044 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17045 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17046
17047 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17048 ins_encode %{
17049 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17050 $result$$Register, true,
17051 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17052 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17053 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17054 %}
17055 ins_pipe(pipe_class_memory);
17056 %}
17057
17058 //----------------------------- CompressBits/ExpandBits ------------------------
17059
17060 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17061 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17062 match(Set dst (CompressBits src mask));
17063 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17064 format %{ "mov $tsrc, $src\n\t"
17065 "mov $tmask, $mask\n\t"
17066 "bext $tdst, $tsrc, $tmask\n\t"
17067 "mov $dst, $tdst"
17068 %}
17069 ins_encode %{
17070 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17071 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17072 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17073 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17074 %}
17075 ins_pipe(pipe_slow);
17076 %}
17077
17078 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17079 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17080 match(Set dst (CompressBits (LoadI mem) mask));
17081 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17082 format %{ "ldrs $tsrc, $mem\n\t"
17083 "ldrs $tmask, $mask\n\t"
17084 "bext $tdst, $tsrc, $tmask\n\t"
17085 "mov $dst, $tdst"
17086 %}
17087 ins_encode %{
17088 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17089 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17090 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17091 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17092 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17093 %}
17094 ins_pipe(pipe_slow);
17095 %}
17096
17097 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17098 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17099 match(Set dst (CompressBits src mask));
17100 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17101 format %{ "mov $tsrc, $src\n\t"
17102 "mov $tmask, $mask\n\t"
17103 "bext $tdst, $tsrc, $tmask\n\t"
17104 "mov $dst, $tdst"
17105 %}
17106 ins_encode %{
17107 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17108 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17109 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17110 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17111 %}
17112 ins_pipe(pipe_slow);
17113 %}
17114
17115 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17116 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17117 match(Set dst (CompressBits (LoadL mem) mask));
17118 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17119 format %{ "ldrd $tsrc, $mem\n\t"
17120 "ldrd $tmask, $mask\n\t"
17121 "bext $tdst, $tsrc, $tmask\n\t"
17122 "mov $dst, $tdst"
17123 %}
17124 ins_encode %{
17125 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17126 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17127 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17128 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17129 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17130 %}
17131 ins_pipe(pipe_slow);
17132 %}
17133
17134 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17135 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17136 match(Set dst (ExpandBits src mask));
17137 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17138 format %{ "mov $tsrc, $src\n\t"
17139 "mov $tmask, $mask\n\t"
17140 "bdep $tdst, $tsrc, $tmask\n\t"
17141 "mov $dst, $tdst"
17142 %}
17143 ins_encode %{
17144 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17145 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17146 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17147 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17148 %}
17149 ins_pipe(pipe_slow);
17150 %}
17151
17152 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17153 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17154 match(Set dst (ExpandBits (LoadI mem) mask));
17155 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17156 format %{ "ldrs $tsrc, $mem\n\t"
17157 "ldrs $tmask, $mask\n\t"
17158 "bdep $tdst, $tsrc, $tmask\n\t"
17159 "mov $dst, $tdst"
17160 %}
17161 ins_encode %{
17162 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17163 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17164 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17165 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17166 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17167 %}
17168 ins_pipe(pipe_slow);
17169 %}
17170
17171 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17172 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17173 match(Set dst (ExpandBits src mask));
17174 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17175 format %{ "mov $tsrc, $src\n\t"
17176 "mov $tmask, $mask\n\t"
17177 "bdep $tdst, $tsrc, $tmask\n\t"
17178 "mov $dst, $tdst"
17179 %}
17180 ins_encode %{
17181 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17182 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17183 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17184 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17185 %}
17186 ins_pipe(pipe_slow);
17187 %}
17188
17189
17190 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17191 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17192 match(Set dst (ExpandBits (LoadL mem) mask));
17193 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17194 format %{ "ldrd $tsrc, $mem\n\t"
17195 "ldrd $tmask, $mask\n\t"
17196 "bdep $tdst, $tsrc, $tmask\n\t"
17197 "mov $dst, $tdst"
17198 %}
17199 ins_encode %{
17200 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17201 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17202 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17203 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17204 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17205 %}
17206 ins_pipe(pipe_slow);
17207 %}
17208
17209 // ============================================================================
17210 // This name is KNOWN by the ADLC and cannot be changed.
17211 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17212 // for this guy.
17213 instruct tlsLoadP(thread_RegP dst)
17214 %{
17215 match(Set dst (ThreadLocal));
17216
17217 ins_cost(0);
17218
17219 format %{ " -- \t// $dst=Thread::current(), empty" %}
17220
17221 size(0);
17222
17223 ins_encode( /*empty*/ );
17224
17225 ins_pipe(pipe_class_empty);
17226 %}
17227
17228 //----------PEEPHOLE RULES-----------------------------------------------------
17229 // These must follow all instruction definitions as they use the names
17230 // defined in the instructions definitions.
17231 //
17232 // peepmatch ( root_instr_name [preceding_instruction]* );
17233 //
17234 // peepconstraint %{
17235 // (instruction_number.operand_name relational_op instruction_number.operand_name
17236 // [, ...] );
17237 // // instruction numbers are zero-based using left to right order in peepmatch
17238 //
17239 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17240 // // provide an instruction_number.operand_name for each operand that appears
17241 // // in the replacement instruction's match rule
17242 //
17243 // ---------VM FLAGS---------------------------------------------------------
17244 //
17245 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17246 //
17247 // Each peephole rule is given an identifying number starting with zero and
17248 // increasing by one in the order seen by the parser. An individual peephole
17249 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17250 // on the command-line.
17251 //
17252 // ---------CURRENT LIMITATIONS----------------------------------------------
17253 //
17254 // Only match adjacent instructions in same basic block
17255 // Only equality constraints
17256 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17257 // Only one replacement instruction
17258 //
17259 // ---------EXAMPLE----------------------------------------------------------
17260 //
17261 // // pertinent parts of existing instructions in architecture description
17262 // instruct movI(iRegINoSp dst, iRegI src)
17263 // %{
17264 // match(Set dst (CopyI src));
17265 // %}
17266 //
17267 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17268 // %{
17269 // match(Set dst (AddI dst src));
17270 // effect(KILL cr);
17271 // %}
17272 //
17273 // // Change (inc mov) to lea
17274 // peephole %{
17275 // // increment preceded by register-register move
17276 // peepmatch ( incI_iReg movI );
17277 // // require that the destination register of the increment
17278 // // match the destination register of the move
17279 // peepconstraint ( 0.dst == 1.dst );
17280 // // construct a replacement instruction that sets
17281 // // the destination to ( move's source register + one )
17282 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17283 // %}
17284 //
17285
17286 // Implementation no longer uses movX instructions since
17287 // machine-independent system no longer uses CopyX nodes.
17288 //
17289 // peephole
17290 // %{
17291 // peepmatch (incI_iReg movI);
17292 // peepconstraint (0.dst == 1.dst);
17293 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17294 // %}
17295
17296 // peephole
17297 // %{
17298 // peepmatch (decI_iReg movI);
17299 // peepconstraint (0.dst == 1.dst);
17300 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17301 // %}
17302
17303 // peephole
17304 // %{
17305 // peepmatch (addI_iReg_imm movI);
17306 // peepconstraint (0.dst == 1.dst);
17307 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17308 // %}
17309
17310 // peephole
17311 // %{
17312 // peepmatch (incL_iReg movL);
17313 // peepconstraint (0.dst == 1.dst);
17314 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17315 // %}
17316
17317 // peephole
17318 // %{
17319 // peepmatch (decL_iReg movL);
17320 // peepconstraint (0.dst == 1.dst);
17321 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17322 // %}
17323
17324 // peephole
17325 // %{
17326 // peepmatch (addL_iReg_imm movL);
17327 // peepconstraint (0.dst == 1.dst);
17328 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17329 // %}
17330
17331 // peephole
17332 // %{
17333 // peepmatch (addP_iReg_imm movP);
17334 // peepconstraint (0.dst == 1.dst);
17335 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17336 // %}
17337
17338 // // Change load of spilled value to only a spill
17339 // instruct storeI(memory mem, iRegI src)
17340 // %{
17341 // match(Set mem (StoreI mem src));
17342 // %}
17343 //
17344 // instruct loadI(iRegINoSp dst, memory mem)
17345 // %{
17346 // match(Set dst (LoadI mem));
17347 // %}
17348 //
17349
17350 //----------SMARTSPILL RULES---------------------------------------------------
17351 // These must follow all instruction definitions as they use the names
17352 // defined in the instructions definitions.
17353
17354 // Local Variables:
17355 // mode: c++
17356 // End: