1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for heapbase register
668 reg_class heapbase_reg(
669 R27, R27_H
670 );
671
672 // Class for thread register
673 reg_class thread_reg(
674 R28, R28_H
675 );
676
677 // Class for frame pointer register
678 reg_class fp_reg(
679 R29, R29_H
680 );
681
682 // Class for link register
683 reg_class lr_reg(
684 R30, R30_H
685 );
686
687 // Class for long sp register
688 reg_class sp_reg(
689 R31, R31_H
690 );
691
692 // Class for all pointer registers
693 reg_class ptr_reg %{
694 return _PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers
698 reg_class no_special_ptr_reg %{
699 return _NO_SPECIAL_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133
1134 class CallStubImpl {
1135
1136 //--------------------------------------------------------------
1137 //---< Used for optimization in Compile::shorten_branches >---
1138 //--------------------------------------------------------------
1139
1140 public:
1141 // Size of call trampoline stub.
1142 static uint size_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145
1146 // number of relocations needed by a call trampoline stub
1147 static uint reloc_call_trampoline() {
1148 return 0; // no call trampolines on this platform
1149 }
1150 };
1151
1152 class HandlerImpl {
1153
1154 public:
1155
1156 static int emit_exception_handler(CodeBuffer &cbuf);
1157 static int emit_deopt_handler(CodeBuffer& cbuf);
1158
1159 static uint size_exception_handler() {
1160 return MacroAssembler::far_codestub_branch_size();
1161 }
1162
1163 static uint size_deopt_handler() {
1164 // count one adr and one far branch instruction
1165 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1166 }
1167 };
1168
1169 class Node::PD {
1170 public:
1171 enum NodeFlags {
1172 _last_flag = Node::_last_flag
1173 };
1174 };
1175
1176 bool is_CAS(int opcode, bool maybe_volatile);
1177
1178 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1179
1180 bool unnecessary_acquire(const Node *barrier);
1181 bool needs_acquiring_load(const Node *load);
1182
1183 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1184
1185 bool unnecessary_release(const Node *barrier);
1186 bool unnecessary_volatile(const Node *barrier);
1187 bool needs_releasing_store(const Node *store);
1188
1189 // predicate controlling translation of CompareAndSwapX
1190 bool needs_acquiring_load_exclusive(const Node *load);
1191
1192 // predicate controlling addressing modes
1193 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1194 %}
1195
1196 source %{
1197
1198 // Derived RegMask with conditionally allocatable registers
1199
1200 void PhaseOutput::pd_perform_mach_node_analysis() {
1201 }
1202
1203 int MachNode::pd_alignment_required() const {
1204 return 1;
1205 }
1206
1207 int MachNode::compute_padding(int current_offset) const {
1208 return 0;
1209 }
1210
1211 RegMask _ANY_REG32_mask;
1212 RegMask _ANY_REG_mask;
1213 RegMask _PTR_REG_mask;
1214 RegMask _NO_SPECIAL_REG32_mask;
1215 RegMask _NO_SPECIAL_REG_mask;
1216 RegMask _NO_SPECIAL_PTR_REG_mask;
1217
1218 void reg_mask_init() {
1219 // We derive below RegMask(s) from the ones which are auto-generated from
1220 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1221 // registers conditionally reserved.
1222
1223 _ANY_REG32_mask = _ALL_REG32_mask;
1224 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1225
1226 _ANY_REG_mask = _ALL_REG_mask;
1227
1228 _PTR_REG_mask = _ALL_REG_mask;
1229
1230 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1231 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1232
1233 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1234 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1235
1236 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1237 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1238
1239 // r27 is not allocatable when compressed oops is on and heapbase is not
1240 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1241 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1242 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1244 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1245 }
1246
1247 // r29 is not allocatable when PreserveFramePointer is on
1248 if (PreserveFramePointer) {
1249 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1251 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1252 }
1253 }
1254
1255 // Optimizaton of volatile gets and puts
1256 // -------------------------------------
1257 //
1258 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1259 // use to implement volatile reads and writes. For a volatile read
1260 // we simply need
1261 //
1262 // ldar<x>
1263 //
1264 // and for a volatile write we need
1265 //
1266 // stlr<x>
1267 //
1268 // Alternatively, we can implement them by pairing a normal
1269 // load/store with a memory barrier. For a volatile read we need
1270 //
1271 // ldr<x>
1272 // dmb ishld
1273 //
1274 // for a volatile write
1275 //
1276 // dmb ish
1277 // str<x>
1278 // dmb ish
1279 //
1280 // We can also use ldaxr and stlxr to implement compare and swap CAS
1281 // sequences. These are normally translated to an instruction
1282 // sequence like the following
1283 //
1284 // dmb ish
1285 // retry:
1286 // ldxr<x> rval raddr
1287 // cmp rval rold
1288 // b.ne done
1289 // stlxr<x> rval, rnew, rold
1290 // cbnz rval retry
1291 // done:
1292 // cset r0, eq
1293 // dmb ishld
1294 //
1295 // Note that the exclusive store is already using an stlxr
1296 // instruction. That is required to ensure visibility to other
1297 // threads of the exclusive write (assuming it succeeds) before that
1298 // of any subsequent writes.
1299 //
1300 // The following instruction sequence is an improvement on the above
1301 //
1302 // retry:
1303 // ldaxr<x> rval raddr
1304 // cmp rval rold
1305 // b.ne done
1306 // stlxr<x> rval, rnew, rold
1307 // cbnz rval retry
1308 // done:
1309 // cset r0, eq
1310 //
1311 // We don't need the leading dmb ish since the stlxr guarantees
1312 // visibility of prior writes in the case that the swap is
1313 // successful. Crucially we don't have to worry about the case where
1314 // the swap is not successful since no valid program should be
1315 // relying on visibility of prior changes by the attempting thread
1316 // in the case where the CAS fails.
1317 //
1318 // Similarly, we don't need the trailing dmb ishld if we substitute
1319 // an ldaxr instruction since that will provide all the guarantees we
1320 // require regarding observation of changes made by other threads
1321 // before any change to the CAS address observed by the load.
1322 //
1323 // In order to generate the desired instruction sequence we need to
1324 // be able to identify specific 'signature' ideal graph node
1325 // sequences which i) occur as a translation of a volatile reads or
1326 // writes or CAS operations and ii) do not occur through any other
1327 // translation or graph transformation. We can then provide
1328 // alternative aldc matching rules which translate these node
1329 // sequences to the desired machine code sequences. Selection of the
1330 // alternative rules can be implemented by predicates which identify
1331 // the relevant node sequences.
1332 //
1333 // The ideal graph generator translates a volatile read to the node
1334 // sequence
1335 //
1336 // LoadX[mo_acquire]
1337 // MemBarAcquire
1338 //
1339 // As a special case when using the compressed oops optimization we
1340 // may also see this variant
1341 //
1342 // LoadN[mo_acquire]
1343 // DecodeN
1344 // MemBarAcquire
1345 //
1346 // A volatile write is translated to the node sequence
1347 //
1348 // MemBarRelease
1349 // StoreX[mo_release] {CardMark}-optional
1350 // MemBarVolatile
1351 //
1352 // n.b. the above node patterns are generated with a strict
1353 // 'signature' configuration of input and output dependencies (see
1354 // the predicates below for exact details). The card mark may be as
1355 // simple as a few extra nodes or, in a few GC configurations, may
1356 // include more complex control flow between the leading and
1357 // trailing memory barriers. However, whatever the card mark
1358 // configuration these signatures are unique to translated volatile
1359 // reads/stores -- they will not appear as a result of any other
1360 // bytecode translation or inlining nor as a consequence of
1361 // optimizing transforms.
1362 //
1363 // We also want to catch inlined unsafe volatile gets and puts and
1364 // be able to implement them using either ldar<x>/stlr<x> or some
1365 // combination of ldr<x>/stlr<x> and dmb instructions.
1366 //
1367 // Inlined unsafe volatiles puts manifest as a minor variant of the
1368 // normal volatile put node sequence containing an extra cpuorder
1369 // membar
1370 //
1371 // MemBarRelease
1372 // MemBarCPUOrder
1373 // StoreX[mo_release] {CardMark}-optional
1374 // MemBarCPUOrder
1375 // MemBarVolatile
1376 //
1377 // n.b. as an aside, a cpuorder membar is not itself subject to
1378 // matching and translation by adlc rules. However, the rule
1379 // predicates need to detect its presence in order to correctly
1380 // select the desired adlc rules.
1381 //
1382 // Inlined unsafe volatile gets manifest as a slightly different
1383 // node sequence to a normal volatile get because of the
1384 // introduction of some CPUOrder memory barriers to bracket the
1385 // Load. However, but the same basic skeleton of a LoadX feeding a
1386 // MemBarAcquire, possibly through an optional DecodeN, is still
1387 // present
1388 //
1389 // MemBarCPUOrder
1390 // || \\
1391 // MemBarCPUOrder LoadX[mo_acquire]
1392 // || |
1393 // || {DecodeN} optional
1394 // || /
1395 // MemBarAcquire
1396 //
1397 // In this case the acquire membar does not directly depend on the
1398 // load. However, we can be sure that the load is generated from an
1399 // inlined unsafe volatile get if we see it dependent on this unique
1400 // sequence of membar nodes. Similarly, given an acquire membar we
1401 // can know that it was added because of an inlined unsafe volatile
1402 // get if it is fed and feeds a cpuorder membar and if its feed
1403 // membar also feeds an acquiring load.
1404 //
1405 // Finally an inlined (Unsafe) CAS operation is translated to the
1406 // following ideal graph
1407 //
1408 // MemBarRelease
1409 // MemBarCPUOrder
1410 // CompareAndSwapX {CardMark}-optional
1411 // MemBarCPUOrder
1412 // MemBarAcquire
1413 //
1414 // So, where we can identify these volatile read and write
1415 // signatures we can choose to plant either of the above two code
1416 // sequences. For a volatile read we can simply plant a normal
1417 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1418 // also choose to inhibit translation of the MemBarAcquire and
1419 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1420 //
1421 // When we recognise a volatile store signature we can choose to
1422 // plant at a dmb ish as a translation for the MemBarRelease, a
1423 // normal str<x> and then a dmb ish for the MemBarVolatile.
1424 // Alternatively, we can inhibit translation of the MemBarRelease
1425 // and MemBarVolatile and instead plant a simple stlr<x>
1426 // instruction.
1427 //
1428 // when we recognise a CAS signature we can choose to plant a dmb
1429 // ish as a translation for the MemBarRelease, the conventional
1430 // macro-instruction sequence for the CompareAndSwap node (which
1431 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1432 // Alternatively, we can elide generation of the dmb instructions
1433 // and plant the alternative CompareAndSwap macro-instruction
1434 // sequence (which uses ldaxr<x>).
1435 //
1436 // Of course, the above only applies when we see these signature
1437 // configurations. We still want to plant dmb instructions in any
1438 // other cases where we may see a MemBarAcquire, MemBarRelease or
1439 // MemBarVolatile. For example, at the end of a constructor which
1440 // writes final/volatile fields we will see a MemBarRelease
1441 // instruction and this needs a 'dmb ish' lest we risk the
1442 // constructed object being visible without making the
1443 // final/volatile field writes visible.
1444 //
1445 // n.b. the translation rules below which rely on detection of the
1446 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1447 // If we see anything other than the signature configurations we
1448 // always just translate the loads and stores to ldr<x> and str<x>
1449 // and translate acquire, release and volatile membars to the
1450 // relevant dmb instructions.
1451 //
1452
1453 // is_CAS(int opcode, bool maybe_volatile)
1454 //
1455 // return true if opcode is one of the possible CompareAndSwapX
1456 // values otherwise false.
1457
1458 bool is_CAS(int opcode, bool maybe_volatile)
1459 {
1460 switch(opcode) {
1461 // We handle these
1462 case Op_CompareAndSwapI:
1463 case Op_CompareAndSwapL:
1464 case Op_CompareAndSwapP:
1465 case Op_CompareAndSwapN:
1466 case Op_ShenandoahCompareAndSwapP:
1467 case Op_ShenandoahCompareAndSwapN:
1468 case Op_CompareAndSwapB:
1469 case Op_CompareAndSwapS:
1470 case Op_GetAndSetI:
1471 case Op_GetAndSetL:
1472 case Op_GetAndSetP:
1473 case Op_GetAndSetN:
1474 case Op_GetAndAddI:
1475 case Op_GetAndAddL:
1476 return true;
1477 case Op_CompareAndExchangeI:
1478 case Op_CompareAndExchangeN:
1479 case Op_CompareAndExchangeB:
1480 case Op_CompareAndExchangeS:
1481 case Op_CompareAndExchangeL:
1482 case Op_CompareAndExchangeP:
1483 case Op_WeakCompareAndSwapB:
1484 case Op_WeakCompareAndSwapS:
1485 case Op_WeakCompareAndSwapI:
1486 case Op_WeakCompareAndSwapL:
1487 case Op_WeakCompareAndSwapP:
1488 case Op_WeakCompareAndSwapN:
1489 case Op_ShenandoahWeakCompareAndSwapP:
1490 case Op_ShenandoahWeakCompareAndSwapN:
1491 case Op_ShenandoahCompareAndExchangeP:
1492 case Op_ShenandoahCompareAndExchangeN:
1493 return maybe_volatile;
1494 default:
1495 return false;
1496 }
1497 }
1498
1499 // helper to determine the maximum number of Phi nodes we may need to
1500 // traverse when searching from a card mark membar for the merge mem
1501 // feeding a trailing membar or vice versa
1502
1503 // predicates controlling emit of ldr<x>/ldar<x>
1504
1505 bool unnecessary_acquire(const Node *barrier)
1506 {
1507 assert(barrier->is_MemBar(), "expecting a membar");
1508
1509 MemBarNode* mb = barrier->as_MemBar();
1510
1511 if (mb->trailing_load()) {
1512 return true;
1513 }
1514
1515 if (mb->trailing_load_store()) {
1516 Node* load_store = mb->in(MemBarNode::Precedent);
1517 assert(load_store->is_LoadStore(), "unexpected graph shape");
1518 return is_CAS(load_store->Opcode(), true);
1519 }
1520
1521 return false;
1522 }
1523
1524 bool needs_acquiring_load(const Node *n)
1525 {
1526 assert(n->is_Load(), "expecting a load");
1527 LoadNode *ld = n->as_Load();
1528 return ld->is_acquire();
1529 }
1530
1531 bool unnecessary_release(const Node *n)
1532 {
1533 assert((n->is_MemBar() &&
1534 n->Opcode() == Op_MemBarRelease),
1535 "expecting a release membar");
1536
1537 MemBarNode *barrier = n->as_MemBar();
1538 if (!barrier->leading()) {
1539 return false;
1540 } else {
1541 Node* trailing = barrier->trailing_membar();
1542 MemBarNode* trailing_mb = trailing->as_MemBar();
1543 assert(trailing_mb->trailing(), "Not a trailing membar?");
1544 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1545
1546 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1547 if (mem->is_Store()) {
1548 assert(mem->as_Store()->is_release(), "");
1549 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1550 return true;
1551 } else {
1552 assert(mem->is_LoadStore(), "");
1553 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1554 return is_CAS(mem->Opcode(), true);
1555 }
1556 }
1557 return false;
1558 }
1559
1560 bool unnecessary_volatile(const Node *n)
1561 {
1562 // assert n->is_MemBar();
1563 MemBarNode *mbvol = n->as_MemBar();
1564
1565 bool release = mbvol->trailing_store();
1566 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1567 #ifdef ASSERT
1568 if (release) {
1569 Node* leading = mbvol->leading_membar();
1570 assert(leading->Opcode() == Op_MemBarRelease, "");
1571 assert(leading->as_MemBar()->leading_store(), "");
1572 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1573 }
1574 #endif
1575
1576 return release;
1577 }
1578
1579 // predicates controlling emit of str<x>/stlr<x>
1580
1581 bool needs_releasing_store(const Node *n)
1582 {
1583 // assert n->is_Store();
1584 StoreNode *st = n->as_Store();
1585 return st->trailing_membar() != NULL;
1586 }
1587
1588 // predicate controlling translation of CAS
1589 //
1590 // returns true if CAS needs to use an acquiring load otherwise false
1591
1592 bool needs_acquiring_load_exclusive(const Node *n)
1593 {
1594 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1595 LoadStoreNode* ldst = n->as_LoadStore();
1596 if (is_CAS(n->Opcode(), false)) {
1597 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1598 } else {
1599 return ldst->trailing_membar() != NULL;
1600 }
1601
1602 // so we can just return true here
1603 return true;
1604 }
1605
1606 #define __ _masm.
1607
1608 // advance declarations for helper functions to convert register
1609 // indices to register objects
1610
1611 // the ad file has to provide implementations of certain methods
1612 // expected by the generic code
1613 //
1614 // REQUIRED FUNCTIONALITY
1615
1616 //=============================================================================
1617
1618 // !!!!! Special hack to get all types of calls to specify the byte offset
1619 // from the start of the call to the point where the return address
1620 // will point.
1621
1622 int MachCallStaticJavaNode::ret_addr_offset()
1623 {
1624 // call should be a simple bl
1625 int off = 4;
1626 return off;
1627 }
1628
1629 int MachCallDynamicJavaNode::ret_addr_offset()
1630 {
1631 return 16; // movz, movk, movk, bl
1632 }
1633
1634 int MachCallRuntimeNode::ret_addr_offset() {
1635 // for generated stubs the call will be
1636 // bl(addr)
1637 // or with far branches
1638 // bl(trampoline_stub)
1639 // for real runtime callouts it will be six instructions
1640 // see aarch64_enc_java_to_runtime
1641 // adr(rscratch2, retaddr)
1642 // lea(rscratch1, RuntimeAddress(addr)
1643 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1644 // blr(rscratch1)
1645 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1646 if (cb) {
1647 return 1 * NativeInstruction::instruction_size;
1648 } else {
1649 return 6 * NativeInstruction::instruction_size;
1650 }
1651 }
1652
1653 //=============================================================================
1654
1655 #ifndef PRODUCT
1656 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1657 st->print("BREAKPOINT");
1658 }
1659 #endif
1660
1661 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1662 C2_MacroAssembler _masm(&cbuf);
1663 __ brk(0);
1664 }
1665
1666 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1667 return MachNode::size(ra_);
1668 }
1669
1670 //=============================================================================
1671
1672 #ifndef PRODUCT
1673 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1674 st->print("nop \t# %d bytes pad for loops and calls", _count);
1675 }
1676 #endif
1677
1678 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1679 C2_MacroAssembler _masm(&cbuf);
1680 for (int i = 0; i < _count; i++) {
1681 __ nop();
1682 }
1683 }
1684
1685 uint MachNopNode::size(PhaseRegAlloc*) const {
1686 return _count * NativeInstruction::instruction_size;
1687 }
1688
1689 //=============================================================================
1690 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1691
1692 int ConstantTable::calculate_table_base_offset() const {
1693 return 0; // absolute addressing, no offset
1694 }
1695
1696 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1697 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1698 ShouldNotReachHere();
1699 }
1700
1701 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1702 // Empty encoding
1703 }
1704
1705 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1706 return 0;
1707 }
1708
1709 #ifndef PRODUCT
1710 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1711 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1712 }
1713 #endif
1714
1715 #ifndef PRODUCT
1716 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1717 Compile* C = ra_->C;
1718
1719 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1720
1721 if (C->output()->need_stack_bang(framesize))
1722 st->print("# stack bang size=%d\n\t", framesize);
1723
1724 if (VM_Version::use_rop_protection()) {
1725 st->print("ldr zr, [lr]\n\t");
1726 st->print("pacia lr, rfp\n\t");
1727 }
1728 if (framesize < ((1 << 9) + 2 * wordSize)) {
1729 st->print("sub sp, sp, #%d\n\t", framesize);
1730 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1731 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1732 } else {
1733 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1734 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1735 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1736 st->print("sub sp, sp, rscratch1");
1737 }
1738 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1739 st->print("\n\t");
1740 st->print("ldr rscratch1, [guard]\n\t");
1741 st->print("dmb ishld\n\t");
1742 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1743 st->print("cmp rscratch1, rscratch2\n\t");
1744 st->print("b.eq skip");
1745 st->print("\n\t");
1746 st->print("blr #nmethod_entry_barrier_stub\n\t");
1747 st->print("b skip\n\t");
1748 st->print("guard: int\n\t");
1749 st->print("\n\t");
1750 st->print("skip:\n\t");
1751 }
1752 }
1753 #endif
1754
1755 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1756 Compile* C = ra_->C;
1757 C2_MacroAssembler _masm(&cbuf);
1758
1759 // n.b. frame size includes space for return pc and rfp
1760 const int framesize = C->output()->frame_size_in_bytes();
1761
1762 // insert a nop at the start of the prolog so we can patch in a
1763 // branch if we need to invalidate the method later
1764 __ nop();
1765
1766 if (C->clinit_barrier_on_entry()) {
1767 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1768
1769 Label L_skip_barrier;
1770
1771 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1772 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1773 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1774 __ bind(L_skip_barrier);
1775 }
1776
1777 if (C->max_vector_size() > 0) {
1778 __ reinitialize_ptrue();
1779 }
1780
1781 int bangsize = C->output()->bang_size_in_bytes();
1782 if (C->output()->need_stack_bang(bangsize))
1783 __ generate_stack_overflow_check(bangsize);
1784
1785 __ build_frame(framesize);
1786
1787 int max_monitors = C->method() != NULL ? C->max_monitors() : 0;
1788 if (UseFastLocking && max_monitors > 0) {
1789 C2CheckLockStackStub* stub = new (C->comp_arena()) C2CheckLockStackStub();
1790 C->output()->add_stub(stub);
1791 __ ldr(r9, Address(rthread, JavaThread::lock_stack_current_offset()));
1792 __ ldr(r10, Address(rthread, JavaThread::lock_stack_limit_offset()));
1793 __ add(r9, r9, max_monitors * oopSize);
1794 __ cmp(r9, r10);
1795 __ br(Assembler::GE, stub->entry());
1796 __ bind(stub->continuation());
1797 }
1798
1799 if (C->stub_function() == NULL) {
1800 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1801 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1802 // Dummy labels for just measuring the code size
1803 Label dummy_slow_path;
1804 Label dummy_continuation;
1805 Label dummy_guard;
1806 Label* slow_path = &dummy_slow_path;
1807 Label* continuation = &dummy_continuation;
1808 Label* guard = &dummy_guard;
1809 if (!Compile::current()->output()->in_scratch_emit_size()) {
1810 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1811 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1812 Compile::current()->output()->add_stub(stub);
1813 slow_path = &stub->entry();
1814 continuation = &stub->continuation();
1815 guard = &stub->guard();
1816 }
1817 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1818 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1819 }
1820 }
1821
1822 if (VerifyStackAtCalls) {
1823 Unimplemented();
1824 }
1825
1826 C->output()->set_frame_complete(cbuf.insts_size());
1827
1828 if (C->has_mach_constant_base_node()) {
1829 // NOTE: We set the table base offset here because users might be
1830 // emitted before MachConstantBaseNode.
1831 ConstantTable& constant_table = C->output()->constant_table();
1832 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1833 }
1834 }
1835
1836 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1837 {
1838 return MachNode::size(ra_); // too many variables; just compute it
1839 // the hard way
1840 }
1841
1842 int MachPrologNode::reloc() const
1843 {
1844 return 0;
1845 }
1846
1847 //=============================================================================
1848
1849 #ifndef PRODUCT
1850 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1851 Compile* C = ra_->C;
1852 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1853
1854 st->print("# pop frame %d\n\t",framesize);
1855
1856 if (framesize == 0) {
1857 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1858 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1859 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1860 st->print("add sp, sp, #%d\n\t", framesize);
1861 } else {
1862 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1863 st->print("add sp, sp, rscratch1\n\t");
1864 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1865 }
1866 if (VM_Version::use_rop_protection()) {
1867 st->print("autia lr, rfp\n\t");
1868 st->print("ldr zr, [lr]\n\t");
1869 }
1870
1871 if (do_polling() && C->is_method_compilation()) {
1872 st->print("# test polling word\n\t");
1873 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1874 st->print("cmp sp, rscratch1\n\t");
1875 st->print("bhi #slow_path");
1876 }
1877 }
1878 #endif
1879
1880 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1881 Compile* C = ra_->C;
1882 C2_MacroAssembler _masm(&cbuf);
1883 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1884
1885 __ remove_frame(framesize);
1886
1887 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1888 __ reserved_stack_check();
1889 }
1890
1891 if (do_polling() && C->is_method_compilation()) {
1892 Label dummy_label;
1893 Label* code_stub = &dummy_label;
1894 if (!C->output()->in_scratch_emit_size()) {
1895 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1896 C->output()->add_stub(stub);
1897 code_stub = &stub->entry();
1898 }
1899 __ relocate(relocInfo::poll_return_type);
1900 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1901 }
1902 }
1903
1904 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1905 // Variable size. Determine dynamically.
1906 return MachNode::size(ra_);
1907 }
1908
1909 int MachEpilogNode::reloc() const {
1910 // Return number of relocatable values contained in this instruction.
1911 return 1; // 1 for polling page.
1912 }
1913
1914 const Pipeline * MachEpilogNode::pipeline() const {
1915 return MachNode::pipeline_class();
1916 }
1917
1918 //=============================================================================
1919
1920 // Figure out which register class each belongs in: rc_int, rc_float or
1921 // rc_stack.
1922 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1923
1924 static enum RC rc_class(OptoReg::Name reg) {
1925
1926 if (reg == OptoReg::Bad) {
1927 return rc_bad;
1928 }
1929
1930 // we have 32 int registers * 2 halves
1931 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1932
1933 if (reg < slots_of_int_registers) {
1934 return rc_int;
1935 }
1936
1937 // we have 32 float register * 8 halves
1938 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1939 if (reg < slots_of_int_registers + slots_of_float_registers) {
1940 return rc_float;
1941 }
1942
1943 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1944 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1945 return rc_predicate;
1946 }
1947
1948 // Between predicate regs & stack is the flags.
1949 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1950
1951 return rc_stack;
1952 }
1953
1954 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1955 Compile* C = ra_->C;
1956
1957 // Get registers to move.
1958 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1959 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1960 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1961 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1962
1963 enum RC src_hi_rc = rc_class(src_hi);
1964 enum RC src_lo_rc = rc_class(src_lo);
1965 enum RC dst_hi_rc = rc_class(dst_hi);
1966 enum RC dst_lo_rc = rc_class(dst_lo);
1967
1968 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1969
1970 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1971 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1972 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1973 "expected aligned-adjacent pairs");
1974 }
1975
1976 if (src_lo == dst_lo && src_hi == dst_hi) {
1977 return 0; // Self copy, no move.
1978 }
1979
1980 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1981 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1982 int src_offset = ra_->reg2offset(src_lo);
1983 int dst_offset = ra_->reg2offset(dst_lo);
1984
1985 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1986 uint ireg = ideal_reg();
1987 if (ireg == Op_VecA && cbuf) {
1988 C2_MacroAssembler _masm(cbuf);
1989 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1990 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1991 // stack->stack
1992 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1993 sve_vector_reg_size_in_bytes);
1994 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1995 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1996 sve_vector_reg_size_in_bytes);
1997 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1998 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1999 sve_vector_reg_size_in_bytes);
2000 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2001 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2002 as_FloatRegister(Matcher::_regEncode[src_lo]),
2003 as_FloatRegister(Matcher::_regEncode[src_lo]));
2004 } else {
2005 ShouldNotReachHere();
2006 }
2007 } else if (cbuf) {
2008 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2009 C2_MacroAssembler _masm(cbuf);
2010 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2011 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2012 // stack->stack
2013 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2014 if (ireg == Op_VecD) {
2015 __ unspill(rscratch1, true, src_offset);
2016 __ spill(rscratch1, true, dst_offset);
2017 } else {
2018 __ spill_copy128(src_offset, dst_offset);
2019 }
2020 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2021 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2022 ireg == Op_VecD ? __ T8B : __ T16B,
2023 as_FloatRegister(Matcher::_regEncode[src_lo]));
2024 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2025 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2026 ireg == Op_VecD ? __ D : __ Q,
2027 ra_->reg2offset(dst_lo));
2028 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2029 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2030 ireg == Op_VecD ? __ D : __ Q,
2031 ra_->reg2offset(src_lo));
2032 } else {
2033 ShouldNotReachHere();
2034 }
2035 }
2036 } else if (cbuf) {
2037 C2_MacroAssembler _masm(cbuf);
2038 switch (src_lo_rc) {
2039 case rc_int:
2040 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2041 if (is64) {
2042 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2043 as_Register(Matcher::_regEncode[src_lo]));
2044 } else {
2045 C2_MacroAssembler _masm(cbuf);
2046 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2047 as_Register(Matcher::_regEncode[src_lo]));
2048 }
2049 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2050 if (is64) {
2051 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2052 as_Register(Matcher::_regEncode[src_lo]));
2053 } else {
2054 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2055 as_Register(Matcher::_regEncode[src_lo]));
2056 }
2057 } else { // gpr --> stack spill
2058 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2059 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2060 }
2061 break;
2062 case rc_float:
2063 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2064 if (is64) {
2065 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2066 as_FloatRegister(Matcher::_regEncode[src_lo]));
2067 } else {
2068 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2069 as_FloatRegister(Matcher::_regEncode[src_lo]));
2070 }
2071 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2072 if (is64) {
2073 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2074 as_FloatRegister(Matcher::_regEncode[src_lo]));
2075 } else {
2076 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2077 as_FloatRegister(Matcher::_regEncode[src_lo]));
2078 }
2079 } else { // fpr --> stack spill
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2082 is64 ? __ D : __ S, dst_offset);
2083 }
2084 break;
2085 case rc_stack:
2086 if (dst_lo_rc == rc_int) { // stack --> gpr load
2087 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2088 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2089 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2090 is64 ? __ D : __ S, src_offset);
2091 } else if (dst_lo_rc == rc_predicate) {
2092 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2093 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2094 } else { // stack --> stack copy
2095 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2096 if (ideal_reg() == Op_RegVectMask) {
2097 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2098 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2099 } else {
2100 __ unspill(rscratch1, is64, src_offset);
2101 __ spill(rscratch1, is64, dst_offset);
2102 }
2103 }
2104 break;
2105 case rc_predicate:
2106 if (dst_lo_rc == rc_predicate) {
2107 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2108 } else if (dst_lo_rc == rc_stack) {
2109 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 assert(false, "bad src and dst rc_class combination.");
2113 ShouldNotReachHere();
2114 }
2115 break;
2116 default:
2117 assert(false, "bad rc_class for spill");
2118 ShouldNotReachHere();
2119 }
2120 }
2121
2122 if (st) {
2123 st->print("spill ");
2124 if (src_lo_rc == rc_stack) {
2125 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2126 } else {
2127 st->print("%s -> ", Matcher::regName[src_lo]);
2128 }
2129 if (dst_lo_rc == rc_stack) {
2130 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2131 } else {
2132 st->print("%s", Matcher::regName[dst_lo]);
2133 }
2134 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2135 int vsize = 0;
2136 switch (ideal_reg()) {
2137 case Op_VecD:
2138 vsize = 64;
2139 break;
2140 case Op_VecX:
2141 vsize = 128;
2142 break;
2143 case Op_VecA:
2144 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2145 break;
2146 default:
2147 assert(false, "bad register type for spill");
2148 ShouldNotReachHere();
2149 }
2150 st->print("\t# vector spill size = %d", vsize);
2151 } else if (ideal_reg() == Op_RegVectMask) {
2152 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2153 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2154 st->print("\t# predicate spill size = %d", vsize);
2155 } else {
2156 st->print("\t# spill size = %d", is64 ? 64 : 32);
2157 }
2158 }
2159
2160 return 0;
2161
2162 }
2163
2164 #ifndef PRODUCT
2165 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2166 if (!ra_)
2167 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2168 else
2169 implementation(NULL, ra_, false, st);
2170 }
2171 #endif
2172
2173 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2174 implementation(&cbuf, ra_, false, NULL);
2175 }
2176
2177 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2178 return MachNode::size(ra_);
2179 }
2180
2181 //=============================================================================
2182
2183 #ifndef PRODUCT
2184 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2185 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2186 int reg = ra_->get_reg_first(this);
2187 st->print("add %s, rsp, #%d]\t# box lock",
2188 Matcher::regName[reg], offset);
2189 }
2190 #endif
2191
2192 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2193 C2_MacroAssembler _masm(&cbuf);
2194
2195 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2196 int reg = ra_->get_encode(this);
2197
2198 // This add will handle any 24-bit signed offset. 24 bits allows an
2199 // 8 megabyte stack frame.
2200 __ add(as_Register(reg), sp, offset);
2201 }
2202
2203 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2204 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206
2207 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2208 return NativeInstruction::instruction_size;
2209 } else {
2210 return 2 * NativeInstruction::instruction_size;
2211 }
2212 }
2213
2214 //=============================================================================
2215
2216 #ifndef PRODUCT
2217 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2218 {
2219 st->print_cr("# MachUEPNode");
2220 if (UseCompressedClassPointers) {
2221 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2222 if (CompressedKlassPointers::shift() != 0) {
2223 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2224 }
2225 } else {
2226 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2227 }
2228 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2229 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2230 }
2231 #endif
2232
2233 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2234 {
2235 // This is the unverified entry point.
2236 C2_MacroAssembler _masm(&cbuf);
2237
2238 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2239 Label skip;
2240 // TODO
2241 // can we avoid this skip and still use a reloc?
2242 __ br(Assembler::EQ, skip);
2243 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2244 __ bind(skip);
2245 }
2246
2247 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2248 {
2249 return MachNode::size(ra_);
2250 }
2251
2252 // REQUIRED EMIT CODE
2253
2254 //=============================================================================
2255
2256 // Emit exception handler code.
2257 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2258 {
2259 // mov rscratch1 #exception_blob_entry_point
2260 // br rscratch1
2261 // Note that the code buffer's insts_mark is always relative to insts.
2262 // That's why we must use the macroassembler to generate a handler.
2263 C2_MacroAssembler _masm(&cbuf);
2264 address base = __ start_a_stub(size_exception_handler());
2265 if (base == NULL) {
2266 ciEnv::current()->record_failure("CodeCache is full");
2267 return 0; // CodeBuffer::expand failed
2268 }
2269 int offset = __ offset();
2270 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2271 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2272 __ end_a_stub();
2273 return offset;
2274 }
2275
2276 // Emit deopt handler code.
2277 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2278 {
2279 // Note that the code buffer's insts_mark is always relative to insts.
2280 // That's why we must use the macroassembler to generate a handler.
2281 C2_MacroAssembler _masm(&cbuf);
2282 address base = __ start_a_stub(size_deopt_handler());
2283 if (base == NULL) {
2284 ciEnv::current()->record_failure("CodeCache is full");
2285 return 0; // CodeBuffer::expand failed
2286 }
2287 int offset = __ offset();
2288
2289 __ adr(lr, __ pc());
2290 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2291
2292 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2293 __ end_a_stub();
2294 return offset;
2295 }
2296
2297 // REQUIRED MATCHER CODE
2298
2299 //=============================================================================
2300
2301 const bool Matcher::match_rule_supported(int opcode) {
2302 if (!has_match_rule(opcode))
2303 return false;
2304
2305 bool ret_value = true;
2306 switch (opcode) {
2307 case Op_OnSpinWait:
2308 return VM_Version::supports_on_spin_wait();
2309 case Op_CacheWB:
2310 case Op_CacheWBPreSync:
2311 case Op_CacheWBPostSync:
2312 if (!VM_Version::supports_data_cache_line_flush()) {
2313 ret_value = false;
2314 }
2315 break;
2316 }
2317
2318 return ret_value; // Per default match rules are supported.
2319 }
2320
2321 const RegMask* Matcher::predicate_reg_mask(void) {
2322 return &_PR_REG_mask;
2323 }
2324
2325 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2326 return new TypeVectMask(elemTy, length);
2327 }
2328
2329 // Vector calling convention not yet implemented.
2330 const bool Matcher::supports_vector_calling_convention(void) {
2331 return false;
2332 }
2333
2334 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2335 Unimplemented();
2336 return OptoRegPair(0, 0);
2337 }
2338
2339 // Is this branch offset short enough that a short branch can be used?
2340 //
2341 // NOTE: If the platform does not provide any short branch variants, then
2342 // this method should return false for offset 0.
2343 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2344 // The passed offset is relative to address of the branch.
2345
2346 return (-32768 <= offset && offset < 32768);
2347 }
2348
2349 // Vector width in bytes.
2350 const int Matcher::vector_width_in_bytes(BasicType bt) {
2351 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2352 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2353 // Minimum 2 values in vector
2354 if (size < 2*type2aelembytes(bt)) size = 0;
2355 // But never < 4
2356 if (size < 4) size = 0;
2357 return size;
2358 }
2359
2360 // Limits on vector size (number of elements) loaded into vector.
2361 const int Matcher::max_vector_size(const BasicType bt) {
2362 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2363 }
2364
2365 const int Matcher::min_vector_size(const BasicType bt) {
2366 int max_size = max_vector_size(bt);
2367 // Limit the min vector size to 8 bytes.
2368 int size = 8 / type2aelembytes(bt);
2369 if (bt == T_BYTE) {
2370 // To support vector api shuffle/rearrange.
2371 size = 4;
2372 } else if (bt == T_BOOLEAN) {
2373 // To support vector api load/store mask.
2374 size = 2;
2375 }
2376 if (size < 2) size = 2;
2377 return MIN2(size, max_size);
2378 }
2379
2380 // Actual max scalable vector register length.
2381 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2382 return Matcher::max_vector_size(bt);
2383 }
2384
2385 // Vector ideal reg.
2386 const uint Matcher::vector_ideal_reg(int len) {
2387 if (UseSVE > 0 && 16 < len && len <= 256) {
2388 return Op_VecA;
2389 }
2390 switch(len) {
2391 // For 16-bit/32-bit mask vector, reuse VecD.
2392 case 2:
2393 case 4:
2394 case 8: return Op_VecD;
2395 case 16: return Op_VecX;
2396 }
2397 ShouldNotReachHere();
2398 return 0;
2399 }
2400
2401 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2402 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2403 switch (ideal_reg) {
2404 case Op_VecA: return new vecAOper();
2405 case Op_VecD: return new vecDOper();
2406 case Op_VecX: return new vecXOper();
2407 }
2408 ShouldNotReachHere();
2409 return NULL;
2410 }
2411
2412 bool Matcher::is_reg2reg_move(MachNode* m) {
2413 return false;
2414 }
2415
2416 bool Matcher::is_generic_vector(MachOper* opnd) {
2417 return opnd->opcode() == VREG;
2418 }
2419
2420 // Return whether or not this register is ever used as an argument.
2421 // This function is used on startup to build the trampoline stubs in
2422 // generateOptoStub. Registers not mentioned will be killed by the VM
2423 // call in the trampoline, and arguments in those registers not be
2424 // available to the callee.
2425 bool Matcher::can_be_java_arg(int reg)
2426 {
2427 return
2428 reg == R0_num || reg == R0_H_num ||
2429 reg == R1_num || reg == R1_H_num ||
2430 reg == R2_num || reg == R2_H_num ||
2431 reg == R3_num || reg == R3_H_num ||
2432 reg == R4_num || reg == R4_H_num ||
2433 reg == R5_num || reg == R5_H_num ||
2434 reg == R6_num || reg == R6_H_num ||
2435 reg == R7_num || reg == R7_H_num ||
2436 reg == V0_num || reg == V0_H_num ||
2437 reg == V1_num || reg == V1_H_num ||
2438 reg == V2_num || reg == V2_H_num ||
2439 reg == V3_num || reg == V3_H_num ||
2440 reg == V4_num || reg == V4_H_num ||
2441 reg == V5_num || reg == V5_H_num ||
2442 reg == V6_num || reg == V6_H_num ||
2443 reg == V7_num || reg == V7_H_num;
2444 }
2445
2446 bool Matcher::is_spillable_arg(int reg)
2447 {
2448 return can_be_java_arg(reg);
2449 }
2450
2451 uint Matcher::int_pressure_limit()
2452 {
2453 // JDK-8183543: When taking the number of available registers as int
2454 // register pressure threshold, the jtreg test:
2455 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2456 // failed due to C2 compilation failure with
2457 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2458 //
2459 // A derived pointer is live at CallNode and then is flagged by RA
2460 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2461 // derived pointers and lastly fail to spill after reaching maximum
2462 // number of iterations. Lowering the default pressure threshold to
2463 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2464 // a high register pressure area of the code so that split_DEF can
2465 // generate DefinitionSpillCopy for the derived pointer.
2466 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2467 if (!PreserveFramePointer) {
2468 // When PreserveFramePointer is off, frame pointer is allocatable,
2469 // but different from other SOC registers, it is excluded from
2470 // fatproj's mask because its save type is No-Save. Decrease 1 to
2471 // ensure high pressure at fatproj when PreserveFramePointer is off.
2472 // See check_pressure_at_fatproj().
2473 default_int_pressure_threshold--;
2474 }
2475 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2476 }
2477
2478 uint Matcher::float_pressure_limit()
2479 {
2480 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2481 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2482 }
2483
2484 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2485 return false;
2486 }
2487
2488 RegMask Matcher::divI_proj_mask() {
2489 ShouldNotReachHere();
2490 return RegMask();
2491 }
2492
2493 // Register for MODI projection of divmodI.
2494 RegMask Matcher::modI_proj_mask() {
2495 ShouldNotReachHere();
2496 return RegMask();
2497 }
2498
2499 // Register for DIVL projection of divmodL.
2500 RegMask Matcher::divL_proj_mask() {
2501 ShouldNotReachHere();
2502 return RegMask();
2503 }
2504
2505 // Register for MODL projection of divmodL.
2506 RegMask Matcher::modL_proj_mask() {
2507 ShouldNotReachHere();
2508 return RegMask();
2509 }
2510
2511 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2512 return FP_REG_mask();
2513 }
2514
2515 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2516 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2517 Node* u = addp->fast_out(i);
2518 if (u->is_LoadStore()) {
2519 // On AArch64, LoadStoreNodes (i.e. compare and swap
2520 // instructions) only take register indirect as an operand, so
2521 // any attempt to use an AddPNode as an input to a LoadStoreNode
2522 // must fail.
2523 return false;
2524 }
2525 if (u->is_Mem()) {
2526 int opsize = u->as_Mem()->memory_size();
2527 assert(opsize > 0, "unexpected memory operand size");
2528 if (u->as_Mem()->memory_size() != (1<<shift)) {
2529 return false;
2530 }
2531 }
2532 }
2533 return true;
2534 }
2535
2536 // Binary src (Replicate con)
2537 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2538 if (n == NULL || m == NULL) {
2539 return false;
2540 }
2541
2542 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2543 return false;
2544 }
2545
2546 Node* imm_node = m->in(1);
2547 if (!imm_node->is_Con()) {
2548 return false;
2549 }
2550
2551 const Type* t = imm_node->bottom_type();
2552 if (!(t->isa_int() || t->isa_long())) {
2553 return false;
2554 }
2555
2556 switch (n->Opcode()) {
2557 case Op_AndV:
2558 case Op_OrV:
2559 case Op_XorV: {
2560 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2561 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2562 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2563 }
2564 case Op_AddVB:
2565 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2566 case Op_AddVS:
2567 case Op_AddVI:
2568 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2569 case Op_AddVL:
2570 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2571 default:
2572 return false;
2573 }
2574 }
2575
2576 // (XorV src (Replicate m1))
2577 // (XorVMask src (MaskAll m1))
2578 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2579 if (n != NULL && m != NULL) {
2580 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2581 VectorNode::is_all_ones_vector(m);
2582 }
2583 return false;
2584 }
2585
2586 // Should the matcher clone input 'm' of node 'n'?
2587 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2588 if (is_vshift_con_pattern(n, m) ||
2589 is_vector_bitwise_not_pattern(n, m) ||
2590 is_valid_sve_arith_imm_pattern(n, m)) {
2591 mstack.push(m, Visit);
2592 return true;
2593 }
2594 return false;
2595 }
2596
2597 // Should the Matcher clone shifts on addressing modes, expecting them
2598 // to be subsumed into complex addressing expressions or compute them
2599 // into registers?
2600 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2601 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2602 return true;
2603 }
2604
2605 Node *off = m->in(AddPNode::Offset);
2606 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2607 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2608 // Are there other uses besides address expressions?
2609 !is_visited(off)) {
2610 address_visited.set(off->_idx); // Flag as address_visited
2611 mstack.push(off->in(2), Visit);
2612 Node *conv = off->in(1);
2613 if (conv->Opcode() == Op_ConvI2L &&
2614 // Are there other uses besides address expressions?
2615 !is_visited(conv)) {
2616 address_visited.set(conv->_idx); // Flag as address_visited
2617 mstack.push(conv->in(1), Pre_Visit);
2618 } else {
2619 mstack.push(conv, Pre_Visit);
2620 }
2621 address_visited.test_set(m->_idx); // Flag as address_visited
2622 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2623 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2624 return true;
2625 } else if (off->Opcode() == Op_ConvI2L &&
2626 // Are there other uses besides address expressions?
2627 !is_visited(off)) {
2628 address_visited.test_set(m->_idx); // Flag as address_visited
2629 address_visited.set(off->_idx); // Flag as address_visited
2630 mstack.push(off->in(1), Pre_Visit);
2631 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2632 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2633 return true;
2634 }
2635 return false;
2636 }
2637
2638 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2639 C2_MacroAssembler _masm(&cbuf); \
2640 { \
2641 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2642 guarantee(DISP == 0, "mode not permitted for volatile"); \
2643 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2644 __ INSN(REG, as_Register(BASE)); \
2645 }
2646
2647
2648 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2649 {
2650 Address::extend scale;
2651
2652 // Hooboy, this is fugly. We need a way to communicate to the
2653 // encoder that the index needs to be sign extended, so we have to
2654 // enumerate all the cases.
2655 switch (opcode) {
2656 case INDINDEXSCALEDI2L:
2657 case INDINDEXSCALEDI2LN:
2658 case INDINDEXI2L:
2659 case INDINDEXI2LN:
2660 scale = Address::sxtw(size);
2661 break;
2662 default:
2663 scale = Address::lsl(size);
2664 }
2665
2666 if (index == -1) {
2667 return Address(base, disp);
2668 } else {
2669 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2670 return Address(base, as_Register(index), scale);
2671 }
2672 }
2673
2674
2675 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2676 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2677 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2678 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2679 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2680
2681 // Used for all non-volatile memory accesses. The use of
2682 // $mem->opcode() to discover whether this pattern uses sign-extended
2683 // offsets is something of a kludge.
2684 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2685 Register reg, int opcode,
2686 Register base, int index, int scale, int disp,
2687 int size_in_memory)
2688 {
2689 Address addr = mem2address(opcode, base, index, scale, disp);
2690 if (addr.getMode() == Address::base_plus_offset) {
2691 /* If we get an out-of-range offset it is a bug in the compiler,
2692 so we assert here. */
2693 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2694 "c2 compiler bug");
2695 /* Fix up any out-of-range offsets. */
2696 assert_different_registers(rscratch1, base);
2697 assert_different_registers(rscratch1, reg);
2698 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2699 }
2700 (masm.*insn)(reg, addr);
2701 }
2702
2703 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2704 FloatRegister reg, int opcode,
2705 Register base, int index, int size, int disp,
2706 int size_in_memory)
2707 {
2708 Address::extend scale;
2709
2710 switch (opcode) {
2711 case INDINDEXSCALEDI2L:
2712 case INDINDEXSCALEDI2LN:
2713 scale = Address::sxtw(size);
2714 break;
2715 default:
2716 scale = Address::lsl(size);
2717 }
2718
2719 if (index == -1) {
2720 /* If we get an out-of-range offset it is a bug in the compiler,
2721 so we assert here. */
2722 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2723 /* Fix up any out-of-range offsets. */
2724 assert_different_registers(rscratch1, base);
2725 Address addr = Address(base, disp);
2726 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2727 (masm.*insn)(reg, addr);
2728 } else {
2729 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2730 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2731 }
2732 }
2733
2734 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2735 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2736 int opcode, Register base, int index, int size, int disp)
2737 {
2738 if (index == -1) {
2739 (masm.*insn)(reg, T, Address(base, disp));
2740 } else {
2741 assert(disp == 0, "unsupported address mode");
2742 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2743 }
2744 }
2745
2746 %}
2747
2748
2749
2750 //----------ENCODING BLOCK-----------------------------------------------------
2751 // This block specifies the encoding classes used by the compiler to
2752 // output byte streams. Encoding classes are parameterized macros
2753 // used by Machine Instruction Nodes in order to generate the bit
2754 // encoding of the instruction. Operands specify their base encoding
2755 // interface with the interface keyword. There are currently
2756 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2757 // COND_INTER. REG_INTER causes an operand to generate a function
2758 // which returns its register number when queried. CONST_INTER causes
2759 // an operand to generate a function which returns the value of the
2760 // constant when queried. MEMORY_INTER causes an operand to generate
2761 // four functions which return the Base Register, the Index Register,
2762 // the Scale Value, and the Offset Value of the operand when queried.
2763 // COND_INTER causes an operand to generate six functions which return
2764 // the encoding code (ie - encoding bits for the instruction)
2765 // associated with each basic boolean condition for a conditional
2766 // instruction.
2767 //
2768 // Instructions specify two basic values for encoding. Again, a
2769 // function is available to check if the constant displacement is an
2770 // oop. They use the ins_encode keyword to specify their encoding
2771 // classes (which must be a sequence of enc_class names, and their
2772 // parameters, specified in the encoding block), and they use the
2773 // opcode keyword to specify, in order, their primary, secondary, and
2774 // tertiary opcode. Only the opcode sections which a particular
2775 // instruction needs for encoding need to be specified.
2776 encode %{
2777 // Build emit functions for each basic byte or larger field in the
2778 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2779 // from C++ code in the enc_class source block. Emit functions will
2780 // live in the main source block for now. In future, we can
2781 // generalize this by adding a syntax that specifies the sizes of
2782 // fields in an order, so that the adlc can build the emit functions
2783 // automagically
2784
2785 // catch all for unimplemented encodings
2786 enc_class enc_unimplemented %{
2787 C2_MacroAssembler _masm(&cbuf);
2788 __ unimplemented("C2 catch all");
2789 %}
2790
2791 // BEGIN Non-volatile memory access
2792
2793 // This encoding class is generated automatically from ad_encode.m4.
2794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2795 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2796 Register dst_reg = as_Register($dst$$reg);
2797 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2798 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2799 %}
2800
2801 // This encoding class is generated automatically from ad_encode.m4.
2802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2803 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2804 Register dst_reg = as_Register($dst$$reg);
2805 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2806 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2807 %}
2808
2809 // This encoding class is generated automatically from ad_encode.m4.
2810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2811 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2812 Register dst_reg = as_Register($dst$$reg);
2813 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2814 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2815 %}
2816
2817 // This encoding class is generated automatically from ad_encode.m4.
2818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2819 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2820 Register dst_reg = as_Register($dst$$reg);
2821 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2822 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2823 %}
2824
2825 // This encoding class is generated automatically from ad_encode.m4.
2826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2827 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2828 Register dst_reg = as_Register($dst$$reg);
2829 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2830 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2831 %}
2832
2833 // This encoding class is generated automatically from ad_encode.m4.
2834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2835 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2836 Register dst_reg = as_Register($dst$$reg);
2837 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2838 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2839 %}
2840
2841 // This encoding class is generated automatically from ad_encode.m4.
2842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2843 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2844 Register dst_reg = as_Register($dst$$reg);
2845 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2847 %}
2848
2849 // This encoding class is generated automatically from ad_encode.m4.
2850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2851 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2852 Register dst_reg = as_Register($dst$$reg);
2853 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2854 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2855 %}
2856
2857 // This encoding class is generated automatically from ad_encode.m4.
2858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2859 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2860 Register dst_reg = as_Register($dst$$reg);
2861 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2862 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2863 %}
2864
2865 // This encoding class is generated automatically from ad_encode.m4.
2866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2867 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2868 Register dst_reg = as_Register($dst$$reg);
2869 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2870 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2871 %}
2872
2873 // This encoding class is generated automatically from ad_encode.m4.
2874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2875 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2876 Register dst_reg = as_Register($dst$$reg);
2877 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2878 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2879 %}
2880
2881 // This encoding class is generated automatically from ad_encode.m4.
2882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2883 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2884 Register dst_reg = as_Register($dst$$reg);
2885 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2886 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2887 %}
2888
2889 // This encoding class is generated automatically from ad_encode.m4.
2890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2891 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2892 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2893 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2894 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2895 %}
2896
2897 // This encoding class is generated automatically from ad_encode.m4.
2898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2899 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2900 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2901 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2902 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2903 %}
2904
2905 // This encoding class is generated automatically from ad_encode.m4.
2906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2907 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2908 Register src_reg = as_Register($src$$reg);
2909 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2910 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2911 %}
2912
2913 // This encoding class is generated automatically from ad_encode.m4.
2914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2915 enc_class aarch64_enc_strb0(memory1 mem) %{
2916 C2_MacroAssembler _masm(&cbuf);
2917 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2918 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2919 %}
2920
2921 // This encoding class is generated automatically from ad_encode.m4.
2922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2923 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2924 Register src_reg = as_Register($src$$reg);
2925 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2926 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2927 %}
2928
2929 // This encoding class is generated automatically from ad_encode.m4.
2930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2931 enc_class aarch64_enc_strh0(memory2 mem) %{
2932 C2_MacroAssembler _masm(&cbuf);
2933 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2934 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2935 %}
2936
2937 // This encoding class is generated automatically from ad_encode.m4.
2938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2939 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2940 Register src_reg = as_Register($src$$reg);
2941 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strw0(memory4 mem) %{
2948 C2_MacroAssembler _masm(&cbuf);
2949 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2956 Register src_reg = as_Register($src$$reg);
2957 // we sometimes get asked to store the stack pointer into the
2958 // current thread -- we cannot do that directly on AArch64
2959 if (src_reg == r31_sp) {
2960 C2_MacroAssembler _masm(&cbuf);
2961 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2962 __ mov(rscratch2, sp);
2963 src_reg = rscratch2;
2964 }
2965 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2966 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2967 %}
2968
2969 // This encoding class is generated automatically from ad_encode.m4.
2970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2971 enc_class aarch64_enc_str0(memory8 mem) %{
2972 C2_MacroAssembler _masm(&cbuf);
2973 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2980 FloatRegister src_reg = as_FloatRegister($src$$reg);
2981 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2988 FloatRegister src_reg = as_FloatRegister($src$$reg);
2989 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2996 C2_MacroAssembler _masm(&cbuf);
2997 __ membar(Assembler::StoreStore);
2998 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3000 %}
3001
3002 // END Non-volatile memory access
3003
3004 // Vector loads and stores
3005 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3006 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3007 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3008 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3009 %}
3010
3011 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3012 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3013 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3014 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3015 %}
3016
3017 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3018 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3019 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3020 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3021 %}
3022
3023 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3024 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3025 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3026 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3027 %}
3028
3029 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3030 FloatRegister src_reg = as_FloatRegister($src$$reg);
3031 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3032 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3033 %}
3034
3035 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3036 FloatRegister src_reg = as_FloatRegister($src$$reg);
3037 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3038 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3039 %}
3040
3041 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3042 FloatRegister src_reg = as_FloatRegister($src$$reg);
3043 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3044 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3045 %}
3046
3047 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3048 FloatRegister src_reg = as_FloatRegister($src$$reg);
3049 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3050 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3051 %}
3052
3053 // volatile loads and stores
3054
3055 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3056 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3057 rscratch1, stlrb);
3058 %}
3059
3060 enc_class aarch64_enc_stlrb0(memory mem) %{
3061 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3062 rscratch1, stlrb);
3063 %}
3064
3065 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3066 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3067 rscratch1, stlrh);
3068 %}
3069
3070 enc_class aarch64_enc_stlrh0(memory mem) %{
3071 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3072 rscratch1, stlrh);
3073 %}
3074
3075 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3076 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3077 rscratch1, stlrw);
3078 %}
3079
3080 enc_class aarch64_enc_stlrw0(memory mem) %{
3081 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3082 rscratch1, stlrw);
3083 %}
3084
3085 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3086 Register dst_reg = as_Register($dst$$reg);
3087 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3088 rscratch1, ldarb);
3089 __ sxtbw(dst_reg, dst_reg);
3090 %}
3091
3092 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3093 Register dst_reg = as_Register($dst$$reg);
3094 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3095 rscratch1, ldarb);
3096 __ sxtb(dst_reg, dst_reg);
3097 %}
3098
3099 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3100 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, ldarb);
3102 %}
3103
3104 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3105 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3106 rscratch1, ldarb);
3107 %}
3108
3109 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3110 Register dst_reg = as_Register($dst$$reg);
3111 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3112 rscratch1, ldarh);
3113 __ sxthw(dst_reg, dst_reg);
3114 %}
3115
3116 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3117 Register dst_reg = as_Register($dst$$reg);
3118 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3119 rscratch1, ldarh);
3120 __ sxth(dst_reg, dst_reg);
3121 %}
3122
3123 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3124 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, ldarh);
3126 %}
3127
3128 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3129 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, ldarh);
3131 %}
3132
3133 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3134 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3135 rscratch1, ldarw);
3136 %}
3137
3138 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3139 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, ldarw);
3141 %}
3142
3143 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3144 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, ldar);
3146 %}
3147
3148 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3149 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, ldarw);
3151 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3152 %}
3153
3154 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3155 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, ldar);
3157 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3158 %}
3159
3160 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3161 Register src_reg = as_Register($src$$reg);
3162 // we sometimes get asked to store the stack pointer into the
3163 // current thread -- we cannot do that directly on AArch64
3164 if (src_reg == r31_sp) {
3165 C2_MacroAssembler _masm(&cbuf);
3166 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3167 __ mov(rscratch2, sp);
3168 src_reg = rscratch2;
3169 }
3170 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, stlr);
3172 %}
3173
3174 enc_class aarch64_enc_stlr0(memory mem) %{
3175 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, stlr);
3177 %}
3178
3179 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3180 {
3181 C2_MacroAssembler _masm(&cbuf);
3182 FloatRegister src_reg = as_FloatRegister($src$$reg);
3183 __ fmovs(rscratch2, src_reg);
3184 }
3185 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, stlrw);
3187 %}
3188
3189 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3190 {
3191 C2_MacroAssembler _masm(&cbuf);
3192 FloatRegister src_reg = as_FloatRegister($src$$reg);
3193 __ fmovd(rscratch2, src_reg);
3194 }
3195 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3196 rscratch1, stlr);
3197 %}
3198
3199 // synchronized read/update encodings
3200
3201 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3202 C2_MacroAssembler _masm(&cbuf);
3203 Register dst_reg = as_Register($dst$$reg);
3204 Register base = as_Register($mem$$base);
3205 int index = $mem$$index;
3206 int scale = $mem$$scale;
3207 int disp = $mem$$disp;
3208 if (index == -1) {
3209 if (disp != 0) {
3210 __ lea(rscratch1, Address(base, disp));
3211 __ ldaxr(dst_reg, rscratch1);
3212 } else {
3213 // TODO
3214 // should we ever get anything other than this case?
3215 __ ldaxr(dst_reg, base);
3216 }
3217 } else {
3218 Register index_reg = as_Register(index);
3219 if (disp == 0) {
3220 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3221 __ ldaxr(dst_reg, rscratch1);
3222 } else {
3223 __ lea(rscratch1, Address(base, disp));
3224 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3225 __ ldaxr(dst_reg, rscratch1);
3226 }
3227 }
3228 %}
3229
3230 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3231 C2_MacroAssembler _masm(&cbuf);
3232 Register src_reg = as_Register($src$$reg);
3233 Register base = as_Register($mem$$base);
3234 int index = $mem$$index;
3235 int scale = $mem$$scale;
3236 int disp = $mem$$disp;
3237 if (index == -1) {
3238 if (disp != 0) {
3239 __ lea(rscratch2, Address(base, disp));
3240 __ stlxr(rscratch1, src_reg, rscratch2);
3241 } else {
3242 // TODO
3243 // should we ever get anything other than this case?
3244 __ stlxr(rscratch1, src_reg, base);
3245 }
3246 } else {
3247 Register index_reg = as_Register(index);
3248 if (disp == 0) {
3249 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3250 __ stlxr(rscratch1, src_reg, rscratch2);
3251 } else {
3252 __ lea(rscratch2, Address(base, disp));
3253 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3254 __ stlxr(rscratch1, src_reg, rscratch2);
3255 }
3256 }
3257 __ cmpw(rscratch1, zr);
3258 %}
3259
3260 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3261 C2_MacroAssembler _masm(&cbuf);
3262 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3263 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3264 Assembler::xword, /*acquire*/ false, /*release*/ true,
3265 /*weak*/ false, noreg);
3266 %}
3267
3268 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3269 C2_MacroAssembler _masm(&cbuf);
3270 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3271 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3272 Assembler::word, /*acquire*/ false, /*release*/ true,
3273 /*weak*/ false, noreg);
3274 %}
3275
3276 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3277 C2_MacroAssembler _masm(&cbuf);
3278 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3279 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3280 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3281 /*weak*/ false, noreg);
3282 %}
3283
3284 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3285 C2_MacroAssembler _masm(&cbuf);
3286 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3287 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3288 Assembler::byte, /*acquire*/ false, /*release*/ true,
3289 /*weak*/ false, noreg);
3290 %}
3291
3292
3293 // The only difference between aarch64_enc_cmpxchg and
3294 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3295 // CompareAndSwap sequence to serve as a barrier on acquiring a
3296 // lock.
3297 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3298 C2_MacroAssembler _masm(&cbuf);
3299 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3300 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3301 Assembler::xword, /*acquire*/ true, /*release*/ true,
3302 /*weak*/ false, noreg);
3303 %}
3304
3305 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3306 C2_MacroAssembler _masm(&cbuf);
3307 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3308 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3309 Assembler::word, /*acquire*/ true, /*release*/ true,
3310 /*weak*/ false, noreg);
3311 %}
3312
3313 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3314 C2_MacroAssembler _masm(&cbuf);
3315 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3316 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3317 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3318 /*weak*/ false, noreg);
3319 %}
3320
3321 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3322 C2_MacroAssembler _masm(&cbuf);
3323 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3324 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3325 Assembler::byte, /*acquire*/ true, /*release*/ true,
3326 /*weak*/ false, noreg);
3327 %}
3328
3329 // auxiliary used for CompareAndSwapX to set result register
3330 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3331 C2_MacroAssembler _masm(&cbuf);
3332 Register res_reg = as_Register($res$$reg);
3333 __ cset(res_reg, Assembler::EQ);
3334 %}
3335
3336 // prefetch encodings
3337
3338 enc_class aarch64_enc_prefetchw(memory mem) %{
3339 C2_MacroAssembler _masm(&cbuf);
3340 Register base = as_Register($mem$$base);
3341 int index = $mem$$index;
3342 int scale = $mem$$scale;
3343 int disp = $mem$$disp;
3344 if (index == -1) {
3345 __ prfm(Address(base, disp), PSTL1KEEP);
3346 } else {
3347 Register index_reg = as_Register(index);
3348 if (disp == 0) {
3349 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3350 } else {
3351 __ lea(rscratch1, Address(base, disp));
3352 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3353 }
3354 }
3355 %}
3356
3357 /// mov envcodings
3358
3359 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3360 C2_MacroAssembler _masm(&cbuf);
3361 uint32_t con = (uint32_t)$src$$constant;
3362 Register dst_reg = as_Register($dst$$reg);
3363 if (con == 0) {
3364 __ movw(dst_reg, zr);
3365 } else {
3366 __ movw(dst_reg, con);
3367 }
3368 %}
3369
3370 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3371 C2_MacroAssembler _masm(&cbuf);
3372 Register dst_reg = as_Register($dst$$reg);
3373 uint64_t con = (uint64_t)$src$$constant;
3374 if (con == 0) {
3375 __ mov(dst_reg, zr);
3376 } else {
3377 __ mov(dst_reg, con);
3378 }
3379 %}
3380
3381 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3382 C2_MacroAssembler _masm(&cbuf);
3383 Register dst_reg = as_Register($dst$$reg);
3384 address con = (address)$src$$constant;
3385 if (con == NULL || con == (address)1) {
3386 ShouldNotReachHere();
3387 } else {
3388 relocInfo::relocType rtype = $src->constant_reloc();
3389 if (rtype == relocInfo::oop_type) {
3390 __ movoop(dst_reg, (jobject)con);
3391 } else if (rtype == relocInfo::metadata_type) {
3392 __ mov_metadata(dst_reg, (Metadata*)con);
3393 } else {
3394 assert(rtype == relocInfo::none, "unexpected reloc type");
3395 if (! __ is_valid_AArch64_address(con) ||
3396 con < (address)(uintptr_t)os::vm_page_size()) {
3397 __ mov(dst_reg, con);
3398 } else {
3399 uint64_t offset;
3400 __ adrp(dst_reg, con, offset);
3401 __ add(dst_reg, dst_reg, offset);
3402 }
3403 }
3404 }
3405 %}
3406
3407 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3408 C2_MacroAssembler _masm(&cbuf);
3409 Register dst_reg = as_Register($dst$$reg);
3410 __ mov(dst_reg, zr);
3411 %}
3412
3413 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3414 C2_MacroAssembler _masm(&cbuf);
3415 Register dst_reg = as_Register($dst$$reg);
3416 __ mov(dst_reg, (uint64_t)1);
3417 %}
3418
3419 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3420 C2_MacroAssembler _masm(&cbuf);
3421 __ load_byte_map_base($dst$$Register);
3422 %}
3423
3424 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3425 C2_MacroAssembler _masm(&cbuf);
3426 Register dst_reg = as_Register($dst$$reg);
3427 address con = (address)$src$$constant;
3428 if (con == NULL) {
3429 ShouldNotReachHere();
3430 } else {
3431 relocInfo::relocType rtype = $src->constant_reloc();
3432 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3433 __ set_narrow_oop(dst_reg, (jobject)con);
3434 }
3435 %}
3436
3437 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3438 C2_MacroAssembler _masm(&cbuf);
3439 Register dst_reg = as_Register($dst$$reg);
3440 __ mov(dst_reg, zr);
3441 %}
3442
3443 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3444 C2_MacroAssembler _masm(&cbuf);
3445 Register dst_reg = as_Register($dst$$reg);
3446 address con = (address)$src$$constant;
3447 if (con == NULL) {
3448 ShouldNotReachHere();
3449 } else {
3450 relocInfo::relocType rtype = $src->constant_reloc();
3451 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3452 __ set_narrow_klass(dst_reg, (Klass *)con);
3453 }
3454 %}
3455
3456 // arithmetic encodings
3457
3458 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3459 C2_MacroAssembler _masm(&cbuf);
3460 Register dst_reg = as_Register($dst$$reg);
3461 Register src_reg = as_Register($src1$$reg);
3462 int32_t con = (int32_t)$src2$$constant;
3463 // add has primary == 0, subtract has primary == 1
3464 if ($primary) { con = -con; }
3465 if (con < 0) {
3466 __ subw(dst_reg, src_reg, -con);
3467 } else {
3468 __ addw(dst_reg, src_reg, con);
3469 }
3470 %}
3471
3472 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3473 C2_MacroAssembler _masm(&cbuf);
3474 Register dst_reg = as_Register($dst$$reg);
3475 Register src_reg = as_Register($src1$$reg);
3476 int32_t con = (int32_t)$src2$$constant;
3477 // add has primary == 0, subtract has primary == 1
3478 if ($primary) { con = -con; }
3479 if (con < 0) {
3480 __ sub(dst_reg, src_reg, -con);
3481 } else {
3482 __ add(dst_reg, src_reg, con);
3483 }
3484 %}
3485
3486 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3487 C2_MacroAssembler _masm(&cbuf);
3488 Register dst_reg = as_Register($dst$$reg);
3489 Register src1_reg = as_Register($src1$$reg);
3490 Register src2_reg = as_Register($src2$$reg);
3491 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3492 %}
3493
3494 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3495 C2_MacroAssembler _masm(&cbuf);
3496 Register dst_reg = as_Register($dst$$reg);
3497 Register src1_reg = as_Register($src1$$reg);
3498 Register src2_reg = as_Register($src2$$reg);
3499 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3500 %}
3501
3502 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3503 C2_MacroAssembler _masm(&cbuf);
3504 Register dst_reg = as_Register($dst$$reg);
3505 Register src1_reg = as_Register($src1$$reg);
3506 Register src2_reg = as_Register($src2$$reg);
3507 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3508 %}
3509
3510 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3511 C2_MacroAssembler _masm(&cbuf);
3512 Register dst_reg = as_Register($dst$$reg);
3513 Register src1_reg = as_Register($src1$$reg);
3514 Register src2_reg = as_Register($src2$$reg);
3515 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3516 %}
3517
3518 // compare instruction encodings
3519
3520 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3521 C2_MacroAssembler _masm(&cbuf);
3522 Register reg1 = as_Register($src1$$reg);
3523 Register reg2 = as_Register($src2$$reg);
3524 __ cmpw(reg1, reg2);
3525 %}
3526
3527 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3528 C2_MacroAssembler _masm(&cbuf);
3529 Register reg = as_Register($src1$$reg);
3530 int32_t val = $src2$$constant;
3531 if (val >= 0) {
3532 __ subsw(zr, reg, val);
3533 } else {
3534 __ addsw(zr, reg, -val);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3539 C2_MacroAssembler _masm(&cbuf);
3540 Register reg1 = as_Register($src1$$reg);
3541 uint32_t val = (uint32_t)$src2$$constant;
3542 __ movw(rscratch1, val);
3543 __ cmpw(reg1, rscratch1);
3544 %}
3545
3546 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3547 C2_MacroAssembler _masm(&cbuf);
3548 Register reg1 = as_Register($src1$$reg);
3549 Register reg2 = as_Register($src2$$reg);
3550 __ cmp(reg1, reg2);
3551 %}
3552
3553 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3554 C2_MacroAssembler _masm(&cbuf);
3555 Register reg = as_Register($src1$$reg);
3556 int64_t val = $src2$$constant;
3557 if (val >= 0) {
3558 __ subs(zr, reg, val);
3559 } else if (val != -val) {
3560 __ adds(zr, reg, -val);
3561 } else {
3562 // aargh, Long.MIN_VALUE is a special case
3563 __ orr(rscratch1, zr, (uint64_t)val);
3564 __ subs(zr, reg, rscratch1);
3565 }
3566 %}
3567
3568 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3569 C2_MacroAssembler _masm(&cbuf);
3570 Register reg1 = as_Register($src1$$reg);
3571 uint64_t val = (uint64_t)$src2$$constant;
3572 __ mov(rscratch1, val);
3573 __ cmp(reg1, rscratch1);
3574 %}
3575
3576 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3577 C2_MacroAssembler _masm(&cbuf);
3578 Register reg1 = as_Register($src1$$reg);
3579 Register reg2 = as_Register($src2$$reg);
3580 __ cmp(reg1, reg2);
3581 %}
3582
3583 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3584 C2_MacroAssembler _masm(&cbuf);
3585 Register reg1 = as_Register($src1$$reg);
3586 Register reg2 = as_Register($src2$$reg);
3587 __ cmpw(reg1, reg2);
3588 %}
3589
3590 enc_class aarch64_enc_testp(iRegP src) %{
3591 C2_MacroAssembler _masm(&cbuf);
3592 Register reg = as_Register($src$$reg);
3593 __ cmp(reg, zr);
3594 %}
3595
3596 enc_class aarch64_enc_testn(iRegN src) %{
3597 C2_MacroAssembler _masm(&cbuf);
3598 Register reg = as_Register($src$$reg);
3599 __ cmpw(reg, zr);
3600 %}
3601
3602 enc_class aarch64_enc_b(label lbl) %{
3603 C2_MacroAssembler _masm(&cbuf);
3604 Label *L = $lbl$$label;
3605 __ b(*L);
3606 %}
3607
3608 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3609 C2_MacroAssembler _masm(&cbuf);
3610 Label *L = $lbl$$label;
3611 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3612 %}
3613
3614 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3615 C2_MacroAssembler _masm(&cbuf);
3616 Label *L = $lbl$$label;
3617 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3618 %}
3619
3620 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3621 %{
3622 Register sub_reg = as_Register($sub$$reg);
3623 Register super_reg = as_Register($super$$reg);
3624 Register temp_reg = as_Register($temp$$reg);
3625 Register result_reg = as_Register($result$$reg);
3626
3627 Label miss;
3628 C2_MacroAssembler _masm(&cbuf);
3629 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3630 NULL, &miss,
3631 /*set_cond_codes:*/ true);
3632 if ($primary) {
3633 __ mov(result_reg, zr);
3634 }
3635 __ bind(miss);
3636 %}
3637
3638 enc_class aarch64_enc_java_static_call(method meth) %{
3639 C2_MacroAssembler _masm(&cbuf);
3640
3641 address addr = (address)$meth$$method;
3642 address call;
3643 if (!_method) {
3644 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3645 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3646 if (call == NULL) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3651 // The NOP here is purely to ensure that eliding a call to
3652 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3653 __ nop();
3654 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3655 } else {
3656 int method_index = resolved_method_index(cbuf);
3657 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3658 : static_call_Relocation::spec(method_index);
3659 call = __ trampoline_call(Address(addr, rspec));
3660 if (call == NULL) {
3661 ciEnv::current()->record_failure("CodeCache is full");
3662 return;
3663 }
3664 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3665 // Calls of the same statically bound method can share
3666 // a stub to the interpreter.
3667 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3668 } else {
3669 // Emit stub for static call
3670 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3671 if (stub == NULL) {
3672 ciEnv::current()->record_failure("CodeCache is full");
3673 return;
3674 }
3675 }
3676 }
3677
3678 __ post_call_nop();
3679
3680 // Only non uncommon_trap calls need to reinitialize ptrue.
3681 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3682 __ reinitialize_ptrue();
3683 }
3684 %}
3685
3686 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3687 C2_MacroAssembler _masm(&cbuf);
3688 int method_index = resolved_method_index(cbuf);
3689 address call = __ ic_call((address)$meth$$method, method_index);
3690 if (call == NULL) {
3691 ciEnv::current()->record_failure("CodeCache is full");
3692 return;
3693 }
3694 __ post_call_nop();
3695 if (Compile::current()->max_vector_size() > 0) {
3696 __ reinitialize_ptrue();
3697 }
3698 %}
3699
3700 enc_class aarch64_enc_call_epilog() %{
3701 C2_MacroAssembler _masm(&cbuf);
3702 if (VerifyStackAtCalls) {
3703 // Check that stack depth is unchanged: find majik cookie on stack
3704 __ call_Unimplemented();
3705 }
3706 %}
3707
3708 enc_class aarch64_enc_java_to_runtime(method meth) %{
3709 C2_MacroAssembler _masm(&cbuf);
3710
3711 // some calls to generated routines (arraycopy code) are scheduled
3712 // by C2 as runtime calls. if so we can call them using a br (they
3713 // will be in a reachable segment) otherwise we have to use a blr
3714 // which loads the absolute address into a register.
3715 address entry = (address)$meth$$method;
3716 CodeBlob *cb = CodeCache::find_blob(entry);
3717 if (cb) {
3718 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3719 if (call == NULL) {
3720 ciEnv::current()->record_failure("CodeCache is full");
3721 return;
3722 }
3723 __ post_call_nop();
3724 } else {
3725 Label retaddr;
3726 __ adr(rscratch2, retaddr);
3727 __ lea(rscratch1, RuntimeAddress(entry));
3728 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3729 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3730 __ blr(rscratch1);
3731 __ bind(retaddr);
3732 __ post_call_nop();
3733 __ add(sp, sp, 2 * wordSize);
3734 }
3735 if (Compile::current()->max_vector_size() > 0) {
3736 __ reinitialize_ptrue();
3737 }
3738 %}
3739
3740 enc_class aarch64_enc_rethrow() %{
3741 C2_MacroAssembler _masm(&cbuf);
3742 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3743 %}
3744
3745 enc_class aarch64_enc_ret() %{
3746 C2_MacroAssembler _masm(&cbuf);
3747 #ifdef ASSERT
3748 if (Compile::current()->max_vector_size() > 0) {
3749 __ verify_ptrue();
3750 }
3751 #endif
3752 __ ret(lr);
3753 %}
3754
3755 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3756 C2_MacroAssembler _masm(&cbuf);
3757 Register target_reg = as_Register($jump_target$$reg);
3758 __ br(target_reg);
3759 %}
3760
3761 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3762 C2_MacroAssembler _masm(&cbuf);
3763 Register target_reg = as_Register($jump_target$$reg);
3764 // exception oop should be in r0
3765 // ret addr has been popped into lr
3766 // callee expects it in r3
3767 __ mov(r3, lr);
3768 __ br(target_reg);
3769 %}
3770
3771 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3772 C2_MacroAssembler _masm(&cbuf);
3773 Register oop = as_Register($object$$reg);
3774 Register box = as_Register($box$$reg);
3775 Register disp_hdr = as_Register($tmp$$reg);
3776 Register tmp = as_Register($tmp2$$reg);
3777 Label cont;
3778 Label object_has_monitor;
3779 Label count, no_count;
3780
3781 assert_different_registers(oop, box, tmp, disp_hdr);
3782
3783 // Load markWord from object into displaced_header.
3784 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3785
3786 if (DiagnoseSyncOnValueBasedClasses != 0) {
3787 __ load_klass(tmp, oop);
3788 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3789 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3790 __ br(Assembler::NE, cont);
3791 }
3792
3793 // Check for existing monitor
3794 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3795
3796 if (!UseHeavyMonitors) {
3797 if (UseFastLocking) {
3798 __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count, false);
3799
3800 // Indicate success at cont.
3801 __ cmp(oop, oop);
3802 __ b(count);
3803 } else {
3804 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3805 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3806
3807 // Initialize the box. (Must happen before we update the object mark!)
3808 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3809
3810 // Compare object markWord with an unlocked value (tmp) and if
3811 // equal exchange the stack address of our box with object markWord.
3812 // On failure disp_hdr contains the possibly locked markWord.
3813 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3814 /*release*/ true, /*weak*/ false, disp_hdr);
3815 __ br(Assembler::EQ, cont);
3816
3817 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3818
3819 // If the compare-and-exchange succeeded, then we found an unlocked
3820 // object, will have now locked it will continue at label cont
3821
3822 // Check if the owner is self by comparing the value in the
3823 // markWord of object (disp_hdr) with the stack pointer.
3824 __ mov(rscratch1, sp);
3825 __ sub(disp_hdr, disp_hdr, rscratch1);
3826 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3827 // If condition is true we are cont and hence we can store 0 as the
3828 // displaced header in the box, which indicates that it is a recursive lock.
3829 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3830 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3831 __ b(cont);
3832 }
3833 } else {
3834 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3835 __ b(cont);
3836 }
3837
3838 // Handle existing monitor.
3839 __ bind(object_has_monitor);
3840
3841 // The object's monitor m is unlocked iff m->owner == NULL,
3842 // otherwise m->owner may contain a thread or a stack address.
3843 //
3844 // Try to CAS m->owner from NULL to current thread.
3845 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3846 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3847 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3848
3849 if (!UseFastLocking) {
3850 // Store a non-null value into the box to avoid looking like a re-entrant
3851 // lock. The fast-path monitor unlock code checks for
3852 // markWord::monitor_value so use markWord::unused_mark which has the
3853 // relevant bit set, and also matches ObjectSynchronizer::enter.
3854 __ mov(tmp, (address)markWord::unused_mark().value());
3855 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3856 }
3857 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3858
3859 __ cmp(rscratch1, rthread);
3860 __ br(Assembler::NE, cont); // Check for recursive locking
3861
3862 // Recursive lock case
3863 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3864 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3865
3866 __ bind(cont);
3867 // flag == EQ indicates success
3868 // flag == NE indicates failure
3869 __ br(Assembler::NE, no_count);
3870
3871 __ bind(count);
3872 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3873
3874 __ bind(no_count);
3875 %}
3876
3877 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3878 C2_MacroAssembler _masm(&cbuf);
3879 Register oop = as_Register($object$$reg);
3880 Register box = as_Register($box$$reg);
3881 Register disp_hdr = as_Register($tmp$$reg);
3882 Register tmp = as_Register($tmp2$$reg);
3883 Label cont;
3884 Label object_has_monitor;
3885 Label count, no_count;
3886
3887 assert_different_registers(oop, box, tmp, disp_hdr);
3888
3889 if (!UseHeavyMonitors && !UseFastLocking) {
3890 // Find the lock address and load the displaced header from the stack.
3891 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3892
3893 // If the displaced header is 0, we have a recursive unlock.
3894 __ cmp(disp_hdr, zr);
3895 __ br(Assembler::EQ, cont);
3896 }
3897
3898 // Handle existing monitor.
3899 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3900 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3901
3902 if (!UseHeavyMonitors) {
3903 if (UseFastLocking) {
3904 __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
3905
3906 // Indicate success at cont.
3907 __ cmp(oop, oop);
3908 __ b(count);
3909 } else {
3910 // Check if it is still a light weight lock, this is is true if we
3911 // see the stack address of the basicLock in the markWord of the
3912 // object.
3913
3914 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3915 /*release*/ true, /*weak*/ false, tmp);
3916 __ b(cont);
3917 }
3918 } else {
3919 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3920 __ b(cont);
3921 }
3922
3923 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3924
3925 // Handle existing monitor.
3926 __ bind(object_has_monitor);
3927 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3928 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3929
3930 if (UseFastLocking) {
3931 // If the owner is anonymous, we need to fix it -- in the slow-path.
3932 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3933 // We cannot use tbnz here: tbnz would leave the condition flags untouched,
3934 // but we want to carry-over the NE condition to the exit at the cont label,
3935 // in order to take the slow-path.
3936 __ tst(disp_hdr, (uint64_t)(intptr_t) ANONYMOUS_OWNER);
3937 __ br(Assembler::NE, no_count);
3938 }
3939
3940 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3941
3942 Label notRecursive;
3943 __ cbz(disp_hdr, notRecursive);
3944
3945 // Recursive lock
3946 __ sub(disp_hdr, disp_hdr, 1u);
3947 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3948 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3949 __ b(cont);
3950
3951 __ bind(notRecursive);
3952 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3953 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3954 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3955 __ cmp(rscratch1, zr); // Sets flags for result
3956 __ cbnz(rscratch1, cont);
3957 // need a release store here
3958 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3959 __ stlr(zr, tmp); // set unowned
3960
3961 __ bind(cont);
3962 // flag == EQ indicates success
3963 // flag == NE indicates failure
3964 __ br(Assembler::NE, no_count);
3965
3966 __ bind(count);
3967 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
3968
3969 __ bind(no_count);
3970 %}
3971
3972 %}
3973
3974 //----------FRAME--------------------------------------------------------------
3975 // Definition of frame structure and management information.
3976 //
3977 // S T A C K L A Y O U T Allocators stack-slot number
3978 // | (to get allocators register number
3979 // G Owned by | | v add OptoReg::stack0())
3980 // r CALLER | |
3981 // o | +--------+ pad to even-align allocators stack-slot
3982 // w V | pad0 | numbers; owned by CALLER
3983 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3984 // h ^ | in | 5
3985 // | | args | 4 Holes in incoming args owned by SELF
3986 // | | | | 3
3987 // | | +--------+
3988 // V | | old out| Empty on Intel, window on Sparc
3989 // | old |preserve| Must be even aligned.
3990 // | SP-+--------+----> Matcher::_old_SP, even aligned
3991 // | | in | 3 area for Intel ret address
3992 // Owned by |preserve| Empty on Sparc.
3993 // SELF +--------+
3994 // | | pad2 | 2 pad to align old SP
3995 // | +--------+ 1
3996 // | | locks | 0
3997 // | +--------+----> OptoReg::stack0(), even aligned
3998 // | | pad1 | 11 pad to align new SP
3999 // | +--------+
4000 // | | | 10
4001 // | | spills | 9 spills
4002 // V | | 8 (pad0 slot for callee)
4003 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4004 // ^ | out | 7
4005 // | | args | 6 Holes in outgoing args owned by CALLEE
4006 // Owned by +--------+
4007 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4008 // | new |preserve| Must be even-aligned.
4009 // | SP-+--------+----> Matcher::_new_SP, even aligned
4010 // | | |
4011 //
4012 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4013 // known from SELF's arguments and the Java calling convention.
4014 // Region 6-7 is determined per call site.
4015 // Note 2: If the calling convention leaves holes in the incoming argument
4016 // area, those holes are owned by SELF. Holes in the outgoing area
4017 // are owned by the CALLEE. Holes should not be necessary in the
4018 // incoming area, as the Java calling convention is completely under
4019 // the control of the AD file. Doubles can be sorted and packed to
4020 // avoid holes. Holes in the outgoing arguments may be necessary for
4021 // varargs C calling conventions.
4022 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4023 // even aligned with pad0 as needed.
4024 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4025 // (the latter is true on Intel but is it false on AArch64?)
4026 // region 6-11 is even aligned; it may be padded out more so that
4027 // the region from SP to FP meets the minimum stack alignment.
4028 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4029 // alignment. Region 11, pad1, may be dynamically extended so that
4030 // SP meets the minimum alignment.
4031
4032 frame %{
4033 // These three registers define part of the calling convention
4034 // between compiled code and the interpreter.
4035
4036 // Inline Cache Register or Method for I2C.
4037 inline_cache_reg(R12);
4038
4039 // Number of stack slots consumed by locking an object
4040 sync_stack_slots(2);
4041
4042 // Compiled code's Frame Pointer
4043 frame_pointer(R31);
4044
4045 // Interpreter stores its frame pointer in a register which is
4046 // stored to the stack by I2CAdaptors.
4047 // I2CAdaptors convert from interpreted java to compiled java.
4048 interpreter_frame_pointer(R29);
4049
4050 // Stack alignment requirement
4051 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4052
4053 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4054 // for calls to C. Supports the var-args backing area for register parms.
4055 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4056
4057 // The after-PROLOG location of the return address. Location of
4058 // return address specifies a type (REG or STACK) and a number
4059 // representing the register number (i.e. - use a register name) or
4060 // stack slot.
4061 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4062 // Otherwise, it is above the locks and verification slot and alignment word
4063 // TODO this may well be correct but need to check why that - 2 is there
4064 // ppc port uses 0 but we definitely need to allow for fixed_slots
4065 // which folds in the space used for monitors
4066 return_addr(STACK - 2 +
4067 align_up((Compile::current()->in_preserve_stack_slots() +
4068 Compile::current()->fixed_slots()),
4069 stack_alignment_in_slots()));
4070
4071 // Location of compiled Java return values. Same as C for now.
4072 return_value
4073 %{
4074 // TODO do we allow ideal_reg == Op_RegN???
4075 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4076 "only return normal values");
4077
4078 static const int lo[Op_RegL + 1] = { // enum name
4079 0, // Op_Node
4080 0, // Op_Set
4081 R0_num, // Op_RegN
4082 R0_num, // Op_RegI
4083 R0_num, // Op_RegP
4084 V0_num, // Op_RegF
4085 V0_num, // Op_RegD
4086 R0_num // Op_RegL
4087 };
4088
4089 static const int hi[Op_RegL + 1] = { // enum name
4090 0, // Op_Node
4091 0, // Op_Set
4092 OptoReg::Bad, // Op_RegN
4093 OptoReg::Bad, // Op_RegI
4094 R0_H_num, // Op_RegP
4095 OptoReg::Bad, // Op_RegF
4096 V0_H_num, // Op_RegD
4097 R0_H_num // Op_RegL
4098 };
4099
4100 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4101 %}
4102 %}
4103
4104 //----------ATTRIBUTES---------------------------------------------------------
4105 //----------Operand Attributes-------------------------------------------------
4106 op_attrib op_cost(1); // Required cost attribute
4107
4108 //----------Instruction Attributes---------------------------------------------
4109 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4110 ins_attrib ins_size(32); // Required size attribute (in bits)
4111 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4112 // a non-matching short branch variant
4113 // of some long branch?
4114 ins_attrib ins_alignment(4); // Required alignment attribute (must
4115 // be a power of 2) specifies the
4116 // alignment that some part of the
4117 // instruction (not necessarily the
4118 // start) requires. If > 1, a
4119 // compute_padding() function must be
4120 // provided for the instruction
4121
4122 //----------OPERANDS-----------------------------------------------------------
4123 // Operand definitions must precede instruction definitions for correct parsing
4124 // in the ADLC because operands constitute user defined types which are used in
4125 // instruction definitions.
4126
4127 //----------Simple Operands----------------------------------------------------
4128
4129 // Integer operands 32 bit
4130 // 32 bit immediate
4131 operand immI()
4132 %{
4133 match(ConI);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 // 32 bit zero
4141 operand immI0()
4142 %{
4143 predicate(n->get_int() == 0);
4144 match(ConI);
4145
4146 op_cost(0);
4147 format %{ %}
4148 interface(CONST_INTER);
4149 %}
4150
4151 // 32 bit unit increment
4152 operand immI_1()
4153 %{
4154 predicate(n->get_int() == 1);
4155 match(ConI);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 // 32 bit unit decrement
4163 operand immI_M1()
4164 %{
4165 predicate(n->get_int() == -1);
4166 match(ConI);
4167
4168 op_cost(0);
4169 format %{ %}
4170 interface(CONST_INTER);
4171 %}
4172
4173 // Shift values for add/sub extension shift
4174 operand immIExt()
4175 %{
4176 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immI_gt_1()
4185 %{
4186 predicate(n->get_int() > 1);
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immI_le_4()
4195 %{
4196 predicate(n->get_int() <= 4);
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immI_16()
4205 %{
4206 predicate(n->get_int() == 16);
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immI_24()
4215 %{
4216 predicate(n->get_int() == 24);
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immI_32()
4225 %{
4226 predicate(n->get_int() == 32);
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immI_48()
4235 %{
4236 predicate(n->get_int() == 48);
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immI_56()
4245 %{
4246 predicate(n->get_int() == 56);
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immI_63()
4255 %{
4256 predicate(n->get_int() == 63);
4257 match(ConI);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immI_64()
4265 %{
4266 predicate(n->get_int() == 64);
4267 match(ConI);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immI_255()
4275 %{
4276 predicate(n->get_int() == 255);
4277 match(ConI);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immI_65535()
4285 %{
4286 predicate(n->get_int() == 65535);
4287 match(ConI);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immI_positive()
4295 %{
4296 predicate(n->get_int() > 0);
4297 match(ConI);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 operand immL_255()
4305 %{
4306 predicate(n->get_long() == 255L);
4307 match(ConL);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immL_65535()
4315 %{
4316 predicate(n->get_long() == 65535L);
4317 match(ConL);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 operand immL_4294967295()
4325 %{
4326 predicate(n->get_long() == 4294967295L);
4327 match(ConL);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 operand immL_bitmask()
4335 %{
4336 predicate((n->get_long() != 0)
4337 && ((n->get_long() & 0xc000000000000000l) == 0)
4338 && is_power_of_2(n->get_long() + 1));
4339 match(ConL);
4340
4341 op_cost(0);
4342 format %{ %}
4343 interface(CONST_INTER);
4344 %}
4345
4346 operand immI_bitmask()
4347 %{
4348 predicate((n->get_int() != 0)
4349 && ((n->get_int() & 0xc0000000) == 0)
4350 && is_power_of_2(n->get_int() + 1));
4351 match(ConI);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 operand immL_positive_bitmaskI()
4359 %{
4360 predicate((n->get_long() != 0)
4361 && ((julong)n->get_long() < 0x80000000ULL)
4362 && is_power_of_2(n->get_long() + 1));
4363 match(ConL);
4364
4365 op_cost(0);
4366 format %{ %}
4367 interface(CONST_INTER);
4368 %}
4369
4370 // Scale values for scaled offset addressing modes (up to long but not quad)
4371 operand immIScale()
4372 %{
4373 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4374 match(ConI);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 26 bit signed offset -- for pc-relative branches
4382 operand immI26()
4383 %{
4384 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4385 match(ConI);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 19 bit signed offset -- for pc-relative loads
4393 operand immI19()
4394 %{
4395 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4396 match(ConI);
4397
4398 op_cost(0);
4399 format %{ %}
4400 interface(CONST_INTER);
4401 %}
4402
4403 // 12 bit unsigned offset -- for base plus immediate loads
4404 operand immIU12()
4405 %{
4406 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 operand immLU12()
4415 %{
4416 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4417 match(ConL);
4418
4419 op_cost(0);
4420 format %{ %}
4421 interface(CONST_INTER);
4422 %}
4423
4424 // Offset for scaled or unscaled immediate loads and stores
4425 operand immIOffset()
4426 %{
4427 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4428 match(ConI);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 operand immIOffset1()
4436 %{
4437 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4438 match(ConI);
4439
4440 op_cost(0);
4441 format %{ %}
4442 interface(CONST_INTER);
4443 %}
4444
4445 operand immIOffset2()
4446 %{
4447 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4448 match(ConI);
4449
4450 op_cost(0);
4451 format %{ %}
4452 interface(CONST_INTER);
4453 %}
4454
4455 operand immIOffset4()
4456 %{
4457 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4458 match(ConI);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 operand immIOffset8()
4466 %{
4467 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4468 match(ConI);
4469
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 operand immIOffset16()
4476 %{
4477 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4478 match(ConI);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 operand immLoffset()
4486 %{
4487 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4488 match(ConL);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 operand immLoffset1()
4496 %{
4497 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4498 match(ConL);
4499
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 operand immLoffset2()
4506 %{
4507 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4508 match(ConL);
4509
4510 op_cost(0);
4511 format %{ %}
4512 interface(CONST_INTER);
4513 %}
4514
4515 operand immLoffset4()
4516 %{
4517 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4518 match(ConL);
4519
4520 op_cost(0);
4521 format %{ %}
4522 interface(CONST_INTER);
4523 %}
4524
4525 operand immLoffset8()
4526 %{
4527 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4528 match(ConL);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 operand immLoffset16()
4536 %{
4537 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4538 match(ConL);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // 8 bit signed value.
4546 operand immI8()
4547 %{
4548 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4549 match(ConI);
4550
4551 op_cost(0);
4552 format %{ %}
4553 interface(CONST_INTER);
4554 %}
4555
4556 // 8 bit signed value (simm8), or #simm8 LSL 8.
4557 operand immI8_shift8()
4558 %{
4559 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4560 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4561 match(ConI);
4562
4563 op_cost(0);
4564 format %{ %}
4565 interface(CONST_INTER);
4566 %}
4567
4568 // 8 bit signed value (simm8), or #simm8 LSL 8.
4569 operand immL8_shift8()
4570 %{
4571 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4572 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4573 match(ConL);
4574
4575 op_cost(0);
4576 format %{ %}
4577 interface(CONST_INTER);
4578 %}
4579
4580 // 8 bit integer valid for vector add sub immediate
4581 operand immBAddSubV()
4582 %{
4583 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4584 match(ConI);
4585
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // 32 bit integer valid for add sub immediate
4592 operand immIAddSub()
4593 %{
4594 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4595 match(ConI);
4596 op_cost(0);
4597 format %{ %}
4598 interface(CONST_INTER);
4599 %}
4600
4601 // 32 bit integer valid for vector add sub immediate
4602 operand immIAddSubV()
4603 %{
4604 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4605 match(ConI);
4606
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 // 32 bit unsigned integer valid for logical immediate
4613
4614 operand immBLog()
4615 %{
4616 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4617 match(ConI);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 operand immSLog()
4625 %{
4626 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4627 match(ConI);
4628
4629 op_cost(0);
4630 format %{ %}
4631 interface(CONST_INTER);
4632 %}
4633
4634 operand immILog()
4635 %{
4636 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4637 match(ConI);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // Integer operands 64 bit
4645 // 64 bit immediate
4646 operand immL()
4647 %{
4648 match(ConL);
4649
4650 op_cost(0);
4651 format %{ %}
4652 interface(CONST_INTER);
4653 %}
4654
4655 // 64 bit zero
4656 operand immL0()
4657 %{
4658 predicate(n->get_long() == 0);
4659 match(ConL);
4660
4661 op_cost(0);
4662 format %{ %}
4663 interface(CONST_INTER);
4664 %}
4665
4666 // 64 bit unit increment
4667 operand immL_1()
4668 %{
4669 predicate(n->get_long() == 1);
4670 match(ConL);
4671
4672 op_cost(0);
4673 format %{ %}
4674 interface(CONST_INTER);
4675 %}
4676
4677 // 64 bit unit decrement
4678 operand immL_M1()
4679 %{
4680 predicate(n->get_long() == -1);
4681 match(ConL);
4682
4683 op_cost(0);
4684 format %{ %}
4685 interface(CONST_INTER);
4686 %}
4687
4688 // 32 bit offset of pc in thread anchor
4689
4690 operand immL_pc_off()
4691 %{
4692 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4693 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4694 match(ConL);
4695
4696 op_cost(0);
4697 format %{ %}
4698 interface(CONST_INTER);
4699 %}
4700
4701 // 64 bit integer valid for add sub immediate
4702 operand immLAddSub()
4703 %{
4704 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4705 match(ConL);
4706 op_cost(0);
4707 format %{ %}
4708 interface(CONST_INTER);
4709 %}
4710
4711 // 64 bit integer valid for addv subv immediate
4712 operand immLAddSubV()
4713 %{
4714 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4715 match(ConL);
4716
4717 op_cost(0);
4718 format %{ %}
4719 interface(CONST_INTER);
4720 %}
4721
4722 // 64 bit integer valid for logical immediate
4723 operand immLLog()
4724 %{
4725 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4726 match(ConL);
4727 op_cost(0);
4728 format %{ %}
4729 interface(CONST_INTER);
4730 %}
4731
4732 // Long Immediate: low 32-bit mask
4733 operand immL_32bits()
4734 %{
4735 predicate(n->get_long() == 0xFFFFFFFFL);
4736 match(ConL);
4737 op_cost(0);
4738 format %{ %}
4739 interface(CONST_INTER);
4740 %}
4741
4742 // Pointer operands
4743 // Pointer Immediate
4744 operand immP()
4745 %{
4746 match(ConP);
4747
4748 op_cost(0);
4749 format %{ %}
4750 interface(CONST_INTER);
4751 %}
4752
4753 // NULL Pointer Immediate
4754 operand immP0()
4755 %{
4756 predicate(n->get_ptr() == 0);
4757 match(ConP);
4758
4759 op_cost(0);
4760 format %{ %}
4761 interface(CONST_INTER);
4762 %}
4763
4764 // Pointer Immediate One
4765 // this is used in object initialization (initial object header)
4766 operand immP_1()
4767 %{
4768 predicate(n->get_ptr() == 1);
4769 match(ConP);
4770
4771 op_cost(0);
4772 format %{ %}
4773 interface(CONST_INTER);
4774 %}
4775
4776 // Card Table Byte Map Base
4777 operand immByteMapBase()
4778 %{
4779 // Get base of card map
4780 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4781 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4782 match(ConP);
4783
4784 op_cost(0);
4785 format %{ %}
4786 interface(CONST_INTER);
4787 %}
4788
4789 // Pointer Immediate Minus One
4790 // this is used when we want to write the current PC to the thread anchor
4791 operand immP_M1()
4792 %{
4793 predicate(n->get_ptr() == -1);
4794 match(ConP);
4795
4796 op_cost(0);
4797 format %{ %}
4798 interface(CONST_INTER);
4799 %}
4800
4801 // Pointer Immediate Minus Two
4802 // this is used when we want to write the current PC to the thread anchor
4803 operand immP_M2()
4804 %{
4805 predicate(n->get_ptr() == -2);
4806 match(ConP);
4807
4808 op_cost(0);
4809 format %{ %}
4810 interface(CONST_INTER);
4811 %}
4812
4813 // Float and Double operands
4814 // Double Immediate
4815 operand immD()
4816 %{
4817 match(ConD);
4818 op_cost(0);
4819 format %{ %}
4820 interface(CONST_INTER);
4821 %}
4822
4823 // Double Immediate: +0.0d
4824 operand immD0()
4825 %{
4826 predicate(jlong_cast(n->getd()) == 0);
4827 match(ConD);
4828
4829 op_cost(0);
4830 format %{ %}
4831 interface(CONST_INTER);
4832 %}
4833
4834 // constant 'double +0.0'.
4835 operand immDPacked()
4836 %{
4837 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4838 match(ConD);
4839 op_cost(0);
4840 format %{ %}
4841 interface(CONST_INTER);
4842 %}
4843
4844 // Float Immediate
4845 operand immF()
4846 %{
4847 match(ConF);
4848 op_cost(0);
4849 format %{ %}
4850 interface(CONST_INTER);
4851 %}
4852
4853 // Float Immediate: +0.0f.
4854 operand immF0()
4855 %{
4856 predicate(jint_cast(n->getf()) == 0);
4857 match(ConF);
4858
4859 op_cost(0);
4860 format %{ %}
4861 interface(CONST_INTER);
4862 %}
4863
4864 //
4865 operand immFPacked()
4866 %{
4867 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4868 match(ConF);
4869 op_cost(0);
4870 format %{ %}
4871 interface(CONST_INTER);
4872 %}
4873
4874 // Narrow pointer operands
4875 // Narrow Pointer Immediate
4876 operand immN()
4877 %{
4878 match(ConN);
4879
4880 op_cost(0);
4881 format %{ %}
4882 interface(CONST_INTER);
4883 %}
4884
4885 // Narrow NULL Pointer Immediate
4886 operand immN0()
4887 %{
4888 predicate(n->get_narrowcon() == 0);
4889 match(ConN);
4890
4891 op_cost(0);
4892 format %{ %}
4893 interface(CONST_INTER);
4894 %}
4895
4896 operand immNKlass()
4897 %{
4898 match(ConNKlass);
4899
4900 op_cost(0);
4901 format %{ %}
4902 interface(CONST_INTER);
4903 %}
4904
4905 // Integer 32 bit Register Operands
4906 // Integer 32 bitRegister (excludes SP)
4907 operand iRegI()
4908 %{
4909 constraint(ALLOC_IN_RC(any_reg32));
4910 match(RegI);
4911 match(iRegINoSp);
4912 op_cost(0);
4913 format %{ %}
4914 interface(REG_INTER);
4915 %}
4916
4917 // Integer 32 bit Register not Special
4918 operand iRegINoSp()
4919 %{
4920 constraint(ALLOC_IN_RC(no_special_reg32));
4921 match(RegI);
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Integer 64 bit Register Operands
4928 // Integer 64 bit Register (includes SP)
4929 operand iRegL()
4930 %{
4931 constraint(ALLOC_IN_RC(any_reg));
4932 match(RegL);
4933 match(iRegLNoSp);
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 // Integer 64 bit Register not Special
4940 operand iRegLNoSp()
4941 %{
4942 constraint(ALLOC_IN_RC(no_special_reg));
4943 match(RegL);
4944 match(iRegL_R0);
4945 format %{ %}
4946 interface(REG_INTER);
4947 %}
4948
4949 // Pointer Register Operands
4950 // Pointer Register
4951 operand iRegP()
4952 %{
4953 constraint(ALLOC_IN_RC(ptr_reg));
4954 match(RegP);
4955 match(iRegPNoSp);
4956 match(iRegP_R0);
4957 //match(iRegP_R2);
4958 //match(iRegP_R4);
4959 match(iRegP_R5);
4960 match(thread_RegP);
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 // Pointer 64 bit Register not Special
4967 operand iRegPNoSp()
4968 %{
4969 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4970 match(RegP);
4971 // match(iRegP);
4972 // match(iRegP_R0);
4973 // match(iRegP_R2);
4974 // match(iRegP_R4);
4975 // match(iRegP_R5);
4976 // match(thread_RegP);
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 // Pointer 64 bit Register R0 only
4983 operand iRegP_R0()
4984 %{
4985 constraint(ALLOC_IN_RC(r0_reg));
4986 match(RegP);
4987 // match(iRegP);
4988 match(iRegPNoSp);
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 // Pointer 64 bit Register R1 only
4995 operand iRegP_R1()
4996 %{
4997 constraint(ALLOC_IN_RC(r1_reg));
4998 match(RegP);
4999 // match(iRegP);
5000 match(iRegPNoSp);
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 // Pointer 64 bit Register R2 only
5007 operand iRegP_R2()
5008 %{
5009 constraint(ALLOC_IN_RC(r2_reg));
5010 match(RegP);
5011 // match(iRegP);
5012 match(iRegPNoSp);
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 // Pointer 64 bit Register R3 only
5019 operand iRegP_R3()
5020 %{
5021 constraint(ALLOC_IN_RC(r3_reg));
5022 match(RegP);
5023 // match(iRegP);
5024 match(iRegPNoSp);
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 // Pointer 64 bit Register R4 only
5031 operand iRegP_R4()
5032 %{
5033 constraint(ALLOC_IN_RC(r4_reg));
5034 match(RegP);
5035 // match(iRegP);
5036 match(iRegPNoSp);
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 // Pointer 64 bit Register R5 only
5043 operand iRegP_R5()
5044 %{
5045 constraint(ALLOC_IN_RC(r5_reg));
5046 match(RegP);
5047 // match(iRegP);
5048 match(iRegPNoSp);
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 // Pointer 64 bit Register R10 only
5055 operand iRegP_R10()
5056 %{
5057 constraint(ALLOC_IN_RC(r10_reg));
5058 match(RegP);
5059 // match(iRegP);
5060 match(iRegPNoSp);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 // Long 64 bit Register R0 only
5067 operand iRegL_R0()
5068 %{
5069 constraint(ALLOC_IN_RC(r0_reg));
5070 match(RegL);
5071 match(iRegLNoSp);
5072 op_cost(0);
5073 format %{ %}
5074 interface(REG_INTER);
5075 %}
5076
5077 // Long 64 bit Register R2 only
5078 operand iRegL_R2()
5079 %{
5080 constraint(ALLOC_IN_RC(r2_reg));
5081 match(RegL);
5082 match(iRegLNoSp);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 // Long 64 bit Register R3 only
5089 operand iRegL_R3()
5090 %{
5091 constraint(ALLOC_IN_RC(r3_reg));
5092 match(RegL);
5093 match(iRegLNoSp);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 // Long 64 bit Register R11 only
5100 operand iRegL_R11()
5101 %{
5102 constraint(ALLOC_IN_RC(r11_reg));
5103 match(RegL);
5104 match(iRegLNoSp);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 // Pointer 64 bit Register FP only
5111 operand iRegP_FP()
5112 %{
5113 constraint(ALLOC_IN_RC(fp_reg));
5114 match(RegP);
5115 // match(iRegP);
5116 op_cost(0);
5117 format %{ %}
5118 interface(REG_INTER);
5119 %}
5120
5121 // Register R0 only
5122 operand iRegI_R0()
5123 %{
5124 constraint(ALLOC_IN_RC(int_r0_reg));
5125 match(RegI);
5126 match(iRegINoSp);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 // Register R2 only
5133 operand iRegI_R2()
5134 %{
5135 constraint(ALLOC_IN_RC(int_r2_reg));
5136 match(RegI);
5137 match(iRegINoSp);
5138 op_cost(0);
5139 format %{ %}
5140 interface(REG_INTER);
5141 %}
5142
5143 // Register R3 only
5144 operand iRegI_R3()
5145 %{
5146 constraint(ALLOC_IN_RC(int_r3_reg));
5147 match(RegI);
5148 match(iRegINoSp);
5149 op_cost(0);
5150 format %{ %}
5151 interface(REG_INTER);
5152 %}
5153
5154
5155 // Register R4 only
5156 operand iRegI_R4()
5157 %{
5158 constraint(ALLOC_IN_RC(int_r4_reg));
5159 match(RegI);
5160 match(iRegINoSp);
5161 op_cost(0);
5162 format %{ %}
5163 interface(REG_INTER);
5164 %}
5165
5166
5167 // Pointer Register Operands
5168 // Narrow Pointer Register
5169 operand iRegN()
5170 %{
5171 constraint(ALLOC_IN_RC(any_reg32));
5172 match(RegN);
5173 match(iRegNNoSp);
5174 op_cost(0);
5175 format %{ %}
5176 interface(REG_INTER);
5177 %}
5178
5179 operand iRegN_R0()
5180 %{
5181 constraint(ALLOC_IN_RC(r0_reg));
5182 match(iRegN);
5183 op_cost(0);
5184 format %{ %}
5185 interface(REG_INTER);
5186 %}
5187
5188 operand iRegN_R2()
5189 %{
5190 constraint(ALLOC_IN_RC(r2_reg));
5191 match(iRegN);
5192 op_cost(0);
5193 format %{ %}
5194 interface(REG_INTER);
5195 %}
5196
5197 operand iRegN_R3()
5198 %{
5199 constraint(ALLOC_IN_RC(r3_reg));
5200 match(iRegN);
5201 op_cost(0);
5202 format %{ %}
5203 interface(REG_INTER);
5204 %}
5205
5206 // Integer 64 bit Register not Special
5207 operand iRegNNoSp()
5208 %{
5209 constraint(ALLOC_IN_RC(no_special_reg32));
5210 match(RegN);
5211 op_cost(0);
5212 format %{ %}
5213 interface(REG_INTER);
5214 %}
5215
5216 // heap base register -- used for encoding immN0
5217
5218 operand iRegIHeapbase()
5219 %{
5220 constraint(ALLOC_IN_RC(heapbase_reg));
5221 match(RegI);
5222 op_cost(0);
5223 format %{ %}
5224 interface(REG_INTER);
5225 %}
5226
5227 // Float Register
5228 // Float register operands
5229 operand vRegF()
5230 %{
5231 constraint(ALLOC_IN_RC(float_reg));
5232 match(RegF);
5233
5234 op_cost(0);
5235 format %{ %}
5236 interface(REG_INTER);
5237 %}
5238
5239 // Double Register
5240 // Double register operands
5241 operand vRegD()
5242 %{
5243 constraint(ALLOC_IN_RC(double_reg));
5244 match(RegD);
5245
5246 op_cost(0);
5247 format %{ %}
5248 interface(REG_INTER);
5249 %}
5250
5251 // Generic vector class. This will be used for
5252 // all vector operands, including NEON and SVE.
5253 operand vReg()
5254 %{
5255 constraint(ALLOC_IN_RC(dynamic));
5256 match(VecA);
5257 match(VecD);
5258 match(VecX);
5259
5260 op_cost(0);
5261 format %{ %}
5262 interface(REG_INTER);
5263 %}
5264
5265 operand vecA()
5266 %{
5267 constraint(ALLOC_IN_RC(vectora_reg));
5268 match(VecA);
5269
5270 op_cost(0);
5271 format %{ %}
5272 interface(REG_INTER);
5273 %}
5274
5275 operand vecD()
5276 %{
5277 constraint(ALLOC_IN_RC(vectord_reg));
5278 match(VecD);
5279
5280 op_cost(0);
5281 format %{ %}
5282 interface(REG_INTER);
5283 %}
5284
5285 operand vecX()
5286 %{
5287 constraint(ALLOC_IN_RC(vectorx_reg));
5288 match(VecX);
5289
5290 op_cost(0);
5291 format %{ %}
5292 interface(REG_INTER);
5293 %}
5294
5295 operand vRegD_V0()
5296 %{
5297 constraint(ALLOC_IN_RC(v0_reg));
5298 match(RegD);
5299 op_cost(0);
5300 format %{ %}
5301 interface(REG_INTER);
5302 %}
5303
5304 operand vRegD_V1()
5305 %{
5306 constraint(ALLOC_IN_RC(v1_reg));
5307 match(RegD);
5308 op_cost(0);
5309 format %{ %}
5310 interface(REG_INTER);
5311 %}
5312
5313 operand vRegD_V2()
5314 %{
5315 constraint(ALLOC_IN_RC(v2_reg));
5316 match(RegD);
5317 op_cost(0);
5318 format %{ %}
5319 interface(REG_INTER);
5320 %}
5321
5322 operand vRegD_V3()
5323 %{
5324 constraint(ALLOC_IN_RC(v3_reg));
5325 match(RegD);
5326 op_cost(0);
5327 format %{ %}
5328 interface(REG_INTER);
5329 %}
5330
5331 operand vRegD_V4()
5332 %{
5333 constraint(ALLOC_IN_RC(v4_reg));
5334 match(RegD);
5335 op_cost(0);
5336 format %{ %}
5337 interface(REG_INTER);
5338 %}
5339
5340 operand vRegD_V5()
5341 %{
5342 constraint(ALLOC_IN_RC(v5_reg));
5343 match(RegD);
5344 op_cost(0);
5345 format %{ %}
5346 interface(REG_INTER);
5347 %}
5348
5349 operand vRegD_V6()
5350 %{
5351 constraint(ALLOC_IN_RC(v6_reg));
5352 match(RegD);
5353 op_cost(0);
5354 format %{ %}
5355 interface(REG_INTER);
5356 %}
5357
5358 operand vRegD_V7()
5359 %{
5360 constraint(ALLOC_IN_RC(v7_reg));
5361 match(RegD);
5362 op_cost(0);
5363 format %{ %}
5364 interface(REG_INTER);
5365 %}
5366
5367 operand vRegD_V8()
5368 %{
5369 constraint(ALLOC_IN_RC(v8_reg));
5370 match(RegD);
5371 op_cost(0);
5372 format %{ %}
5373 interface(REG_INTER);
5374 %}
5375
5376 operand vRegD_V9()
5377 %{
5378 constraint(ALLOC_IN_RC(v9_reg));
5379 match(RegD);
5380 op_cost(0);
5381 format %{ %}
5382 interface(REG_INTER);
5383 %}
5384
5385 operand vRegD_V10()
5386 %{
5387 constraint(ALLOC_IN_RC(v10_reg));
5388 match(RegD);
5389 op_cost(0);
5390 format %{ %}
5391 interface(REG_INTER);
5392 %}
5393
5394 operand vRegD_V11()
5395 %{
5396 constraint(ALLOC_IN_RC(v11_reg));
5397 match(RegD);
5398 op_cost(0);
5399 format %{ %}
5400 interface(REG_INTER);
5401 %}
5402
5403 operand vRegD_V12()
5404 %{
5405 constraint(ALLOC_IN_RC(v12_reg));
5406 match(RegD);
5407 op_cost(0);
5408 format %{ %}
5409 interface(REG_INTER);
5410 %}
5411
5412 operand vRegD_V13()
5413 %{
5414 constraint(ALLOC_IN_RC(v13_reg));
5415 match(RegD);
5416 op_cost(0);
5417 format %{ %}
5418 interface(REG_INTER);
5419 %}
5420
5421 operand vRegD_V14()
5422 %{
5423 constraint(ALLOC_IN_RC(v14_reg));
5424 match(RegD);
5425 op_cost(0);
5426 format %{ %}
5427 interface(REG_INTER);
5428 %}
5429
5430 operand vRegD_V15()
5431 %{
5432 constraint(ALLOC_IN_RC(v15_reg));
5433 match(RegD);
5434 op_cost(0);
5435 format %{ %}
5436 interface(REG_INTER);
5437 %}
5438
5439 operand vRegD_V16()
5440 %{
5441 constraint(ALLOC_IN_RC(v16_reg));
5442 match(RegD);
5443 op_cost(0);
5444 format %{ %}
5445 interface(REG_INTER);
5446 %}
5447
5448 operand vRegD_V17()
5449 %{
5450 constraint(ALLOC_IN_RC(v17_reg));
5451 match(RegD);
5452 op_cost(0);
5453 format %{ %}
5454 interface(REG_INTER);
5455 %}
5456
5457 operand vRegD_V18()
5458 %{
5459 constraint(ALLOC_IN_RC(v18_reg));
5460 match(RegD);
5461 op_cost(0);
5462 format %{ %}
5463 interface(REG_INTER);
5464 %}
5465
5466 operand vRegD_V19()
5467 %{
5468 constraint(ALLOC_IN_RC(v19_reg));
5469 match(RegD);
5470 op_cost(0);
5471 format %{ %}
5472 interface(REG_INTER);
5473 %}
5474
5475 operand vRegD_V20()
5476 %{
5477 constraint(ALLOC_IN_RC(v20_reg));
5478 match(RegD);
5479 op_cost(0);
5480 format %{ %}
5481 interface(REG_INTER);
5482 %}
5483
5484 operand vRegD_V21()
5485 %{
5486 constraint(ALLOC_IN_RC(v21_reg));
5487 match(RegD);
5488 op_cost(0);
5489 format %{ %}
5490 interface(REG_INTER);
5491 %}
5492
5493 operand vRegD_V22()
5494 %{
5495 constraint(ALLOC_IN_RC(v22_reg));
5496 match(RegD);
5497 op_cost(0);
5498 format %{ %}
5499 interface(REG_INTER);
5500 %}
5501
5502 operand vRegD_V23()
5503 %{
5504 constraint(ALLOC_IN_RC(v23_reg));
5505 match(RegD);
5506 op_cost(0);
5507 format %{ %}
5508 interface(REG_INTER);
5509 %}
5510
5511 operand vRegD_V24()
5512 %{
5513 constraint(ALLOC_IN_RC(v24_reg));
5514 match(RegD);
5515 op_cost(0);
5516 format %{ %}
5517 interface(REG_INTER);
5518 %}
5519
5520 operand vRegD_V25()
5521 %{
5522 constraint(ALLOC_IN_RC(v25_reg));
5523 match(RegD);
5524 op_cost(0);
5525 format %{ %}
5526 interface(REG_INTER);
5527 %}
5528
5529 operand vRegD_V26()
5530 %{
5531 constraint(ALLOC_IN_RC(v26_reg));
5532 match(RegD);
5533 op_cost(0);
5534 format %{ %}
5535 interface(REG_INTER);
5536 %}
5537
5538 operand vRegD_V27()
5539 %{
5540 constraint(ALLOC_IN_RC(v27_reg));
5541 match(RegD);
5542 op_cost(0);
5543 format %{ %}
5544 interface(REG_INTER);
5545 %}
5546
5547 operand vRegD_V28()
5548 %{
5549 constraint(ALLOC_IN_RC(v28_reg));
5550 match(RegD);
5551 op_cost(0);
5552 format %{ %}
5553 interface(REG_INTER);
5554 %}
5555
5556 operand vRegD_V29()
5557 %{
5558 constraint(ALLOC_IN_RC(v29_reg));
5559 match(RegD);
5560 op_cost(0);
5561 format %{ %}
5562 interface(REG_INTER);
5563 %}
5564
5565 operand vRegD_V30()
5566 %{
5567 constraint(ALLOC_IN_RC(v30_reg));
5568 match(RegD);
5569 op_cost(0);
5570 format %{ %}
5571 interface(REG_INTER);
5572 %}
5573
5574 operand vRegD_V31()
5575 %{
5576 constraint(ALLOC_IN_RC(v31_reg));
5577 match(RegD);
5578 op_cost(0);
5579 format %{ %}
5580 interface(REG_INTER);
5581 %}
5582
5583 operand pReg()
5584 %{
5585 constraint(ALLOC_IN_RC(pr_reg));
5586 match(RegVectMask);
5587 match(pRegGov);
5588 op_cost(0);
5589 format %{ %}
5590 interface(REG_INTER);
5591 %}
5592
5593 operand pRegGov()
5594 %{
5595 constraint(ALLOC_IN_RC(gov_pr));
5596 match(RegVectMask);
5597 match(pReg);
5598 op_cost(0);
5599 format %{ %}
5600 interface(REG_INTER);
5601 %}
5602
5603 operand pRegGov_P0()
5604 %{
5605 constraint(ALLOC_IN_RC(p0_reg));
5606 match(RegVectMask);
5607 op_cost(0);
5608 format %{ %}
5609 interface(REG_INTER);
5610 %}
5611
5612 operand pRegGov_P1()
5613 %{
5614 constraint(ALLOC_IN_RC(p1_reg));
5615 match(RegVectMask);
5616 op_cost(0);
5617 format %{ %}
5618 interface(REG_INTER);
5619 %}
5620
5621 // Flags register, used as output of signed compare instructions
5622
5623 // note that on AArch64 we also use this register as the output for
5624 // for floating point compare instructions (CmpF CmpD). this ensures
5625 // that ordered inequality tests use GT, GE, LT or LE none of which
5626 // pass through cases where the result is unordered i.e. one or both
5627 // inputs to the compare is a NaN. this means that the ideal code can
5628 // replace e.g. a GT with an LE and not end up capturing the NaN case
5629 // (where the comparison should always fail). EQ and NE tests are
5630 // always generated in ideal code so that unordered folds into the NE
5631 // case, matching the behaviour of AArch64 NE.
5632 //
5633 // This differs from x86 where the outputs of FP compares use a
5634 // special FP flags registers and where compares based on this
5635 // register are distinguished into ordered inequalities (cmpOpUCF) and
5636 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5637 // to explicitly handle the unordered case in branches. x86 also has
5638 // to include extra CMoveX rules to accept a cmpOpUCF input.
5639
5640 operand rFlagsReg()
5641 %{
5642 constraint(ALLOC_IN_RC(int_flags));
5643 match(RegFlags);
5644
5645 op_cost(0);
5646 format %{ "RFLAGS" %}
5647 interface(REG_INTER);
5648 %}
5649
5650 // Flags register, used as output of unsigned compare instructions
5651 operand rFlagsRegU()
5652 %{
5653 constraint(ALLOC_IN_RC(int_flags));
5654 match(RegFlags);
5655
5656 op_cost(0);
5657 format %{ "RFLAGSU" %}
5658 interface(REG_INTER);
5659 %}
5660
5661 // Special Registers
5662
5663 // Method Register
5664 operand inline_cache_RegP(iRegP reg)
5665 %{
5666 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5667 match(reg);
5668 match(iRegPNoSp);
5669 op_cost(0);
5670 format %{ %}
5671 interface(REG_INTER);
5672 %}
5673
5674 // Thread Register
5675 operand thread_RegP(iRegP reg)
5676 %{
5677 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5678 match(reg);
5679 op_cost(0);
5680 format %{ %}
5681 interface(REG_INTER);
5682 %}
5683
5684 operand lr_RegP(iRegP reg)
5685 %{
5686 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5687 match(reg);
5688 op_cost(0);
5689 format %{ %}
5690 interface(REG_INTER);
5691 %}
5692
5693 //----------Memory Operands----------------------------------------------------
5694
5695 operand indirect(iRegP reg)
5696 %{
5697 constraint(ALLOC_IN_RC(ptr_reg));
5698 match(reg);
5699 op_cost(0);
5700 format %{ "[$reg]" %}
5701 interface(MEMORY_INTER) %{
5702 base($reg);
5703 index(0xffffffff);
5704 scale(0x0);
5705 disp(0x0);
5706 %}
5707 %}
5708
5709 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5710 %{
5711 constraint(ALLOC_IN_RC(ptr_reg));
5712 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5713 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5714 op_cost(0);
5715 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5716 interface(MEMORY_INTER) %{
5717 base($reg);
5718 index($ireg);
5719 scale($scale);
5720 disp(0x0);
5721 %}
5722 %}
5723
5724 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5725 %{
5726 constraint(ALLOC_IN_RC(ptr_reg));
5727 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5728 match(AddP reg (LShiftL lreg scale));
5729 op_cost(0);
5730 format %{ "$reg, $lreg lsl($scale)" %}
5731 interface(MEMORY_INTER) %{
5732 base($reg);
5733 index($lreg);
5734 scale($scale);
5735 disp(0x0);
5736 %}
5737 %}
5738
5739 operand indIndexI2L(iRegP reg, iRegI ireg)
5740 %{
5741 constraint(ALLOC_IN_RC(ptr_reg));
5742 match(AddP reg (ConvI2L ireg));
5743 op_cost(0);
5744 format %{ "$reg, $ireg, 0, I2L" %}
5745 interface(MEMORY_INTER) %{
5746 base($reg);
5747 index($ireg);
5748 scale(0x0);
5749 disp(0x0);
5750 %}
5751 %}
5752
5753 operand indIndex(iRegP reg, iRegL lreg)
5754 %{
5755 constraint(ALLOC_IN_RC(ptr_reg));
5756 match(AddP reg lreg);
5757 op_cost(0);
5758 format %{ "$reg, $lreg" %}
5759 interface(MEMORY_INTER) %{
5760 base($reg);
5761 index($lreg);
5762 scale(0x0);
5763 disp(0x0);
5764 %}
5765 %}
5766
5767 operand indOffI(iRegP reg, immIOffset off)
5768 %{
5769 constraint(ALLOC_IN_RC(ptr_reg));
5770 match(AddP reg off);
5771 op_cost(0);
5772 format %{ "[$reg, $off]" %}
5773 interface(MEMORY_INTER) %{
5774 base($reg);
5775 index(0xffffffff);
5776 scale(0x0);
5777 disp($off);
5778 %}
5779 %}
5780
5781 operand indOffI1(iRegP reg, immIOffset1 off)
5782 %{
5783 constraint(ALLOC_IN_RC(ptr_reg));
5784 match(AddP reg off);
5785 op_cost(0);
5786 format %{ "[$reg, $off]" %}
5787 interface(MEMORY_INTER) %{
5788 base($reg);
5789 index(0xffffffff);
5790 scale(0x0);
5791 disp($off);
5792 %}
5793 %}
5794
5795 operand indOffI2(iRegP reg, immIOffset2 off)
5796 %{
5797 constraint(ALLOC_IN_RC(ptr_reg));
5798 match(AddP reg off);
5799 op_cost(0);
5800 format %{ "[$reg, $off]" %}
5801 interface(MEMORY_INTER) %{
5802 base($reg);
5803 index(0xffffffff);
5804 scale(0x0);
5805 disp($off);
5806 %}
5807 %}
5808
5809 operand indOffI4(iRegP reg, immIOffset4 off)
5810 %{
5811 constraint(ALLOC_IN_RC(ptr_reg));
5812 match(AddP reg off);
5813 op_cost(0);
5814 format %{ "[$reg, $off]" %}
5815 interface(MEMORY_INTER) %{
5816 base($reg);
5817 index(0xffffffff);
5818 scale(0x0);
5819 disp($off);
5820 %}
5821 %}
5822
5823 operand indOffI8(iRegP reg, immIOffset8 off)
5824 %{
5825 constraint(ALLOC_IN_RC(ptr_reg));
5826 match(AddP reg off);
5827 op_cost(0);
5828 format %{ "[$reg, $off]" %}
5829 interface(MEMORY_INTER) %{
5830 base($reg);
5831 index(0xffffffff);
5832 scale(0x0);
5833 disp($off);
5834 %}
5835 %}
5836
5837 operand indOffI16(iRegP reg, immIOffset16 off)
5838 %{
5839 constraint(ALLOC_IN_RC(ptr_reg));
5840 match(AddP reg off);
5841 op_cost(0);
5842 format %{ "[$reg, $off]" %}
5843 interface(MEMORY_INTER) %{
5844 base($reg);
5845 index(0xffffffff);
5846 scale(0x0);
5847 disp($off);
5848 %}
5849 %}
5850
5851 operand indOffL(iRegP reg, immLoffset off)
5852 %{
5853 constraint(ALLOC_IN_RC(ptr_reg));
5854 match(AddP reg off);
5855 op_cost(0);
5856 format %{ "[$reg, $off]" %}
5857 interface(MEMORY_INTER) %{
5858 base($reg);
5859 index(0xffffffff);
5860 scale(0x0);
5861 disp($off);
5862 %}
5863 %}
5864
5865 operand indOffL1(iRegP reg, immLoffset1 off)
5866 %{
5867 constraint(ALLOC_IN_RC(ptr_reg));
5868 match(AddP reg off);
5869 op_cost(0);
5870 format %{ "[$reg, $off]" %}
5871 interface(MEMORY_INTER) %{
5872 base($reg);
5873 index(0xffffffff);
5874 scale(0x0);
5875 disp($off);
5876 %}
5877 %}
5878
5879 operand indOffL2(iRegP reg, immLoffset2 off)
5880 %{
5881 constraint(ALLOC_IN_RC(ptr_reg));
5882 match(AddP reg off);
5883 op_cost(0);
5884 format %{ "[$reg, $off]" %}
5885 interface(MEMORY_INTER) %{
5886 base($reg);
5887 index(0xffffffff);
5888 scale(0x0);
5889 disp($off);
5890 %}
5891 %}
5892
5893 operand indOffL4(iRegP reg, immLoffset4 off)
5894 %{
5895 constraint(ALLOC_IN_RC(ptr_reg));
5896 match(AddP reg off);
5897 op_cost(0);
5898 format %{ "[$reg, $off]" %}
5899 interface(MEMORY_INTER) %{
5900 base($reg);
5901 index(0xffffffff);
5902 scale(0x0);
5903 disp($off);
5904 %}
5905 %}
5906
5907 operand indOffL8(iRegP reg, immLoffset8 off)
5908 %{
5909 constraint(ALLOC_IN_RC(ptr_reg));
5910 match(AddP reg off);
5911 op_cost(0);
5912 format %{ "[$reg, $off]" %}
5913 interface(MEMORY_INTER) %{
5914 base($reg);
5915 index(0xffffffff);
5916 scale(0x0);
5917 disp($off);
5918 %}
5919 %}
5920
5921 operand indOffL16(iRegP reg, immLoffset16 off)
5922 %{
5923 constraint(ALLOC_IN_RC(ptr_reg));
5924 match(AddP reg off);
5925 op_cost(0);
5926 format %{ "[$reg, $off]" %}
5927 interface(MEMORY_INTER) %{
5928 base($reg);
5929 index(0xffffffff);
5930 scale(0x0);
5931 disp($off);
5932 %}
5933 %}
5934
5935 operand indirectN(iRegN reg)
5936 %{
5937 predicate(CompressedOops::shift() == 0);
5938 constraint(ALLOC_IN_RC(ptr_reg));
5939 match(DecodeN reg);
5940 op_cost(0);
5941 format %{ "[$reg]\t# narrow" %}
5942 interface(MEMORY_INTER) %{
5943 base($reg);
5944 index(0xffffffff);
5945 scale(0x0);
5946 disp(0x0);
5947 %}
5948 %}
5949
5950 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5951 %{
5952 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5953 constraint(ALLOC_IN_RC(ptr_reg));
5954 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5955 op_cost(0);
5956 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5957 interface(MEMORY_INTER) %{
5958 base($reg);
5959 index($ireg);
5960 scale($scale);
5961 disp(0x0);
5962 %}
5963 %}
5964
5965 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5966 %{
5967 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5968 constraint(ALLOC_IN_RC(ptr_reg));
5969 match(AddP (DecodeN reg) (LShiftL lreg scale));
5970 op_cost(0);
5971 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5972 interface(MEMORY_INTER) %{
5973 base($reg);
5974 index($lreg);
5975 scale($scale);
5976 disp(0x0);
5977 %}
5978 %}
5979
5980 operand indIndexI2LN(iRegN reg, iRegI ireg)
5981 %{
5982 predicate(CompressedOops::shift() == 0);
5983 constraint(ALLOC_IN_RC(ptr_reg));
5984 match(AddP (DecodeN reg) (ConvI2L ireg));
5985 op_cost(0);
5986 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5987 interface(MEMORY_INTER) %{
5988 base($reg);
5989 index($ireg);
5990 scale(0x0);
5991 disp(0x0);
5992 %}
5993 %}
5994
5995 operand indIndexN(iRegN reg, iRegL lreg)
5996 %{
5997 predicate(CompressedOops::shift() == 0);
5998 constraint(ALLOC_IN_RC(ptr_reg));
5999 match(AddP (DecodeN reg) lreg);
6000 op_cost(0);
6001 format %{ "$reg, $lreg\t# narrow" %}
6002 interface(MEMORY_INTER) %{
6003 base($reg);
6004 index($lreg);
6005 scale(0x0);
6006 disp(0x0);
6007 %}
6008 %}
6009
6010 operand indOffIN(iRegN reg, immIOffset off)
6011 %{
6012 predicate(CompressedOops::shift() == 0);
6013 constraint(ALLOC_IN_RC(ptr_reg));
6014 match(AddP (DecodeN reg) off);
6015 op_cost(0);
6016 format %{ "[$reg, $off]\t# narrow" %}
6017 interface(MEMORY_INTER) %{
6018 base($reg);
6019 index(0xffffffff);
6020 scale(0x0);
6021 disp($off);
6022 %}
6023 %}
6024
6025 operand indOffLN(iRegN reg, immLoffset off)
6026 %{
6027 predicate(CompressedOops::shift() == 0);
6028 constraint(ALLOC_IN_RC(ptr_reg));
6029 match(AddP (DecodeN reg) off);
6030 op_cost(0);
6031 format %{ "[$reg, $off]\t# narrow" %}
6032 interface(MEMORY_INTER) %{
6033 base($reg);
6034 index(0xffffffff);
6035 scale(0x0);
6036 disp($off);
6037 %}
6038 %}
6039
6040
6041
6042 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6043 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6044 %{
6045 constraint(ALLOC_IN_RC(ptr_reg));
6046 match(AddP reg off);
6047 op_cost(0);
6048 format %{ "[$reg, $off]" %}
6049 interface(MEMORY_INTER) %{
6050 base($reg);
6051 index(0xffffffff);
6052 scale(0x0);
6053 disp($off);
6054 %}
6055 %}
6056
6057 //----------Special Memory Operands--------------------------------------------
6058 // Stack Slot Operand - This operand is used for loading and storing temporary
6059 // values on the stack where a match requires a value to
6060 // flow through memory.
6061 operand stackSlotP(sRegP reg)
6062 %{
6063 constraint(ALLOC_IN_RC(stack_slots));
6064 op_cost(100);
6065 // No match rule because this operand is only generated in matching
6066 // match(RegP);
6067 format %{ "[$reg]" %}
6068 interface(MEMORY_INTER) %{
6069 base(0x1e); // RSP
6070 index(0x0); // No Index
6071 scale(0x0); // No Scale
6072 disp($reg); // Stack Offset
6073 %}
6074 %}
6075
6076 operand stackSlotI(sRegI reg)
6077 %{
6078 constraint(ALLOC_IN_RC(stack_slots));
6079 // No match rule because this operand is only generated in matching
6080 // match(RegI);
6081 format %{ "[$reg]" %}
6082 interface(MEMORY_INTER) %{
6083 base(0x1e); // RSP
6084 index(0x0); // No Index
6085 scale(0x0); // No Scale
6086 disp($reg); // Stack Offset
6087 %}
6088 %}
6089
6090 operand stackSlotF(sRegF reg)
6091 %{
6092 constraint(ALLOC_IN_RC(stack_slots));
6093 // No match rule because this operand is only generated in matching
6094 // match(RegF);
6095 format %{ "[$reg]" %}
6096 interface(MEMORY_INTER) %{
6097 base(0x1e); // RSP
6098 index(0x0); // No Index
6099 scale(0x0); // No Scale
6100 disp($reg); // Stack Offset
6101 %}
6102 %}
6103
6104 operand stackSlotD(sRegD reg)
6105 %{
6106 constraint(ALLOC_IN_RC(stack_slots));
6107 // No match rule because this operand is only generated in matching
6108 // match(RegD);
6109 format %{ "[$reg]" %}
6110 interface(MEMORY_INTER) %{
6111 base(0x1e); // RSP
6112 index(0x0); // No Index
6113 scale(0x0); // No Scale
6114 disp($reg); // Stack Offset
6115 %}
6116 %}
6117
6118 operand stackSlotL(sRegL reg)
6119 %{
6120 constraint(ALLOC_IN_RC(stack_slots));
6121 // No match rule because this operand is only generated in matching
6122 // match(RegL);
6123 format %{ "[$reg]" %}
6124 interface(MEMORY_INTER) %{
6125 base(0x1e); // RSP
6126 index(0x0); // No Index
6127 scale(0x0); // No Scale
6128 disp($reg); // Stack Offset
6129 %}
6130 %}
6131
6132 // Operands for expressing Control Flow
6133 // NOTE: Label is a predefined operand which should not be redefined in
6134 // the AD file. It is generically handled within the ADLC.
6135
6136 //----------Conditional Branch Operands----------------------------------------
6137 // Comparison Op - This is the operation of the comparison, and is limited to
6138 // the following set of codes:
6139 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6140 //
6141 // Other attributes of the comparison, such as unsignedness, are specified
6142 // by the comparison instruction that sets a condition code flags register.
6143 // That result is represented by a flags operand whose subtype is appropriate
6144 // to the unsignedness (etc.) of the comparison.
6145 //
6146 // Later, the instruction which matches both the Comparison Op (a Bool) and
6147 // the flags (produced by the Cmp) specifies the coding of the comparison op
6148 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6149
6150 // used for signed integral comparisons and fp comparisons
6151
6152 operand cmpOp()
6153 %{
6154 match(Bool);
6155
6156 format %{ "" %}
6157 interface(COND_INTER) %{
6158 equal(0x0, "eq");
6159 not_equal(0x1, "ne");
6160 less(0xb, "lt");
6161 greater_equal(0xa, "ge");
6162 less_equal(0xd, "le");
6163 greater(0xc, "gt");
6164 overflow(0x6, "vs");
6165 no_overflow(0x7, "vc");
6166 %}
6167 %}
6168
6169 // used for unsigned integral comparisons
6170
6171 operand cmpOpU()
6172 %{
6173 match(Bool);
6174
6175 format %{ "" %}
6176 interface(COND_INTER) %{
6177 equal(0x0, "eq");
6178 not_equal(0x1, "ne");
6179 less(0x3, "lo");
6180 greater_equal(0x2, "hs");
6181 less_equal(0x9, "ls");
6182 greater(0x8, "hi");
6183 overflow(0x6, "vs");
6184 no_overflow(0x7, "vc");
6185 %}
6186 %}
6187
6188 // used for certain integral comparisons which can be
6189 // converted to cbxx or tbxx instructions
6190
6191 operand cmpOpEqNe()
6192 %{
6193 match(Bool);
6194 op_cost(0);
6195 predicate(n->as_Bool()->_test._test == BoolTest::ne
6196 || n->as_Bool()->_test._test == BoolTest::eq);
6197
6198 format %{ "" %}
6199 interface(COND_INTER) %{
6200 equal(0x0, "eq");
6201 not_equal(0x1, "ne");
6202 less(0xb, "lt");
6203 greater_equal(0xa, "ge");
6204 less_equal(0xd, "le");
6205 greater(0xc, "gt");
6206 overflow(0x6, "vs");
6207 no_overflow(0x7, "vc");
6208 %}
6209 %}
6210
6211 // used for certain integral comparisons which can be
6212 // converted to cbxx or tbxx instructions
6213
6214 operand cmpOpLtGe()
6215 %{
6216 match(Bool);
6217 op_cost(0);
6218
6219 predicate(n->as_Bool()->_test._test == BoolTest::lt
6220 || n->as_Bool()->_test._test == BoolTest::ge);
6221
6222 format %{ "" %}
6223 interface(COND_INTER) %{
6224 equal(0x0, "eq");
6225 not_equal(0x1, "ne");
6226 less(0xb, "lt");
6227 greater_equal(0xa, "ge");
6228 less_equal(0xd, "le");
6229 greater(0xc, "gt");
6230 overflow(0x6, "vs");
6231 no_overflow(0x7, "vc");
6232 %}
6233 %}
6234
6235 // used for certain unsigned integral comparisons which can be
6236 // converted to cbxx or tbxx instructions
6237
6238 operand cmpOpUEqNeLtGe()
6239 %{
6240 match(Bool);
6241 op_cost(0);
6242
6243 predicate(n->as_Bool()->_test._test == BoolTest::eq
6244 || n->as_Bool()->_test._test == BoolTest::ne
6245 || n->as_Bool()->_test._test == BoolTest::lt
6246 || n->as_Bool()->_test._test == BoolTest::ge);
6247
6248 format %{ "" %}
6249 interface(COND_INTER) %{
6250 equal(0x0, "eq");
6251 not_equal(0x1, "ne");
6252 less(0xb, "lt");
6253 greater_equal(0xa, "ge");
6254 less_equal(0xd, "le");
6255 greater(0xc, "gt");
6256 overflow(0x6, "vs");
6257 no_overflow(0x7, "vc");
6258 %}
6259 %}
6260
6261 // Special operand allowing long args to int ops to be truncated for free
6262
6263 operand iRegL2I(iRegL reg) %{
6264
6265 op_cost(0);
6266
6267 match(ConvL2I reg);
6268
6269 format %{ "l2i($reg)" %}
6270
6271 interface(REG_INTER)
6272 %}
6273
6274 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6275 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6276 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6277 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6278
6279 //----------OPERAND CLASSES----------------------------------------------------
6280 // Operand Classes are groups of operands that are used as to simplify
6281 // instruction definitions by not requiring the AD writer to specify
6282 // separate instructions for every form of operand when the
6283 // instruction accepts multiple operand types with the same basic
6284 // encoding and format. The classic case of this is memory operands.
6285
6286 // memory is used to define read/write location for load/store
6287 // instruction defs. we can turn a memory op into an Address
6288
6289 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6290 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6291
6292 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6293 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6294
6295 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6296 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6297
6298 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6299 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6300
6301 // All of the memory operands. For the pipeline description.
6302 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6303 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6304 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6305
6306
6307 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6308 // operations. it allows the src to be either an iRegI or a (ConvL2I
6309 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6310 // can be elided because the 32-bit instruction will just employ the
6311 // lower 32 bits anyway.
6312 //
6313 // n.b. this does not elide all L2I conversions. if the truncated
6314 // value is consumed by more than one operation then the ConvL2I
6315 // cannot be bundled into the consuming nodes so an l2i gets planted
6316 // (actually a movw $dst $src) and the downstream instructions consume
6317 // the result of the l2i as an iRegI input. That's a shame since the
6318 // movw is actually redundant but its not too costly.
6319
6320 opclass iRegIorL2I(iRegI, iRegL2I);
6321
6322 //----------PIPELINE-----------------------------------------------------------
6323 // Rules which define the behavior of the target architectures pipeline.
6324
6325 // For specific pipelines, eg A53, define the stages of that pipeline
6326 //pipe_desc(ISS, EX1, EX2, WR);
6327 #define ISS S0
6328 #define EX1 S1
6329 #define EX2 S2
6330 #define WR S3
6331
6332 // Integer ALU reg operation
6333 pipeline %{
6334
6335 attributes %{
6336 // ARM instructions are of fixed length
6337 fixed_size_instructions; // Fixed size instructions TODO does
6338 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6339 // ARM instructions come in 32-bit word units
6340 instruction_unit_size = 4; // An instruction is 4 bytes long
6341 instruction_fetch_unit_size = 64; // The processor fetches one line
6342 instruction_fetch_units = 1; // of 64 bytes
6343
6344 // List of nop instructions
6345 nops( MachNop );
6346 %}
6347
6348 // We don't use an actual pipeline model so don't care about resources
6349 // or description. we do use pipeline classes to introduce fixed
6350 // latencies
6351
6352 //----------RESOURCES----------------------------------------------------------
6353 // Resources are the functional units available to the machine
6354
6355 resources( INS0, INS1, INS01 = INS0 | INS1,
6356 ALU0, ALU1, ALU = ALU0 | ALU1,
6357 MAC,
6358 DIV,
6359 BRANCH,
6360 LDST,
6361 NEON_FP);
6362
6363 //----------PIPELINE DESCRIPTION-----------------------------------------------
6364 // Pipeline Description specifies the stages in the machine's pipeline
6365
6366 // Define the pipeline as a generic 6 stage pipeline
6367 pipe_desc(S0, S1, S2, S3, S4, S5);
6368
6369 //----------PIPELINE CLASSES---------------------------------------------------
6370 // Pipeline Classes describe the stages in which input and output are
6371 // referenced by the hardware pipeline.
6372
6373 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6374 %{
6375 single_instruction;
6376 src1 : S1(read);
6377 src2 : S2(read);
6378 dst : S5(write);
6379 INS01 : ISS;
6380 NEON_FP : S5;
6381 %}
6382
6383 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6384 %{
6385 single_instruction;
6386 src1 : S1(read);
6387 src2 : S2(read);
6388 dst : S5(write);
6389 INS01 : ISS;
6390 NEON_FP : S5;
6391 %}
6392
6393 pipe_class fp_uop_s(vRegF dst, vRegF src)
6394 %{
6395 single_instruction;
6396 src : S1(read);
6397 dst : S5(write);
6398 INS01 : ISS;
6399 NEON_FP : S5;
6400 %}
6401
6402 pipe_class fp_uop_d(vRegD dst, vRegD src)
6403 %{
6404 single_instruction;
6405 src : S1(read);
6406 dst : S5(write);
6407 INS01 : ISS;
6408 NEON_FP : S5;
6409 %}
6410
6411 pipe_class fp_d2f(vRegF dst, vRegD src)
6412 %{
6413 single_instruction;
6414 src : S1(read);
6415 dst : S5(write);
6416 INS01 : ISS;
6417 NEON_FP : S5;
6418 %}
6419
6420 pipe_class fp_f2d(vRegD dst, vRegF src)
6421 %{
6422 single_instruction;
6423 src : S1(read);
6424 dst : S5(write);
6425 INS01 : ISS;
6426 NEON_FP : S5;
6427 %}
6428
6429 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6430 %{
6431 single_instruction;
6432 src : S1(read);
6433 dst : S5(write);
6434 INS01 : ISS;
6435 NEON_FP : S5;
6436 %}
6437
6438 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6439 %{
6440 single_instruction;
6441 src : S1(read);
6442 dst : S5(write);
6443 INS01 : ISS;
6444 NEON_FP : S5;
6445 %}
6446
6447 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6448 %{
6449 single_instruction;
6450 src : S1(read);
6451 dst : S5(write);
6452 INS01 : ISS;
6453 NEON_FP : S5;
6454 %}
6455
6456 pipe_class fp_l2f(vRegF dst, iRegL src)
6457 %{
6458 single_instruction;
6459 src : S1(read);
6460 dst : S5(write);
6461 INS01 : ISS;
6462 NEON_FP : S5;
6463 %}
6464
6465 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6466 %{
6467 single_instruction;
6468 src : S1(read);
6469 dst : S5(write);
6470 INS01 : ISS;
6471 NEON_FP : S5;
6472 %}
6473
6474 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6475 %{
6476 single_instruction;
6477 src : S1(read);
6478 dst : S5(write);
6479 INS01 : ISS;
6480 NEON_FP : S5;
6481 %}
6482
6483 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6484 %{
6485 single_instruction;
6486 src : S1(read);
6487 dst : S5(write);
6488 INS01 : ISS;
6489 NEON_FP : S5;
6490 %}
6491
6492 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6493 %{
6494 single_instruction;
6495 src : S1(read);
6496 dst : S5(write);
6497 INS01 : ISS;
6498 NEON_FP : S5;
6499 %}
6500
6501 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6502 %{
6503 single_instruction;
6504 src1 : S1(read);
6505 src2 : S2(read);
6506 dst : S5(write);
6507 INS0 : ISS;
6508 NEON_FP : S5;
6509 %}
6510
6511 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6512 %{
6513 single_instruction;
6514 src1 : S1(read);
6515 src2 : S2(read);
6516 dst : S5(write);
6517 INS0 : ISS;
6518 NEON_FP : S5;
6519 %}
6520
6521 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6522 %{
6523 single_instruction;
6524 cr : S1(read);
6525 src1 : S1(read);
6526 src2 : S1(read);
6527 dst : S3(write);
6528 INS01 : ISS;
6529 NEON_FP : S3;
6530 %}
6531
6532 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6533 %{
6534 single_instruction;
6535 cr : S1(read);
6536 src1 : S1(read);
6537 src2 : S1(read);
6538 dst : S3(write);
6539 INS01 : ISS;
6540 NEON_FP : S3;
6541 %}
6542
6543 pipe_class fp_imm_s(vRegF dst)
6544 %{
6545 single_instruction;
6546 dst : S3(write);
6547 INS01 : ISS;
6548 NEON_FP : S3;
6549 %}
6550
6551 pipe_class fp_imm_d(vRegD dst)
6552 %{
6553 single_instruction;
6554 dst : S3(write);
6555 INS01 : ISS;
6556 NEON_FP : S3;
6557 %}
6558
6559 pipe_class fp_load_constant_s(vRegF dst)
6560 %{
6561 single_instruction;
6562 dst : S4(write);
6563 INS01 : ISS;
6564 NEON_FP : S4;
6565 %}
6566
6567 pipe_class fp_load_constant_d(vRegD dst)
6568 %{
6569 single_instruction;
6570 dst : S4(write);
6571 INS01 : ISS;
6572 NEON_FP : S4;
6573 %}
6574
6575 //------- Integer ALU operations --------------------------
6576
6577 // Integer ALU reg-reg operation
6578 // Operands needed in EX1, result generated in EX2
6579 // Eg. ADD x0, x1, x2
6580 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6581 %{
6582 single_instruction;
6583 dst : EX2(write);
6584 src1 : EX1(read);
6585 src2 : EX1(read);
6586 INS01 : ISS; // Dual issue as instruction 0 or 1
6587 ALU : EX2;
6588 %}
6589
6590 // Integer ALU reg-reg operation with constant shift
6591 // Shifted register must be available in LATE_ISS instead of EX1
6592 // Eg. ADD x0, x1, x2, LSL #2
6593 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6594 %{
6595 single_instruction;
6596 dst : EX2(write);
6597 src1 : EX1(read);
6598 src2 : ISS(read);
6599 INS01 : ISS;
6600 ALU : EX2;
6601 %}
6602
6603 // Integer ALU reg operation with constant shift
6604 // Eg. LSL x0, x1, #shift
6605 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6606 %{
6607 single_instruction;
6608 dst : EX2(write);
6609 src1 : ISS(read);
6610 INS01 : ISS;
6611 ALU : EX2;
6612 %}
6613
6614 // Integer ALU reg-reg operation with variable shift
6615 // Both operands must be available in LATE_ISS instead of EX1
6616 // Result is available in EX1 instead of EX2
6617 // Eg. LSLV x0, x1, x2
6618 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6619 %{
6620 single_instruction;
6621 dst : EX1(write);
6622 src1 : ISS(read);
6623 src2 : ISS(read);
6624 INS01 : ISS;
6625 ALU : EX1;
6626 %}
6627
6628 // Integer ALU reg-reg operation with extract
6629 // As for _vshift above, but result generated in EX2
6630 // Eg. EXTR x0, x1, x2, #N
6631 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6632 %{
6633 single_instruction;
6634 dst : EX2(write);
6635 src1 : ISS(read);
6636 src2 : ISS(read);
6637 INS1 : ISS; // Can only dual issue as Instruction 1
6638 ALU : EX1;
6639 %}
6640
6641 // Integer ALU reg operation
6642 // Eg. NEG x0, x1
6643 pipe_class ialu_reg(iRegI dst, iRegI src)
6644 %{
6645 single_instruction;
6646 dst : EX2(write);
6647 src : EX1(read);
6648 INS01 : ISS;
6649 ALU : EX2;
6650 %}
6651
6652 // Integer ALU reg mmediate operation
6653 // Eg. ADD x0, x1, #N
6654 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6655 %{
6656 single_instruction;
6657 dst : EX2(write);
6658 src1 : EX1(read);
6659 INS01 : ISS;
6660 ALU : EX2;
6661 %}
6662
6663 // Integer ALU immediate operation (no source operands)
6664 // Eg. MOV x0, #N
6665 pipe_class ialu_imm(iRegI dst)
6666 %{
6667 single_instruction;
6668 dst : EX1(write);
6669 INS01 : ISS;
6670 ALU : EX1;
6671 %}
6672
6673 //------- Compare operation -------------------------------
6674
6675 // Compare reg-reg
6676 // Eg. CMP x0, x1
6677 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6678 %{
6679 single_instruction;
6680 // fixed_latency(16);
6681 cr : EX2(write);
6682 op1 : EX1(read);
6683 op2 : EX1(read);
6684 INS01 : ISS;
6685 ALU : EX2;
6686 %}
6687
6688 // Compare reg-reg
6689 // Eg. CMP x0, #N
6690 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6691 %{
6692 single_instruction;
6693 // fixed_latency(16);
6694 cr : EX2(write);
6695 op1 : EX1(read);
6696 INS01 : ISS;
6697 ALU : EX2;
6698 %}
6699
6700 //------- Conditional instructions ------------------------
6701
6702 // Conditional no operands
6703 // Eg. CSINC x0, zr, zr, <cond>
6704 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6705 %{
6706 single_instruction;
6707 cr : EX1(read);
6708 dst : EX2(write);
6709 INS01 : ISS;
6710 ALU : EX2;
6711 %}
6712
6713 // Conditional 2 operand
6714 // EG. CSEL X0, X1, X2, <cond>
6715 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6716 %{
6717 single_instruction;
6718 cr : EX1(read);
6719 src1 : EX1(read);
6720 src2 : EX1(read);
6721 dst : EX2(write);
6722 INS01 : ISS;
6723 ALU : EX2;
6724 %}
6725
6726 // Conditional 2 operand
6727 // EG. CSEL X0, X1, X2, <cond>
6728 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6729 %{
6730 single_instruction;
6731 cr : EX1(read);
6732 src : EX1(read);
6733 dst : EX2(write);
6734 INS01 : ISS;
6735 ALU : EX2;
6736 %}
6737
6738 //------- Multiply pipeline operations --------------------
6739
6740 // Multiply reg-reg
6741 // Eg. MUL w0, w1, w2
6742 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6743 %{
6744 single_instruction;
6745 dst : WR(write);
6746 src1 : ISS(read);
6747 src2 : ISS(read);
6748 INS01 : ISS;
6749 MAC : WR;
6750 %}
6751
6752 // Multiply accumulate
6753 // Eg. MADD w0, w1, w2, w3
6754 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6755 %{
6756 single_instruction;
6757 dst : WR(write);
6758 src1 : ISS(read);
6759 src2 : ISS(read);
6760 src3 : ISS(read);
6761 INS01 : ISS;
6762 MAC : WR;
6763 %}
6764
6765 // Eg. MUL w0, w1, w2
6766 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6767 %{
6768 single_instruction;
6769 fixed_latency(3); // Maximum latency for 64 bit mul
6770 dst : WR(write);
6771 src1 : ISS(read);
6772 src2 : ISS(read);
6773 INS01 : ISS;
6774 MAC : WR;
6775 %}
6776
6777 // Multiply accumulate
6778 // Eg. MADD w0, w1, w2, w3
6779 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6780 %{
6781 single_instruction;
6782 fixed_latency(3); // Maximum latency for 64 bit mul
6783 dst : WR(write);
6784 src1 : ISS(read);
6785 src2 : ISS(read);
6786 src3 : ISS(read);
6787 INS01 : ISS;
6788 MAC : WR;
6789 %}
6790
6791 //------- Divide pipeline operations --------------------
6792
6793 // Eg. SDIV w0, w1, w2
6794 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6795 %{
6796 single_instruction;
6797 fixed_latency(8); // Maximum latency for 32 bit divide
6798 dst : WR(write);
6799 src1 : ISS(read);
6800 src2 : ISS(read);
6801 INS0 : ISS; // Can only dual issue as instruction 0
6802 DIV : WR;
6803 %}
6804
6805 // Eg. SDIV x0, x1, x2
6806 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6807 %{
6808 single_instruction;
6809 fixed_latency(16); // Maximum latency for 64 bit divide
6810 dst : WR(write);
6811 src1 : ISS(read);
6812 src2 : ISS(read);
6813 INS0 : ISS; // Can only dual issue as instruction 0
6814 DIV : WR;
6815 %}
6816
6817 //------- Load pipeline operations ------------------------
6818
6819 // Load - prefetch
6820 // Eg. PFRM <mem>
6821 pipe_class iload_prefetch(memory mem)
6822 %{
6823 single_instruction;
6824 mem : ISS(read);
6825 INS01 : ISS;
6826 LDST : WR;
6827 %}
6828
6829 // Load - reg, mem
6830 // Eg. LDR x0, <mem>
6831 pipe_class iload_reg_mem(iRegI dst, memory mem)
6832 %{
6833 single_instruction;
6834 dst : WR(write);
6835 mem : ISS(read);
6836 INS01 : ISS;
6837 LDST : WR;
6838 %}
6839
6840 // Load - reg, reg
6841 // Eg. LDR x0, [sp, x1]
6842 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6843 %{
6844 single_instruction;
6845 dst : WR(write);
6846 src : ISS(read);
6847 INS01 : ISS;
6848 LDST : WR;
6849 %}
6850
6851 //------- Store pipeline operations -----------------------
6852
6853 // Store - zr, mem
6854 // Eg. STR zr, <mem>
6855 pipe_class istore_mem(memory mem)
6856 %{
6857 single_instruction;
6858 mem : ISS(read);
6859 INS01 : ISS;
6860 LDST : WR;
6861 %}
6862
6863 // Store - reg, mem
6864 // Eg. STR x0, <mem>
6865 pipe_class istore_reg_mem(iRegI src, memory mem)
6866 %{
6867 single_instruction;
6868 mem : ISS(read);
6869 src : EX2(read);
6870 INS01 : ISS;
6871 LDST : WR;
6872 %}
6873
6874 // Store - reg, reg
6875 // Eg. STR x0, [sp, x1]
6876 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6877 %{
6878 single_instruction;
6879 dst : ISS(read);
6880 src : EX2(read);
6881 INS01 : ISS;
6882 LDST : WR;
6883 %}
6884
6885 //------- Store pipeline operations -----------------------
6886
6887 // Branch
6888 pipe_class pipe_branch()
6889 %{
6890 single_instruction;
6891 INS01 : ISS;
6892 BRANCH : EX1;
6893 %}
6894
6895 // Conditional branch
6896 pipe_class pipe_branch_cond(rFlagsReg cr)
6897 %{
6898 single_instruction;
6899 cr : EX1(read);
6900 INS01 : ISS;
6901 BRANCH : EX1;
6902 %}
6903
6904 // Compare & Branch
6905 // EG. CBZ/CBNZ
6906 pipe_class pipe_cmp_branch(iRegI op1)
6907 %{
6908 single_instruction;
6909 op1 : EX1(read);
6910 INS01 : ISS;
6911 BRANCH : EX1;
6912 %}
6913
6914 //------- Synchronisation operations ----------------------
6915
6916 // Any operation requiring serialization.
6917 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6918 pipe_class pipe_serial()
6919 %{
6920 single_instruction;
6921 force_serialization;
6922 fixed_latency(16);
6923 INS01 : ISS(2); // Cannot dual issue with any other instruction
6924 LDST : WR;
6925 %}
6926
6927 // Generic big/slow expanded idiom - also serialized
6928 pipe_class pipe_slow()
6929 %{
6930 instruction_count(10);
6931 multiple_bundles;
6932 force_serialization;
6933 fixed_latency(16);
6934 INS01 : ISS(2); // Cannot dual issue with any other instruction
6935 LDST : WR;
6936 %}
6937
6938 // Empty pipeline class
6939 pipe_class pipe_class_empty()
6940 %{
6941 single_instruction;
6942 fixed_latency(0);
6943 %}
6944
6945 // Default pipeline class.
6946 pipe_class pipe_class_default()
6947 %{
6948 single_instruction;
6949 fixed_latency(2);
6950 %}
6951
6952 // Pipeline class for compares.
6953 pipe_class pipe_class_compare()
6954 %{
6955 single_instruction;
6956 fixed_latency(16);
6957 %}
6958
6959 // Pipeline class for memory operations.
6960 pipe_class pipe_class_memory()
6961 %{
6962 single_instruction;
6963 fixed_latency(16);
6964 %}
6965
6966 // Pipeline class for call.
6967 pipe_class pipe_class_call()
6968 %{
6969 single_instruction;
6970 fixed_latency(100);
6971 %}
6972
6973 // Define the class for the Nop node.
6974 define %{
6975 MachNop = pipe_class_empty;
6976 %}
6977
6978 %}
6979 //----------INSTRUCTIONS-------------------------------------------------------
6980 //
6981 // match -- States which machine-independent subtree may be replaced
6982 // by this instruction.
6983 // ins_cost -- The estimated cost of this instruction is used by instruction
6984 // selection to identify a minimum cost tree of machine
6985 // instructions that matches a tree of machine-independent
6986 // instructions.
6987 // format -- A string providing the disassembly for this instruction.
6988 // The value of an instruction's operand may be inserted
6989 // by referring to it with a '$' prefix.
6990 // opcode -- Three instruction opcodes may be provided. These are referred
6991 // to within an encode class as $primary, $secondary, and $tertiary
6992 // rrspectively. The primary opcode is commonly used to
6993 // indicate the type of machine instruction, while secondary
6994 // and tertiary are often used for prefix options or addressing
6995 // modes.
6996 // ins_encode -- A list of encode classes with parameters. The encode class
6997 // name must have been defined in an 'enc_class' specification
6998 // in the encode section of the architecture description.
6999
7000 // ============================================================================
7001 // Memory (Load/Store) Instructions
7002
7003 // Load Instructions
7004
7005 // Load Byte (8 bit signed)
7006 instruct loadB(iRegINoSp dst, memory1 mem)
7007 %{
7008 match(Set dst (LoadB mem));
7009 predicate(!needs_acquiring_load(n));
7010
7011 ins_cost(4 * INSN_COST);
7012 format %{ "ldrsbw $dst, $mem\t# byte" %}
7013
7014 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7015
7016 ins_pipe(iload_reg_mem);
7017 %}
7018
7019 // Load Byte (8 bit signed) into long
7020 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7021 %{
7022 match(Set dst (ConvI2L (LoadB mem)));
7023 predicate(!needs_acquiring_load(n->in(1)));
7024
7025 ins_cost(4 * INSN_COST);
7026 format %{ "ldrsb $dst, $mem\t# byte" %}
7027
7028 ins_encode(aarch64_enc_ldrsb(dst, mem));
7029
7030 ins_pipe(iload_reg_mem);
7031 %}
7032
7033 // Load Byte (8 bit unsigned)
7034 instruct loadUB(iRegINoSp dst, memory1 mem)
7035 %{
7036 match(Set dst (LoadUB mem));
7037 predicate(!needs_acquiring_load(n));
7038
7039 ins_cost(4 * INSN_COST);
7040 format %{ "ldrbw $dst, $mem\t# byte" %}
7041
7042 ins_encode(aarch64_enc_ldrb(dst, mem));
7043
7044 ins_pipe(iload_reg_mem);
7045 %}
7046
7047 // Load Byte (8 bit unsigned) into long
7048 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7049 %{
7050 match(Set dst (ConvI2L (LoadUB mem)));
7051 predicate(!needs_acquiring_load(n->in(1)));
7052
7053 ins_cost(4 * INSN_COST);
7054 format %{ "ldrb $dst, $mem\t# byte" %}
7055
7056 ins_encode(aarch64_enc_ldrb(dst, mem));
7057
7058 ins_pipe(iload_reg_mem);
7059 %}
7060
7061 // Load Short (16 bit signed)
7062 instruct loadS(iRegINoSp dst, memory2 mem)
7063 %{
7064 match(Set dst (LoadS mem));
7065 predicate(!needs_acquiring_load(n));
7066
7067 ins_cost(4 * INSN_COST);
7068 format %{ "ldrshw $dst, $mem\t# short" %}
7069
7070 ins_encode(aarch64_enc_ldrshw(dst, mem));
7071
7072 ins_pipe(iload_reg_mem);
7073 %}
7074
7075 // Load Short (16 bit signed) into long
7076 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7077 %{
7078 match(Set dst (ConvI2L (LoadS mem)));
7079 predicate(!needs_acquiring_load(n->in(1)));
7080
7081 ins_cost(4 * INSN_COST);
7082 format %{ "ldrsh $dst, $mem\t# short" %}
7083
7084 ins_encode(aarch64_enc_ldrsh(dst, mem));
7085
7086 ins_pipe(iload_reg_mem);
7087 %}
7088
7089 // Load Char (16 bit unsigned)
7090 instruct loadUS(iRegINoSp dst, memory2 mem)
7091 %{
7092 match(Set dst (LoadUS mem));
7093 predicate(!needs_acquiring_load(n));
7094
7095 ins_cost(4 * INSN_COST);
7096 format %{ "ldrh $dst, $mem\t# short" %}
7097
7098 ins_encode(aarch64_enc_ldrh(dst, mem));
7099
7100 ins_pipe(iload_reg_mem);
7101 %}
7102
7103 // Load Short/Char (16 bit unsigned) into long
7104 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7105 %{
7106 match(Set dst (ConvI2L (LoadUS mem)));
7107 predicate(!needs_acquiring_load(n->in(1)));
7108
7109 ins_cost(4 * INSN_COST);
7110 format %{ "ldrh $dst, $mem\t# short" %}
7111
7112 ins_encode(aarch64_enc_ldrh(dst, mem));
7113
7114 ins_pipe(iload_reg_mem);
7115 %}
7116
7117 // Load Integer (32 bit signed)
7118 instruct loadI(iRegINoSp dst, memory4 mem)
7119 %{
7120 match(Set dst (LoadI mem));
7121 predicate(!needs_acquiring_load(n));
7122
7123 ins_cost(4 * INSN_COST);
7124 format %{ "ldrw $dst, $mem\t# int" %}
7125
7126 ins_encode(aarch64_enc_ldrw(dst, mem));
7127
7128 ins_pipe(iload_reg_mem);
7129 %}
7130
7131 // Load Integer (32 bit signed) into long
7132 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7133 %{
7134 match(Set dst (ConvI2L (LoadI mem)));
7135 predicate(!needs_acquiring_load(n->in(1)));
7136
7137 ins_cost(4 * INSN_COST);
7138 format %{ "ldrsw $dst, $mem\t# int" %}
7139
7140 ins_encode(aarch64_enc_ldrsw(dst, mem));
7141
7142 ins_pipe(iload_reg_mem);
7143 %}
7144
7145 // Load Integer (32 bit unsigned) into long
7146 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7147 %{
7148 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7149 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7150
7151 ins_cost(4 * INSN_COST);
7152 format %{ "ldrw $dst, $mem\t# int" %}
7153
7154 ins_encode(aarch64_enc_ldrw(dst, mem));
7155
7156 ins_pipe(iload_reg_mem);
7157 %}
7158
7159 // Load Long (64 bit signed)
7160 instruct loadL(iRegLNoSp dst, memory8 mem)
7161 %{
7162 match(Set dst (LoadL mem));
7163 predicate(!needs_acquiring_load(n));
7164
7165 ins_cost(4 * INSN_COST);
7166 format %{ "ldr $dst, $mem\t# int" %}
7167
7168 ins_encode(aarch64_enc_ldr(dst, mem));
7169
7170 ins_pipe(iload_reg_mem);
7171 %}
7172
7173 // Load Range
7174 instruct loadRange(iRegINoSp dst, memory4 mem)
7175 %{
7176 match(Set dst (LoadRange mem));
7177
7178 ins_cost(4 * INSN_COST);
7179 format %{ "ldrw $dst, $mem\t# range" %}
7180
7181 ins_encode(aarch64_enc_ldrw(dst, mem));
7182
7183 ins_pipe(iload_reg_mem);
7184 %}
7185
7186 // Load Pointer
7187 instruct loadP(iRegPNoSp dst, memory8 mem)
7188 %{
7189 match(Set dst (LoadP mem));
7190 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7191
7192 ins_cost(4 * INSN_COST);
7193 format %{ "ldr $dst, $mem\t# ptr" %}
7194
7195 ins_encode(aarch64_enc_ldr(dst, mem));
7196
7197 ins_pipe(iload_reg_mem);
7198 %}
7199
7200 // Load Compressed Pointer
7201 instruct loadN(iRegNNoSp dst, memory4 mem)
7202 %{
7203 match(Set dst (LoadN mem));
7204 predicate(!needs_acquiring_load(n));
7205
7206 ins_cost(4 * INSN_COST);
7207 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7208
7209 ins_encode(aarch64_enc_ldrw(dst, mem));
7210
7211 ins_pipe(iload_reg_mem);
7212 %}
7213
7214 // Load Klass Pointer
7215 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7216 %{
7217 match(Set dst (LoadKlass mem));
7218 predicate(!needs_acquiring_load(n));
7219
7220 ins_cost(4 * INSN_COST);
7221 format %{ "ldr $dst, $mem\t# class" %}
7222
7223 ins_encode(aarch64_enc_ldr(dst, mem));
7224
7225 ins_pipe(iload_reg_mem);
7226 %}
7227
7228 // Load Narrow Klass Pointer
7229 instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
7230 %{
7231 match(Set dst (LoadNKlass mem));
7232 effect(TEMP_DEF dst, KILL cr);
7233 predicate(!needs_acquiring_load(n));
7234
7235 ins_cost(4 * INSN_COST);
7236 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7237 ins_encode %{
7238 assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
7239 assert($mem$$index$$Register == noreg, "expect no index");
7240 Register dst = $dst$$Register;
7241 Register obj = $mem$$base$$Register;
7242 C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
7243 Compile::current()->output()->add_stub(stub);
7244 __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
7245 // NOTE: We can't use tbnz here, because the target is sometimes too far away
7246 // and cannot be encoded.
7247 __ tst(dst, markWord::monitor_value);
7248 __ br(Assembler::NE, stub->entry());
7249 __ bind(stub->continuation());
7250 __ lsr(dst, dst, markWord::klass_shift);
7251 %}
7252 ins_pipe(pipe_slow);
7253 %}
7254
7255 // Load Float
7256 instruct loadF(vRegF dst, memory4 mem)
7257 %{
7258 match(Set dst (LoadF mem));
7259 predicate(!needs_acquiring_load(n));
7260
7261 ins_cost(4 * INSN_COST);
7262 format %{ "ldrs $dst, $mem\t# float" %}
7263
7264 ins_encode( aarch64_enc_ldrs(dst, mem) );
7265
7266 ins_pipe(pipe_class_memory);
7267 %}
7268
7269 // Load Double
7270 instruct loadD(vRegD dst, memory8 mem)
7271 %{
7272 match(Set dst (LoadD mem));
7273 predicate(!needs_acquiring_load(n));
7274
7275 ins_cost(4 * INSN_COST);
7276 format %{ "ldrd $dst, $mem\t# double" %}
7277
7278 ins_encode( aarch64_enc_ldrd(dst, mem) );
7279
7280 ins_pipe(pipe_class_memory);
7281 %}
7282
7283
7284 // Load Int Constant
7285 instruct loadConI(iRegINoSp dst, immI src)
7286 %{
7287 match(Set dst src);
7288
7289 ins_cost(INSN_COST);
7290 format %{ "mov $dst, $src\t# int" %}
7291
7292 ins_encode( aarch64_enc_movw_imm(dst, src) );
7293
7294 ins_pipe(ialu_imm);
7295 %}
7296
7297 // Load Long Constant
7298 instruct loadConL(iRegLNoSp dst, immL src)
7299 %{
7300 match(Set dst src);
7301
7302 ins_cost(INSN_COST);
7303 format %{ "mov $dst, $src\t# long" %}
7304
7305 ins_encode( aarch64_enc_mov_imm(dst, src) );
7306
7307 ins_pipe(ialu_imm);
7308 %}
7309
7310 // Load Pointer Constant
7311
7312 instruct loadConP(iRegPNoSp dst, immP con)
7313 %{
7314 match(Set dst con);
7315
7316 ins_cost(INSN_COST * 4);
7317 format %{
7318 "mov $dst, $con\t# ptr\n\t"
7319 %}
7320
7321 ins_encode(aarch64_enc_mov_p(dst, con));
7322
7323 ins_pipe(ialu_imm);
7324 %}
7325
7326 // Load Null Pointer Constant
7327
7328 instruct loadConP0(iRegPNoSp dst, immP0 con)
7329 %{
7330 match(Set dst con);
7331
7332 ins_cost(INSN_COST);
7333 format %{ "mov $dst, $con\t# NULL ptr" %}
7334
7335 ins_encode(aarch64_enc_mov_p0(dst, con));
7336
7337 ins_pipe(ialu_imm);
7338 %}
7339
7340 // Load Pointer Constant One
7341
7342 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7343 %{
7344 match(Set dst con);
7345
7346 ins_cost(INSN_COST);
7347 format %{ "mov $dst, $con\t# NULL ptr" %}
7348
7349 ins_encode(aarch64_enc_mov_p1(dst, con));
7350
7351 ins_pipe(ialu_imm);
7352 %}
7353
7354 // Load Byte Map Base Constant
7355
7356 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7357 %{
7358 match(Set dst con);
7359
7360 ins_cost(INSN_COST);
7361 format %{ "adr $dst, $con\t# Byte Map Base" %}
7362
7363 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7364
7365 ins_pipe(ialu_imm);
7366 %}
7367
7368 // Load Narrow Pointer Constant
7369
7370 instruct loadConN(iRegNNoSp dst, immN con)
7371 %{
7372 match(Set dst con);
7373
7374 ins_cost(INSN_COST * 4);
7375 format %{ "mov $dst, $con\t# compressed ptr" %}
7376
7377 ins_encode(aarch64_enc_mov_n(dst, con));
7378
7379 ins_pipe(ialu_imm);
7380 %}
7381
7382 // Load Narrow Null Pointer Constant
7383
7384 instruct loadConN0(iRegNNoSp dst, immN0 con)
7385 %{
7386 match(Set dst con);
7387
7388 ins_cost(INSN_COST);
7389 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7390
7391 ins_encode(aarch64_enc_mov_n0(dst, con));
7392
7393 ins_pipe(ialu_imm);
7394 %}
7395
7396 // Load Narrow Klass Constant
7397
7398 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7399 %{
7400 match(Set dst con);
7401
7402 ins_cost(INSN_COST);
7403 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7404
7405 ins_encode(aarch64_enc_mov_nk(dst, con));
7406
7407 ins_pipe(ialu_imm);
7408 %}
7409
7410 // Load Packed Float Constant
7411
7412 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7413 match(Set dst con);
7414 ins_cost(INSN_COST * 4);
7415 format %{ "fmovs $dst, $con"%}
7416 ins_encode %{
7417 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7418 %}
7419
7420 ins_pipe(fp_imm_s);
7421 %}
7422
7423 // Load Float Constant
7424
7425 instruct loadConF(vRegF dst, immF con) %{
7426 match(Set dst con);
7427
7428 ins_cost(INSN_COST * 4);
7429
7430 format %{
7431 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7432 %}
7433
7434 ins_encode %{
7435 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7436 %}
7437
7438 ins_pipe(fp_load_constant_s);
7439 %}
7440
7441 // Load Packed Double Constant
7442
7443 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7444 match(Set dst con);
7445 ins_cost(INSN_COST);
7446 format %{ "fmovd $dst, $con"%}
7447 ins_encode %{
7448 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7449 %}
7450
7451 ins_pipe(fp_imm_d);
7452 %}
7453
7454 // Load Double Constant
7455
7456 instruct loadConD(vRegD dst, immD con) %{
7457 match(Set dst con);
7458
7459 ins_cost(INSN_COST * 5);
7460 format %{
7461 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7462 %}
7463
7464 ins_encode %{
7465 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7466 %}
7467
7468 ins_pipe(fp_load_constant_d);
7469 %}
7470
7471 // Store Instructions
7472
7473 // Store CMS card-mark Immediate
7474 instruct storeimmCM0(immI0 zero, memory1 mem)
7475 %{
7476 match(Set mem (StoreCM mem zero));
7477
7478 ins_cost(INSN_COST);
7479 format %{ "storestore (elided)\n\t"
7480 "strb zr, $mem\t# byte" %}
7481
7482 ins_encode(aarch64_enc_strb0(mem));
7483
7484 ins_pipe(istore_mem);
7485 %}
7486
7487 // Store CMS card-mark Immediate with intervening StoreStore
7488 // needed when using CMS with no conditional card marking
7489 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7490 %{
7491 match(Set mem (StoreCM mem zero));
7492
7493 ins_cost(INSN_COST * 2);
7494 format %{ "storestore\n\t"
7495 "dmb ishst"
7496 "\n\tstrb zr, $mem\t# byte" %}
7497
7498 ins_encode(aarch64_enc_strb0_ordered(mem));
7499
7500 ins_pipe(istore_mem);
7501 %}
7502
7503 // Store Byte
7504 instruct storeB(iRegIorL2I src, memory1 mem)
7505 %{
7506 match(Set mem (StoreB mem src));
7507 predicate(!needs_releasing_store(n));
7508
7509 ins_cost(INSN_COST);
7510 format %{ "strb $src, $mem\t# byte" %}
7511
7512 ins_encode(aarch64_enc_strb(src, mem));
7513
7514 ins_pipe(istore_reg_mem);
7515 %}
7516
7517
7518 instruct storeimmB0(immI0 zero, memory1 mem)
7519 %{
7520 match(Set mem (StoreB mem zero));
7521 predicate(!needs_releasing_store(n));
7522
7523 ins_cost(INSN_COST);
7524 format %{ "strb rscractch2, $mem\t# byte" %}
7525
7526 ins_encode(aarch64_enc_strb0(mem));
7527
7528 ins_pipe(istore_mem);
7529 %}
7530
7531 // Store Char/Short
7532 instruct storeC(iRegIorL2I src, memory2 mem)
7533 %{
7534 match(Set mem (StoreC mem src));
7535 predicate(!needs_releasing_store(n));
7536
7537 ins_cost(INSN_COST);
7538 format %{ "strh $src, $mem\t# short" %}
7539
7540 ins_encode(aarch64_enc_strh(src, mem));
7541
7542 ins_pipe(istore_reg_mem);
7543 %}
7544
7545 instruct storeimmC0(immI0 zero, memory2 mem)
7546 %{
7547 match(Set mem (StoreC mem zero));
7548 predicate(!needs_releasing_store(n));
7549
7550 ins_cost(INSN_COST);
7551 format %{ "strh zr, $mem\t# short" %}
7552
7553 ins_encode(aarch64_enc_strh0(mem));
7554
7555 ins_pipe(istore_mem);
7556 %}
7557
7558 // Store Integer
7559
7560 instruct storeI(iRegIorL2I src, memory4 mem)
7561 %{
7562 match(Set mem(StoreI mem src));
7563 predicate(!needs_releasing_store(n));
7564
7565 ins_cost(INSN_COST);
7566 format %{ "strw $src, $mem\t# int" %}
7567
7568 ins_encode(aarch64_enc_strw(src, mem));
7569
7570 ins_pipe(istore_reg_mem);
7571 %}
7572
7573 instruct storeimmI0(immI0 zero, memory4 mem)
7574 %{
7575 match(Set mem(StoreI mem zero));
7576 predicate(!needs_releasing_store(n));
7577
7578 ins_cost(INSN_COST);
7579 format %{ "strw zr, $mem\t# int" %}
7580
7581 ins_encode(aarch64_enc_strw0(mem));
7582
7583 ins_pipe(istore_mem);
7584 %}
7585
7586 // Store Long (64 bit signed)
7587 instruct storeL(iRegL src, memory8 mem)
7588 %{
7589 match(Set mem (StoreL mem src));
7590 predicate(!needs_releasing_store(n));
7591
7592 ins_cost(INSN_COST);
7593 format %{ "str $src, $mem\t# int" %}
7594
7595 ins_encode(aarch64_enc_str(src, mem));
7596
7597 ins_pipe(istore_reg_mem);
7598 %}
7599
7600 // Store Long (64 bit signed)
7601 instruct storeimmL0(immL0 zero, memory8 mem)
7602 %{
7603 match(Set mem (StoreL mem zero));
7604 predicate(!needs_releasing_store(n));
7605
7606 ins_cost(INSN_COST);
7607 format %{ "str zr, $mem\t# int" %}
7608
7609 ins_encode(aarch64_enc_str0(mem));
7610
7611 ins_pipe(istore_mem);
7612 %}
7613
7614 // Store Pointer
7615 instruct storeP(iRegP src, memory8 mem)
7616 %{
7617 match(Set mem (StoreP mem src));
7618 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7619
7620 ins_cost(INSN_COST);
7621 format %{ "str $src, $mem\t# ptr" %}
7622
7623 ins_encode(aarch64_enc_str(src, mem));
7624
7625 ins_pipe(istore_reg_mem);
7626 %}
7627
7628 // Store Pointer
7629 instruct storeimmP0(immP0 zero, memory8 mem)
7630 %{
7631 match(Set mem (StoreP mem zero));
7632 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7633
7634 ins_cost(INSN_COST);
7635 format %{ "str zr, $mem\t# ptr" %}
7636
7637 ins_encode(aarch64_enc_str0(mem));
7638
7639 ins_pipe(istore_mem);
7640 %}
7641
7642 // Store Compressed Pointer
7643 instruct storeN(iRegN src, memory4 mem)
7644 %{
7645 match(Set mem (StoreN mem src));
7646 predicate(!needs_releasing_store(n));
7647
7648 ins_cost(INSN_COST);
7649 format %{ "strw $src, $mem\t# compressed ptr" %}
7650
7651 ins_encode(aarch64_enc_strw(src, mem));
7652
7653 ins_pipe(istore_reg_mem);
7654 %}
7655
7656 instruct storeImmN0(immN0 zero, memory4 mem)
7657 %{
7658 match(Set mem (StoreN mem zero));
7659 predicate(!needs_releasing_store(n));
7660
7661 ins_cost(INSN_COST);
7662 format %{ "strw zr, $mem\t# compressed ptr" %}
7663
7664 ins_encode(aarch64_enc_strw0(mem));
7665
7666 ins_pipe(istore_mem);
7667 %}
7668
7669 // Store Float
7670 instruct storeF(vRegF src, memory4 mem)
7671 %{
7672 match(Set mem (StoreF mem src));
7673 predicate(!needs_releasing_store(n));
7674
7675 ins_cost(INSN_COST);
7676 format %{ "strs $src, $mem\t# float" %}
7677
7678 ins_encode( aarch64_enc_strs(src, mem) );
7679
7680 ins_pipe(pipe_class_memory);
7681 %}
7682
7683 // TODO
7684 // implement storeImmF0 and storeFImmPacked
7685
7686 // Store Double
7687 instruct storeD(vRegD src, memory8 mem)
7688 %{
7689 match(Set mem (StoreD mem src));
7690 predicate(!needs_releasing_store(n));
7691
7692 ins_cost(INSN_COST);
7693 format %{ "strd $src, $mem\t# double" %}
7694
7695 ins_encode( aarch64_enc_strd(src, mem) );
7696
7697 ins_pipe(pipe_class_memory);
7698 %}
7699
7700 // Store Compressed Klass Pointer
7701 instruct storeNKlass(iRegN src, memory4 mem)
7702 %{
7703 predicate(!needs_releasing_store(n));
7704 match(Set mem (StoreNKlass mem src));
7705
7706 ins_cost(INSN_COST);
7707 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7708
7709 ins_encode(aarch64_enc_strw(src, mem));
7710
7711 ins_pipe(istore_reg_mem);
7712 %}
7713
7714 // TODO
7715 // implement storeImmD0 and storeDImmPacked
7716
7717 // prefetch instructions
7718 // Must be safe to execute with invalid address (cannot fault).
7719
7720 instruct prefetchalloc( memory8 mem ) %{
7721 match(PrefetchAllocation mem);
7722
7723 ins_cost(INSN_COST);
7724 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7725
7726 ins_encode( aarch64_enc_prefetchw(mem) );
7727
7728 ins_pipe(iload_prefetch);
7729 %}
7730
7731 // ---------------- volatile loads and stores ----------------
7732
7733 // Load Byte (8 bit signed)
7734 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7735 %{
7736 match(Set dst (LoadB mem));
7737
7738 ins_cost(VOLATILE_REF_COST);
7739 format %{ "ldarsb $dst, $mem\t# byte" %}
7740
7741 ins_encode(aarch64_enc_ldarsb(dst, mem));
7742
7743 ins_pipe(pipe_serial);
7744 %}
7745
7746 // Load Byte (8 bit signed) into long
7747 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7748 %{
7749 match(Set dst (ConvI2L (LoadB mem)));
7750
7751 ins_cost(VOLATILE_REF_COST);
7752 format %{ "ldarsb $dst, $mem\t# byte" %}
7753
7754 ins_encode(aarch64_enc_ldarsb(dst, mem));
7755
7756 ins_pipe(pipe_serial);
7757 %}
7758
7759 // Load Byte (8 bit unsigned)
7760 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7761 %{
7762 match(Set dst (LoadUB mem));
7763
7764 ins_cost(VOLATILE_REF_COST);
7765 format %{ "ldarb $dst, $mem\t# byte" %}
7766
7767 ins_encode(aarch64_enc_ldarb(dst, mem));
7768
7769 ins_pipe(pipe_serial);
7770 %}
7771
7772 // Load Byte (8 bit unsigned) into long
7773 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7774 %{
7775 match(Set dst (ConvI2L (LoadUB mem)));
7776
7777 ins_cost(VOLATILE_REF_COST);
7778 format %{ "ldarb $dst, $mem\t# byte" %}
7779
7780 ins_encode(aarch64_enc_ldarb(dst, mem));
7781
7782 ins_pipe(pipe_serial);
7783 %}
7784
7785 // Load Short (16 bit signed)
7786 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7787 %{
7788 match(Set dst (LoadS mem));
7789
7790 ins_cost(VOLATILE_REF_COST);
7791 format %{ "ldarshw $dst, $mem\t# short" %}
7792
7793 ins_encode(aarch64_enc_ldarshw(dst, mem));
7794
7795 ins_pipe(pipe_serial);
7796 %}
7797
7798 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7799 %{
7800 match(Set dst (LoadUS mem));
7801
7802 ins_cost(VOLATILE_REF_COST);
7803 format %{ "ldarhw $dst, $mem\t# short" %}
7804
7805 ins_encode(aarch64_enc_ldarhw(dst, mem));
7806
7807 ins_pipe(pipe_serial);
7808 %}
7809
7810 // Load Short/Char (16 bit unsigned) into long
7811 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7812 %{
7813 match(Set dst (ConvI2L (LoadUS mem)));
7814
7815 ins_cost(VOLATILE_REF_COST);
7816 format %{ "ldarh $dst, $mem\t# short" %}
7817
7818 ins_encode(aarch64_enc_ldarh(dst, mem));
7819
7820 ins_pipe(pipe_serial);
7821 %}
7822
7823 // Load Short/Char (16 bit signed) into long
7824 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7825 %{
7826 match(Set dst (ConvI2L (LoadS mem)));
7827
7828 ins_cost(VOLATILE_REF_COST);
7829 format %{ "ldarh $dst, $mem\t# short" %}
7830
7831 ins_encode(aarch64_enc_ldarsh(dst, mem));
7832
7833 ins_pipe(pipe_serial);
7834 %}
7835
7836 // Load Integer (32 bit signed)
7837 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7838 %{
7839 match(Set dst (LoadI mem));
7840
7841 ins_cost(VOLATILE_REF_COST);
7842 format %{ "ldarw $dst, $mem\t# int" %}
7843
7844 ins_encode(aarch64_enc_ldarw(dst, mem));
7845
7846 ins_pipe(pipe_serial);
7847 %}
7848
7849 // Load Integer (32 bit unsigned) into long
7850 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7851 %{
7852 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7853
7854 ins_cost(VOLATILE_REF_COST);
7855 format %{ "ldarw $dst, $mem\t# int" %}
7856
7857 ins_encode(aarch64_enc_ldarw(dst, mem));
7858
7859 ins_pipe(pipe_serial);
7860 %}
7861
7862 // Load Long (64 bit signed)
7863 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7864 %{
7865 match(Set dst (LoadL mem));
7866
7867 ins_cost(VOLATILE_REF_COST);
7868 format %{ "ldar $dst, $mem\t# int" %}
7869
7870 ins_encode(aarch64_enc_ldar(dst, mem));
7871
7872 ins_pipe(pipe_serial);
7873 %}
7874
7875 // Load Pointer
7876 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7877 %{
7878 match(Set dst (LoadP mem));
7879 predicate(n->as_Load()->barrier_data() == 0);
7880
7881 ins_cost(VOLATILE_REF_COST);
7882 format %{ "ldar $dst, $mem\t# ptr" %}
7883
7884 ins_encode(aarch64_enc_ldar(dst, mem));
7885
7886 ins_pipe(pipe_serial);
7887 %}
7888
7889 // Load Compressed Pointer
7890 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7891 %{
7892 match(Set dst (LoadN mem));
7893
7894 ins_cost(VOLATILE_REF_COST);
7895 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7896
7897 ins_encode(aarch64_enc_ldarw(dst, mem));
7898
7899 ins_pipe(pipe_serial);
7900 %}
7901
7902 // Load Float
7903 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7904 %{
7905 match(Set dst (LoadF mem));
7906
7907 ins_cost(VOLATILE_REF_COST);
7908 format %{ "ldars $dst, $mem\t# float" %}
7909
7910 ins_encode( aarch64_enc_fldars(dst, mem) );
7911
7912 ins_pipe(pipe_serial);
7913 %}
7914
7915 // Load Double
7916 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7917 %{
7918 match(Set dst (LoadD mem));
7919
7920 ins_cost(VOLATILE_REF_COST);
7921 format %{ "ldard $dst, $mem\t# double" %}
7922
7923 ins_encode( aarch64_enc_fldard(dst, mem) );
7924
7925 ins_pipe(pipe_serial);
7926 %}
7927
7928 // Store Byte
7929 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7930 %{
7931 match(Set mem (StoreB mem src));
7932
7933 ins_cost(VOLATILE_REF_COST);
7934 format %{ "stlrb $src, $mem\t# byte" %}
7935
7936 ins_encode(aarch64_enc_stlrb(src, mem));
7937
7938 ins_pipe(pipe_class_memory);
7939 %}
7940
7941 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7942 %{
7943 match(Set mem (StoreB mem zero));
7944
7945 ins_cost(VOLATILE_REF_COST);
7946 format %{ "stlrb zr, $mem\t# byte" %}
7947
7948 ins_encode(aarch64_enc_stlrb0(mem));
7949
7950 ins_pipe(pipe_class_memory);
7951 %}
7952
7953 // Store Char/Short
7954 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7955 %{
7956 match(Set mem (StoreC mem src));
7957
7958 ins_cost(VOLATILE_REF_COST);
7959 format %{ "stlrh $src, $mem\t# short" %}
7960
7961 ins_encode(aarch64_enc_stlrh(src, mem));
7962
7963 ins_pipe(pipe_class_memory);
7964 %}
7965
7966 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7967 %{
7968 match(Set mem (StoreC mem zero));
7969
7970 ins_cost(VOLATILE_REF_COST);
7971 format %{ "stlrh zr, $mem\t# short" %}
7972
7973 ins_encode(aarch64_enc_stlrh0(mem));
7974
7975 ins_pipe(pipe_class_memory);
7976 %}
7977
7978 // Store Integer
7979
7980 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7981 %{
7982 match(Set mem(StoreI mem src));
7983
7984 ins_cost(VOLATILE_REF_COST);
7985 format %{ "stlrw $src, $mem\t# int" %}
7986
7987 ins_encode(aarch64_enc_stlrw(src, mem));
7988
7989 ins_pipe(pipe_class_memory);
7990 %}
7991
7992 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7993 %{
7994 match(Set mem(StoreI mem zero));
7995
7996 ins_cost(VOLATILE_REF_COST);
7997 format %{ "stlrw zr, $mem\t# int" %}
7998
7999 ins_encode(aarch64_enc_stlrw0(mem));
8000
8001 ins_pipe(pipe_class_memory);
8002 %}
8003
8004 // Store Long (64 bit signed)
8005 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8006 %{
8007 match(Set mem (StoreL mem src));
8008
8009 ins_cost(VOLATILE_REF_COST);
8010 format %{ "stlr $src, $mem\t# int" %}
8011
8012 ins_encode(aarch64_enc_stlr(src, mem));
8013
8014 ins_pipe(pipe_class_memory);
8015 %}
8016
8017 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8018 %{
8019 match(Set mem (StoreL mem zero));
8020
8021 ins_cost(VOLATILE_REF_COST);
8022 format %{ "stlr zr, $mem\t# int" %}
8023
8024 ins_encode(aarch64_enc_stlr0(mem));
8025
8026 ins_pipe(pipe_class_memory);
8027 %}
8028
8029 // Store Pointer
8030 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8031 %{
8032 match(Set mem (StoreP mem src));
8033 predicate(n->as_Store()->barrier_data() == 0);
8034
8035 ins_cost(VOLATILE_REF_COST);
8036 format %{ "stlr $src, $mem\t# ptr" %}
8037
8038 ins_encode(aarch64_enc_stlr(src, mem));
8039
8040 ins_pipe(pipe_class_memory);
8041 %}
8042
8043 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8044 %{
8045 match(Set mem (StoreP mem zero));
8046 predicate(n->as_Store()->barrier_data() == 0);
8047
8048 ins_cost(VOLATILE_REF_COST);
8049 format %{ "stlr zr, $mem\t# ptr" %}
8050
8051 ins_encode(aarch64_enc_stlr0(mem));
8052
8053 ins_pipe(pipe_class_memory);
8054 %}
8055
8056 // Store Compressed Pointer
8057 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8058 %{
8059 match(Set mem (StoreN mem src));
8060
8061 ins_cost(VOLATILE_REF_COST);
8062 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8063
8064 ins_encode(aarch64_enc_stlrw(src, mem));
8065
8066 ins_pipe(pipe_class_memory);
8067 %}
8068
8069 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8070 %{
8071 match(Set mem (StoreN mem zero));
8072
8073 ins_cost(VOLATILE_REF_COST);
8074 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8075
8076 ins_encode(aarch64_enc_stlrw0(mem));
8077
8078 ins_pipe(pipe_class_memory);
8079 %}
8080
8081 // Store Float
8082 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8083 %{
8084 match(Set mem (StoreF mem src));
8085
8086 ins_cost(VOLATILE_REF_COST);
8087 format %{ "stlrs $src, $mem\t# float" %}
8088
8089 ins_encode( aarch64_enc_fstlrs(src, mem) );
8090
8091 ins_pipe(pipe_class_memory);
8092 %}
8093
8094 // TODO
8095 // implement storeImmF0 and storeFImmPacked
8096
8097 // Store Double
8098 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8099 %{
8100 match(Set mem (StoreD mem src));
8101
8102 ins_cost(VOLATILE_REF_COST);
8103 format %{ "stlrd $src, $mem\t# double" %}
8104
8105 ins_encode( aarch64_enc_fstlrd(src, mem) );
8106
8107 ins_pipe(pipe_class_memory);
8108 %}
8109
8110 // ---------------- end of volatile loads and stores ----------------
8111
8112 instruct cacheWB(indirect addr)
8113 %{
8114 predicate(VM_Version::supports_data_cache_line_flush());
8115 match(CacheWB addr);
8116
8117 ins_cost(100);
8118 format %{"cache wb $addr" %}
8119 ins_encode %{
8120 assert($addr->index_position() < 0, "should be");
8121 assert($addr$$disp == 0, "should be");
8122 __ cache_wb(Address($addr$$base$$Register, 0));
8123 %}
8124 ins_pipe(pipe_slow); // XXX
8125 %}
8126
8127 instruct cacheWBPreSync()
8128 %{
8129 predicate(VM_Version::supports_data_cache_line_flush());
8130 match(CacheWBPreSync);
8131
8132 ins_cost(100);
8133 format %{"cache wb presync" %}
8134 ins_encode %{
8135 __ cache_wbsync(true);
8136 %}
8137 ins_pipe(pipe_slow); // XXX
8138 %}
8139
8140 instruct cacheWBPostSync()
8141 %{
8142 predicate(VM_Version::supports_data_cache_line_flush());
8143 match(CacheWBPostSync);
8144
8145 ins_cost(100);
8146 format %{"cache wb postsync" %}
8147 ins_encode %{
8148 __ cache_wbsync(false);
8149 %}
8150 ins_pipe(pipe_slow); // XXX
8151 %}
8152
8153 // ============================================================================
8154 // BSWAP Instructions
8155
8156 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8157 match(Set dst (ReverseBytesI src));
8158
8159 ins_cost(INSN_COST);
8160 format %{ "revw $dst, $src" %}
8161
8162 ins_encode %{
8163 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8164 %}
8165
8166 ins_pipe(ialu_reg);
8167 %}
8168
8169 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8170 match(Set dst (ReverseBytesL src));
8171
8172 ins_cost(INSN_COST);
8173 format %{ "rev $dst, $src" %}
8174
8175 ins_encode %{
8176 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8177 %}
8178
8179 ins_pipe(ialu_reg);
8180 %}
8181
8182 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8183 match(Set dst (ReverseBytesUS src));
8184
8185 ins_cost(INSN_COST);
8186 format %{ "rev16w $dst, $src" %}
8187
8188 ins_encode %{
8189 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8190 %}
8191
8192 ins_pipe(ialu_reg);
8193 %}
8194
8195 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8196 match(Set dst (ReverseBytesS src));
8197
8198 ins_cost(INSN_COST);
8199 format %{ "rev16w $dst, $src\n\t"
8200 "sbfmw $dst, $dst, #0, #15" %}
8201
8202 ins_encode %{
8203 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8204 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8205 %}
8206
8207 ins_pipe(ialu_reg);
8208 %}
8209
8210 // ============================================================================
8211 // Zero Count Instructions
8212
8213 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8214 match(Set dst (CountLeadingZerosI src));
8215
8216 ins_cost(INSN_COST);
8217 format %{ "clzw $dst, $src" %}
8218 ins_encode %{
8219 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8220 %}
8221
8222 ins_pipe(ialu_reg);
8223 %}
8224
8225 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8226 match(Set dst (CountLeadingZerosL src));
8227
8228 ins_cost(INSN_COST);
8229 format %{ "clz $dst, $src" %}
8230 ins_encode %{
8231 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8232 %}
8233
8234 ins_pipe(ialu_reg);
8235 %}
8236
8237 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8238 match(Set dst (CountTrailingZerosI src));
8239
8240 ins_cost(INSN_COST * 2);
8241 format %{ "rbitw $dst, $src\n\t"
8242 "clzw $dst, $dst" %}
8243 ins_encode %{
8244 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8245 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8246 %}
8247
8248 ins_pipe(ialu_reg);
8249 %}
8250
8251 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8252 match(Set dst (CountTrailingZerosL src));
8253
8254 ins_cost(INSN_COST * 2);
8255 format %{ "rbit $dst, $src\n\t"
8256 "clz $dst, $dst" %}
8257 ins_encode %{
8258 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8259 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8260 %}
8261
8262 ins_pipe(ialu_reg);
8263 %}
8264
8265 //---------- Population Count Instructions -------------------------------------
8266 //
8267
8268 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8269 match(Set dst (PopCountI src));
8270 effect(TEMP tmp);
8271 ins_cost(INSN_COST * 13);
8272
8273 format %{ "movw $src, $src\n\t"
8274 "mov $tmp, $src\t# vector (1D)\n\t"
8275 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8276 "addv $tmp, $tmp\t# vector (8B)\n\t"
8277 "mov $dst, $tmp\t# vector (1D)" %}
8278 ins_encode %{
8279 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8280 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8281 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8282 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8283 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8284 %}
8285
8286 ins_pipe(pipe_class_default);
8287 %}
8288
8289 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8290 match(Set dst (PopCountI (LoadI mem)));
8291 effect(TEMP tmp);
8292 ins_cost(INSN_COST * 13);
8293
8294 format %{ "ldrs $tmp, $mem\n\t"
8295 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8296 "addv $tmp, $tmp\t# vector (8B)\n\t"
8297 "mov $dst, $tmp\t# vector (1D)" %}
8298 ins_encode %{
8299 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8300 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8301 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8302 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8303 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8304 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8305 %}
8306
8307 ins_pipe(pipe_class_default);
8308 %}
8309
8310 // Note: Long.bitCount(long) returns an int.
8311 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8312 match(Set dst (PopCountL src));
8313 effect(TEMP tmp);
8314 ins_cost(INSN_COST * 13);
8315
8316 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8317 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8318 "addv $tmp, $tmp\t# vector (8B)\n\t"
8319 "mov $dst, $tmp\t# vector (1D)" %}
8320 ins_encode %{
8321 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8322 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8323 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8324 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8325 %}
8326
8327 ins_pipe(pipe_class_default);
8328 %}
8329
8330 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8331 match(Set dst (PopCountL (LoadL mem)));
8332 effect(TEMP tmp);
8333 ins_cost(INSN_COST * 13);
8334
8335 format %{ "ldrd $tmp, $mem\n\t"
8336 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8337 "addv $tmp, $tmp\t# vector (8B)\n\t"
8338 "mov $dst, $tmp\t# vector (1D)" %}
8339 ins_encode %{
8340 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8341 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8342 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8343 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8344 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8345 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8346 %}
8347
8348 ins_pipe(pipe_class_default);
8349 %}
8350
8351 // ============================================================================
8352 // MemBar Instruction
8353
8354 instruct load_fence() %{
8355 match(LoadFence);
8356 ins_cost(VOLATILE_REF_COST);
8357
8358 format %{ "load_fence" %}
8359
8360 ins_encode %{
8361 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8362 %}
8363 ins_pipe(pipe_serial);
8364 %}
8365
8366 instruct unnecessary_membar_acquire() %{
8367 predicate(unnecessary_acquire(n));
8368 match(MemBarAcquire);
8369 ins_cost(0);
8370
8371 format %{ "membar_acquire (elided)" %}
8372
8373 ins_encode %{
8374 __ block_comment("membar_acquire (elided)");
8375 %}
8376
8377 ins_pipe(pipe_class_empty);
8378 %}
8379
8380 instruct membar_acquire() %{
8381 match(MemBarAcquire);
8382 ins_cost(VOLATILE_REF_COST);
8383
8384 format %{ "membar_acquire\n\t"
8385 "dmb ish" %}
8386
8387 ins_encode %{
8388 __ block_comment("membar_acquire");
8389 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8390 %}
8391
8392 ins_pipe(pipe_serial);
8393 %}
8394
8395
8396 instruct membar_acquire_lock() %{
8397 match(MemBarAcquireLock);
8398 ins_cost(VOLATILE_REF_COST);
8399
8400 format %{ "membar_acquire_lock (elided)" %}
8401
8402 ins_encode %{
8403 __ block_comment("membar_acquire_lock (elided)");
8404 %}
8405
8406 ins_pipe(pipe_serial);
8407 %}
8408
8409 instruct store_fence() %{
8410 match(StoreFence);
8411 ins_cost(VOLATILE_REF_COST);
8412
8413 format %{ "store_fence" %}
8414
8415 ins_encode %{
8416 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8417 %}
8418 ins_pipe(pipe_serial);
8419 %}
8420
8421 instruct unnecessary_membar_release() %{
8422 predicate(unnecessary_release(n));
8423 match(MemBarRelease);
8424 ins_cost(0);
8425
8426 format %{ "membar_release (elided)" %}
8427
8428 ins_encode %{
8429 __ block_comment("membar_release (elided)");
8430 %}
8431 ins_pipe(pipe_serial);
8432 %}
8433
8434 instruct membar_release() %{
8435 match(MemBarRelease);
8436 ins_cost(VOLATILE_REF_COST);
8437
8438 format %{ "membar_release\n\t"
8439 "dmb ish" %}
8440
8441 ins_encode %{
8442 __ block_comment("membar_release");
8443 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8444 %}
8445 ins_pipe(pipe_serial);
8446 %}
8447
8448 instruct membar_storestore() %{
8449 match(MemBarStoreStore);
8450 match(StoreStoreFence);
8451 ins_cost(VOLATILE_REF_COST);
8452
8453 format %{ "MEMBAR-store-store" %}
8454
8455 ins_encode %{
8456 __ membar(Assembler::StoreStore);
8457 %}
8458 ins_pipe(pipe_serial);
8459 %}
8460
8461 instruct membar_release_lock() %{
8462 match(MemBarReleaseLock);
8463 ins_cost(VOLATILE_REF_COST);
8464
8465 format %{ "membar_release_lock (elided)" %}
8466
8467 ins_encode %{
8468 __ block_comment("membar_release_lock (elided)");
8469 %}
8470
8471 ins_pipe(pipe_serial);
8472 %}
8473
8474 instruct unnecessary_membar_volatile() %{
8475 predicate(unnecessary_volatile(n));
8476 match(MemBarVolatile);
8477 ins_cost(0);
8478
8479 format %{ "membar_volatile (elided)" %}
8480
8481 ins_encode %{
8482 __ block_comment("membar_volatile (elided)");
8483 %}
8484
8485 ins_pipe(pipe_serial);
8486 %}
8487
8488 instruct membar_volatile() %{
8489 match(MemBarVolatile);
8490 ins_cost(VOLATILE_REF_COST*100);
8491
8492 format %{ "membar_volatile\n\t"
8493 "dmb ish"%}
8494
8495 ins_encode %{
8496 __ block_comment("membar_volatile");
8497 __ membar(Assembler::StoreLoad);
8498 %}
8499
8500 ins_pipe(pipe_serial);
8501 %}
8502
8503 // ============================================================================
8504 // Cast/Convert Instructions
8505
8506 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8507 match(Set dst (CastX2P src));
8508
8509 ins_cost(INSN_COST);
8510 format %{ "mov $dst, $src\t# long -> ptr" %}
8511
8512 ins_encode %{
8513 if ($dst$$reg != $src$$reg) {
8514 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8515 }
8516 %}
8517
8518 ins_pipe(ialu_reg);
8519 %}
8520
8521 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8522 match(Set dst (CastP2X src));
8523
8524 ins_cost(INSN_COST);
8525 format %{ "mov $dst, $src\t# ptr -> long" %}
8526
8527 ins_encode %{
8528 if ($dst$$reg != $src$$reg) {
8529 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8530 }
8531 %}
8532
8533 ins_pipe(ialu_reg);
8534 %}
8535
8536 // Convert oop into int for vectors alignment masking
8537 instruct convP2I(iRegINoSp dst, iRegP src) %{
8538 match(Set dst (ConvL2I (CastP2X src)));
8539
8540 ins_cost(INSN_COST);
8541 format %{ "movw $dst, $src\t# ptr -> int" %}
8542 ins_encode %{
8543 __ movw($dst$$Register, $src$$Register);
8544 %}
8545
8546 ins_pipe(ialu_reg);
8547 %}
8548
8549 // Convert compressed oop into int for vectors alignment masking
8550 // in case of 32bit oops (heap < 4Gb).
8551 instruct convN2I(iRegINoSp dst, iRegN src)
8552 %{
8553 predicate(CompressedOops::shift() == 0);
8554 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8555
8556 ins_cost(INSN_COST);
8557 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8558 ins_encode %{
8559 __ movw($dst$$Register, $src$$Register);
8560 %}
8561
8562 ins_pipe(ialu_reg);
8563 %}
8564
8565
8566 // Convert oop pointer into compressed form
8567 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8568 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8569 match(Set dst (EncodeP src));
8570 effect(KILL cr);
8571 ins_cost(INSN_COST * 3);
8572 format %{ "encode_heap_oop $dst, $src" %}
8573 ins_encode %{
8574 Register s = $src$$Register;
8575 Register d = $dst$$Register;
8576 __ encode_heap_oop(d, s);
8577 %}
8578 ins_pipe(ialu_reg);
8579 %}
8580
8581 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8582 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8583 match(Set dst (EncodeP src));
8584 ins_cost(INSN_COST * 3);
8585 format %{ "encode_heap_oop_not_null $dst, $src" %}
8586 ins_encode %{
8587 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8588 %}
8589 ins_pipe(ialu_reg);
8590 %}
8591
8592 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8593 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8594 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8595 match(Set dst (DecodeN src));
8596 ins_cost(INSN_COST * 3);
8597 format %{ "decode_heap_oop $dst, $src" %}
8598 ins_encode %{
8599 Register s = $src$$Register;
8600 Register d = $dst$$Register;
8601 __ decode_heap_oop(d, s);
8602 %}
8603 ins_pipe(ialu_reg);
8604 %}
8605
8606 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8607 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8608 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8609 match(Set dst (DecodeN src));
8610 ins_cost(INSN_COST * 3);
8611 format %{ "decode_heap_oop_not_null $dst, $src" %}
8612 ins_encode %{
8613 Register s = $src$$Register;
8614 Register d = $dst$$Register;
8615 __ decode_heap_oop_not_null(d, s);
8616 %}
8617 ins_pipe(ialu_reg);
8618 %}
8619
8620 // n.b. AArch64 implementations of encode_klass_not_null and
8621 // decode_klass_not_null do not modify the flags register so, unlike
8622 // Intel, we don't kill CR as a side effect here
8623
8624 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8625 match(Set dst (EncodePKlass src));
8626
8627 ins_cost(INSN_COST * 3);
8628 format %{ "encode_klass_not_null $dst,$src" %}
8629
8630 ins_encode %{
8631 Register src_reg = as_Register($src$$reg);
8632 Register dst_reg = as_Register($dst$$reg);
8633 __ encode_klass_not_null(dst_reg, src_reg);
8634 %}
8635
8636 ins_pipe(ialu_reg);
8637 %}
8638
8639 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8640 match(Set dst (DecodeNKlass src));
8641
8642 ins_cost(INSN_COST * 3);
8643 format %{ "decode_klass_not_null $dst,$src" %}
8644
8645 ins_encode %{
8646 Register src_reg = as_Register($src$$reg);
8647 Register dst_reg = as_Register($dst$$reg);
8648 if (dst_reg != src_reg) {
8649 __ decode_klass_not_null(dst_reg, src_reg);
8650 } else {
8651 __ decode_klass_not_null(dst_reg);
8652 }
8653 %}
8654
8655 ins_pipe(ialu_reg);
8656 %}
8657
8658 instruct checkCastPP(iRegPNoSp dst)
8659 %{
8660 match(Set dst (CheckCastPP dst));
8661
8662 size(0);
8663 format %{ "# checkcastPP of $dst" %}
8664 ins_encode(/* empty encoding */);
8665 ins_pipe(pipe_class_empty);
8666 %}
8667
8668 instruct castPP(iRegPNoSp dst)
8669 %{
8670 match(Set dst (CastPP dst));
8671
8672 size(0);
8673 format %{ "# castPP of $dst" %}
8674 ins_encode(/* empty encoding */);
8675 ins_pipe(pipe_class_empty);
8676 %}
8677
8678 instruct castII(iRegI dst)
8679 %{
8680 match(Set dst (CastII dst));
8681
8682 size(0);
8683 format %{ "# castII of $dst" %}
8684 ins_encode(/* empty encoding */);
8685 ins_cost(0);
8686 ins_pipe(pipe_class_empty);
8687 %}
8688
8689 instruct castLL(iRegL dst)
8690 %{
8691 match(Set dst (CastLL dst));
8692
8693 size(0);
8694 format %{ "# castLL of $dst" %}
8695 ins_encode(/* empty encoding */);
8696 ins_cost(0);
8697 ins_pipe(pipe_class_empty);
8698 %}
8699
8700 instruct castFF(vRegF dst)
8701 %{
8702 match(Set dst (CastFF dst));
8703
8704 size(0);
8705 format %{ "# castFF of $dst" %}
8706 ins_encode(/* empty encoding */);
8707 ins_cost(0);
8708 ins_pipe(pipe_class_empty);
8709 %}
8710
8711 instruct castDD(vRegD dst)
8712 %{
8713 match(Set dst (CastDD dst));
8714
8715 size(0);
8716 format %{ "# castDD of $dst" %}
8717 ins_encode(/* empty encoding */);
8718 ins_cost(0);
8719 ins_pipe(pipe_class_empty);
8720 %}
8721
8722 instruct castVV(vReg dst)
8723 %{
8724 match(Set dst (CastVV dst));
8725
8726 size(0);
8727 format %{ "# castVV of $dst" %}
8728 ins_encode(/* empty encoding */);
8729 ins_cost(0);
8730 ins_pipe(pipe_class_empty);
8731 %}
8732
8733 instruct castVVMask(pRegGov dst)
8734 %{
8735 match(Set dst (CastVV dst));
8736
8737 size(0);
8738 format %{ "# castVV of $dst" %}
8739 ins_encode(/* empty encoding */);
8740 ins_cost(0);
8741 ins_pipe(pipe_class_empty);
8742 %}
8743
8744 // ============================================================================
8745 // Atomic operation instructions
8746 //
8747
8748 // standard CompareAndSwapX when we are using barriers
8749 // these have higher priority than the rules selected by a predicate
8750
8751 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8752 // can't match them
8753
8754 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8755
8756 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8757 ins_cost(2 * VOLATILE_REF_COST);
8758
8759 effect(KILL cr);
8760
8761 format %{
8762 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8763 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8764 %}
8765
8766 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8767 aarch64_enc_cset_eq(res));
8768
8769 ins_pipe(pipe_slow);
8770 %}
8771
8772 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8773
8774 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8775 ins_cost(2 * VOLATILE_REF_COST);
8776
8777 effect(KILL cr);
8778
8779 format %{
8780 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8781 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8782 %}
8783
8784 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8785 aarch64_enc_cset_eq(res));
8786
8787 ins_pipe(pipe_slow);
8788 %}
8789
8790 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8791
8792 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8793 ins_cost(2 * VOLATILE_REF_COST);
8794
8795 effect(KILL cr);
8796
8797 format %{
8798 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8799 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8800 %}
8801
8802 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8803 aarch64_enc_cset_eq(res));
8804
8805 ins_pipe(pipe_slow);
8806 %}
8807
8808 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8809
8810 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8811 ins_cost(2 * VOLATILE_REF_COST);
8812
8813 effect(KILL cr);
8814
8815 format %{
8816 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8817 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8818 %}
8819
8820 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8821 aarch64_enc_cset_eq(res));
8822
8823 ins_pipe(pipe_slow);
8824 %}
8825
8826 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8827
8828 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8829 predicate(n->as_LoadStore()->barrier_data() == 0);
8830 ins_cost(2 * VOLATILE_REF_COST);
8831
8832 effect(KILL cr);
8833
8834 format %{
8835 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8836 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8837 %}
8838
8839 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8840 aarch64_enc_cset_eq(res));
8841
8842 ins_pipe(pipe_slow);
8843 %}
8844
8845 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8846
8847 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8848 ins_cost(2 * VOLATILE_REF_COST);
8849
8850 effect(KILL cr);
8851
8852 format %{
8853 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8854 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8855 %}
8856
8857 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8858 aarch64_enc_cset_eq(res));
8859
8860 ins_pipe(pipe_slow);
8861 %}
8862
8863 // alternative CompareAndSwapX when we are eliding barriers
8864
8865 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8866
8867 predicate(needs_acquiring_load_exclusive(n));
8868 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8869 ins_cost(VOLATILE_REF_COST);
8870
8871 effect(KILL cr);
8872
8873 format %{
8874 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8875 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8876 %}
8877
8878 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8879 aarch64_enc_cset_eq(res));
8880
8881 ins_pipe(pipe_slow);
8882 %}
8883
8884 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8885
8886 predicate(needs_acquiring_load_exclusive(n));
8887 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8888 ins_cost(VOLATILE_REF_COST);
8889
8890 effect(KILL cr);
8891
8892 format %{
8893 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8894 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8895 %}
8896
8897 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8898 aarch64_enc_cset_eq(res));
8899
8900 ins_pipe(pipe_slow);
8901 %}
8902
8903 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8904
8905 predicate(needs_acquiring_load_exclusive(n));
8906 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8907 ins_cost(VOLATILE_REF_COST);
8908
8909 effect(KILL cr);
8910
8911 format %{
8912 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8913 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8914 %}
8915
8916 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8917 aarch64_enc_cset_eq(res));
8918
8919 ins_pipe(pipe_slow);
8920 %}
8921
8922 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8923
8924 predicate(needs_acquiring_load_exclusive(n));
8925 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8926 ins_cost(VOLATILE_REF_COST);
8927
8928 effect(KILL cr);
8929
8930 format %{
8931 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8932 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8933 %}
8934
8935 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8936 aarch64_enc_cset_eq(res));
8937
8938 ins_pipe(pipe_slow);
8939 %}
8940
8941 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8942
8943 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8944 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8945 ins_cost(VOLATILE_REF_COST);
8946
8947 effect(KILL cr);
8948
8949 format %{
8950 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8951 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8952 %}
8953
8954 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8955 aarch64_enc_cset_eq(res));
8956
8957 ins_pipe(pipe_slow);
8958 %}
8959
8960 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8961
8962 predicate(needs_acquiring_load_exclusive(n));
8963 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8964 ins_cost(VOLATILE_REF_COST);
8965
8966 effect(KILL cr);
8967
8968 format %{
8969 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8970 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8971 %}
8972
8973 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8974 aarch64_enc_cset_eq(res));
8975
8976 ins_pipe(pipe_slow);
8977 %}
8978
8979
8980 // ---------------------------------------------------------------------
8981
8982 // BEGIN This section of the file is automatically generated. Do not edit --------------
8983
8984 // Sundry CAS operations. Note that release is always true,
8985 // regardless of the memory ordering of the CAS. This is because we
8986 // need the volatile case to be sequentially consistent but there is
8987 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8988 // can't check the type of memory ordering here, so we always emit a
8989 // STLXR.
8990
8991 // This section is generated from aarch64_ad_cas.m4
8992
8993
8994
8995 // This pattern is generated automatically from cas.m4.
8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8997 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8998
8999 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9000 ins_cost(2 * VOLATILE_REF_COST);
9001 effect(TEMP_DEF res, KILL cr);
9002 format %{
9003 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9004 %}
9005 ins_encode %{
9006 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9007 Assembler::byte, /*acquire*/ false, /*release*/ true,
9008 /*weak*/ false, $res$$Register);
9009 __ sxtbw($res$$Register, $res$$Register);
9010 %}
9011 ins_pipe(pipe_slow);
9012 %}
9013
9014 // This pattern is generated automatically from cas.m4.
9015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9016 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9017
9018 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9019 ins_cost(2 * VOLATILE_REF_COST);
9020 effect(TEMP_DEF res, KILL cr);
9021 format %{
9022 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9023 %}
9024 ins_encode %{
9025 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9026 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9027 /*weak*/ false, $res$$Register);
9028 __ sxthw($res$$Register, $res$$Register);
9029 %}
9030 ins_pipe(pipe_slow);
9031 %}
9032
9033 // This pattern is generated automatically from cas.m4.
9034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9035 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9036
9037 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9038 ins_cost(2 * VOLATILE_REF_COST);
9039 effect(TEMP_DEF res, KILL cr);
9040 format %{
9041 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9042 %}
9043 ins_encode %{
9044 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9045 Assembler::word, /*acquire*/ false, /*release*/ true,
9046 /*weak*/ false, $res$$Register);
9047 %}
9048 ins_pipe(pipe_slow);
9049 %}
9050
9051 // This pattern is generated automatically from cas.m4.
9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9053 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9054
9055 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9056 ins_cost(2 * VOLATILE_REF_COST);
9057 effect(TEMP_DEF res, KILL cr);
9058 format %{
9059 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9060 %}
9061 ins_encode %{
9062 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9063 Assembler::xword, /*acquire*/ false, /*release*/ true,
9064 /*weak*/ false, $res$$Register);
9065 %}
9066 ins_pipe(pipe_slow);
9067 %}
9068
9069 // This pattern is generated automatically from cas.m4.
9070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9071 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9072
9073 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9074 ins_cost(2 * VOLATILE_REF_COST);
9075 effect(TEMP_DEF res, KILL cr);
9076 format %{
9077 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9078 %}
9079 ins_encode %{
9080 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9081 Assembler::word, /*acquire*/ false, /*release*/ true,
9082 /*weak*/ false, $res$$Register);
9083 %}
9084 ins_pipe(pipe_slow);
9085 %}
9086
9087 // This pattern is generated automatically from cas.m4.
9088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9089 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9090 predicate(n->as_LoadStore()->barrier_data() == 0);
9091 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9092 ins_cost(2 * VOLATILE_REF_COST);
9093 effect(TEMP_DEF res, KILL cr);
9094 format %{
9095 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9096 %}
9097 ins_encode %{
9098 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9099 Assembler::xword, /*acquire*/ false, /*release*/ true,
9100 /*weak*/ false, $res$$Register);
9101 %}
9102 ins_pipe(pipe_slow);
9103 %}
9104
9105 // This pattern is generated automatically from cas.m4.
9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9107 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9108 predicate(needs_acquiring_load_exclusive(n));
9109 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9110 ins_cost(VOLATILE_REF_COST);
9111 effect(TEMP_DEF res, KILL cr);
9112 format %{
9113 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9114 %}
9115 ins_encode %{
9116 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9117 Assembler::byte, /*acquire*/ true, /*release*/ true,
9118 /*weak*/ false, $res$$Register);
9119 __ sxtbw($res$$Register, $res$$Register);
9120 %}
9121 ins_pipe(pipe_slow);
9122 %}
9123
9124 // This pattern is generated automatically from cas.m4.
9125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9126 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9127 predicate(needs_acquiring_load_exclusive(n));
9128 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9129 ins_cost(VOLATILE_REF_COST);
9130 effect(TEMP_DEF res, KILL cr);
9131 format %{
9132 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9133 %}
9134 ins_encode %{
9135 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9136 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9137 /*weak*/ false, $res$$Register);
9138 __ sxthw($res$$Register, $res$$Register);
9139 %}
9140 ins_pipe(pipe_slow);
9141 %}
9142
9143 // This pattern is generated automatically from cas.m4.
9144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9145 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9146 predicate(needs_acquiring_load_exclusive(n));
9147 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9148 ins_cost(VOLATILE_REF_COST);
9149 effect(TEMP_DEF res, KILL cr);
9150 format %{
9151 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9152 %}
9153 ins_encode %{
9154 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9155 Assembler::word, /*acquire*/ true, /*release*/ true,
9156 /*weak*/ false, $res$$Register);
9157 %}
9158 ins_pipe(pipe_slow);
9159 %}
9160
9161 // This pattern is generated automatically from cas.m4.
9162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9163 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9164 predicate(needs_acquiring_load_exclusive(n));
9165 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9166 ins_cost(VOLATILE_REF_COST);
9167 effect(TEMP_DEF res, KILL cr);
9168 format %{
9169 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9170 %}
9171 ins_encode %{
9172 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9173 Assembler::xword, /*acquire*/ true, /*release*/ true,
9174 /*weak*/ false, $res$$Register);
9175 %}
9176 ins_pipe(pipe_slow);
9177 %}
9178
9179 // This pattern is generated automatically from cas.m4.
9180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9181 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9182 predicate(needs_acquiring_load_exclusive(n));
9183 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9184 ins_cost(VOLATILE_REF_COST);
9185 effect(TEMP_DEF res, KILL cr);
9186 format %{
9187 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9188 %}
9189 ins_encode %{
9190 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9191 Assembler::word, /*acquire*/ true, /*release*/ true,
9192 /*weak*/ false, $res$$Register);
9193 %}
9194 ins_pipe(pipe_slow);
9195 %}
9196
9197 // This pattern is generated automatically from cas.m4.
9198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9199 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9200 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9201 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9202 ins_cost(VOLATILE_REF_COST);
9203 effect(TEMP_DEF res, KILL cr);
9204 format %{
9205 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9206 %}
9207 ins_encode %{
9208 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9209 Assembler::xword, /*acquire*/ true, /*release*/ true,
9210 /*weak*/ false, $res$$Register);
9211 %}
9212 ins_pipe(pipe_slow);
9213 %}
9214
9215 // This pattern is generated automatically from cas.m4.
9216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9217 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9218
9219 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9220 ins_cost(2 * VOLATILE_REF_COST);
9221 effect(KILL cr);
9222 format %{
9223 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9224 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9225 %}
9226 ins_encode %{
9227 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9228 Assembler::byte, /*acquire*/ false, /*release*/ true,
9229 /*weak*/ true, noreg);
9230 __ csetw($res$$Register, Assembler::EQ);
9231 %}
9232 ins_pipe(pipe_slow);
9233 %}
9234
9235 // This pattern is generated automatically from cas.m4.
9236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9237 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9238
9239 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9240 ins_cost(2 * VOLATILE_REF_COST);
9241 effect(KILL cr);
9242 format %{
9243 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9244 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9245 %}
9246 ins_encode %{
9247 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9248 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9249 /*weak*/ true, noreg);
9250 __ csetw($res$$Register, Assembler::EQ);
9251 %}
9252 ins_pipe(pipe_slow);
9253 %}
9254
9255 // This pattern is generated automatically from cas.m4.
9256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9257 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9258
9259 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9260 ins_cost(2 * VOLATILE_REF_COST);
9261 effect(KILL cr);
9262 format %{
9263 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9264 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9265 %}
9266 ins_encode %{
9267 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9268 Assembler::word, /*acquire*/ false, /*release*/ true,
9269 /*weak*/ true, noreg);
9270 __ csetw($res$$Register, Assembler::EQ);
9271 %}
9272 ins_pipe(pipe_slow);
9273 %}
9274
9275 // This pattern is generated automatically from cas.m4.
9276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9277 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9278
9279 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9280 ins_cost(2 * VOLATILE_REF_COST);
9281 effect(KILL cr);
9282 format %{
9283 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9284 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9285 %}
9286 ins_encode %{
9287 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9288 Assembler::xword, /*acquire*/ false, /*release*/ true,
9289 /*weak*/ true, noreg);
9290 __ csetw($res$$Register, Assembler::EQ);
9291 %}
9292 ins_pipe(pipe_slow);
9293 %}
9294
9295 // This pattern is generated automatically from cas.m4.
9296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9297 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9298
9299 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9300 ins_cost(2 * VOLATILE_REF_COST);
9301 effect(KILL cr);
9302 format %{
9303 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9304 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9305 %}
9306 ins_encode %{
9307 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9308 Assembler::word, /*acquire*/ false, /*release*/ true,
9309 /*weak*/ true, noreg);
9310 __ csetw($res$$Register, Assembler::EQ);
9311 %}
9312 ins_pipe(pipe_slow);
9313 %}
9314
9315 // This pattern is generated automatically from cas.m4.
9316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9317 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9318 predicate(n->as_LoadStore()->barrier_data() == 0);
9319 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9320 ins_cost(2 * VOLATILE_REF_COST);
9321 effect(KILL cr);
9322 format %{
9323 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9324 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9325 %}
9326 ins_encode %{
9327 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9328 Assembler::xword, /*acquire*/ false, /*release*/ true,
9329 /*weak*/ true, noreg);
9330 __ csetw($res$$Register, Assembler::EQ);
9331 %}
9332 ins_pipe(pipe_slow);
9333 %}
9334
9335 // This pattern is generated automatically from cas.m4.
9336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9337 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9338 predicate(needs_acquiring_load_exclusive(n));
9339 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9340 ins_cost(VOLATILE_REF_COST);
9341 effect(KILL cr);
9342 format %{
9343 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9344 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9345 %}
9346 ins_encode %{
9347 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9348 Assembler::byte, /*acquire*/ true, /*release*/ true,
9349 /*weak*/ true, noreg);
9350 __ csetw($res$$Register, Assembler::EQ);
9351 %}
9352 ins_pipe(pipe_slow);
9353 %}
9354
9355 // This pattern is generated automatically from cas.m4.
9356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9357 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9358 predicate(needs_acquiring_load_exclusive(n));
9359 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9360 ins_cost(VOLATILE_REF_COST);
9361 effect(KILL cr);
9362 format %{
9363 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9364 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9365 %}
9366 ins_encode %{
9367 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9368 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9369 /*weak*/ true, noreg);
9370 __ csetw($res$$Register, Assembler::EQ);
9371 %}
9372 ins_pipe(pipe_slow);
9373 %}
9374
9375 // This pattern is generated automatically from cas.m4.
9376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9377 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9378 predicate(needs_acquiring_load_exclusive(n));
9379 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9380 ins_cost(VOLATILE_REF_COST);
9381 effect(KILL cr);
9382 format %{
9383 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9384 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9385 %}
9386 ins_encode %{
9387 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9388 Assembler::word, /*acquire*/ true, /*release*/ true,
9389 /*weak*/ true, noreg);
9390 __ csetw($res$$Register, Assembler::EQ);
9391 %}
9392 ins_pipe(pipe_slow);
9393 %}
9394
9395 // This pattern is generated automatically from cas.m4.
9396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9397 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9398 predicate(needs_acquiring_load_exclusive(n));
9399 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9400 ins_cost(VOLATILE_REF_COST);
9401 effect(KILL cr);
9402 format %{
9403 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9404 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9405 %}
9406 ins_encode %{
9407 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9408 Assembler::xword, /*acquire*/ true, /*release*/ true,
9409 /*weak*/ true, noreg);
9410 __ csetw($res$$Register, Assembler::EQ);
9411 %}
9412 ins_pipe(pipe_slow);
9413 %}
9414
9415 // This pattern is generated automatically from cas.m4.
9416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9417 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9418 predicate(needs_acquiring_load_exclusive(n));
9419 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9420 ins_cost(VOLATILE_REF_COST);
9421 effect(KILL cr);
9422 format %{
9423 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9424 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9425 %}
9426 ins_encode %{
9427 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9428 Assembler::word, /*acquire*/ true, /*release*/ true,
9429 /*weak*/ true, noreg);
9430 __ csetw($res$$Register, Assembler::EQ);
9431 %}
9432 ins_pipe(pipe_slow);
9433 %}
9434
9435 // This pattern is generated automatically from cas.m4.
9436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9437 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9438 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9439 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9440 ins_cost(VOLATILE_REF_COST);
9441 effect(KILL cr);
9442 format %{
9443 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9444 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9445 %}
9446 ins_encode %{
9447 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9448 Assembler::xword, /*acquire*/ true, /*release*/ true,
9449 /*weak*/ true, noreg);
9450 __ csetw($res$$Register, Assembler::EQ);
9451 %}
9452 ins_pipe(pipe_slow);
9453 %}
9454
9455 // END This section of the file is automatically generated. Do not edit --------------
9456 // ---------------------------------------------------------------------
9457
9458 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9459 match(Set prev (GetAndSetI mem newv));
9460 ins_cost(2 * VOLATILE_REF_COST);
9461 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9462 ins_encode %{
9463 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9464 %}
9465 ins_pipe(pipe_serial);
9466 %}
9467
9468 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9469 match(Set prev (GetAndSetL mem newv));
9470 ins_cost(2 * VOLATILE_REF_COST);
9471 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9472 ins_encode %{
9473 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9474 %}
9475 ins_pipe(pipe_serial);
9476 %}
9477
9478 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9479 match(Set prev (GetAndSetN mem newv));
9480 ins_cost(2 * VOLATILE_REF_COST);
9481 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9482 ins_encode %{
9483 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9484 %}
9485 ins_pipe(pipe_serial);
9486 %}
9487
9488 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9489 predicate(n->as_LoadStore()->barrier_data() == 0);
9490 match(Set prev (GetAndSetP mem newv));
9491 ins_cost(2 * VOLATILE_REF_COST);
9492 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9493 ins_encode %{
9494 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9495 %}
9496 ins_pipe(pipe_serial);
9497 %}
9498
9499 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9500 predicate(needs_acquiring_load_exclusive(n));
9501 match(Set prev (GetAndSetI mem newv));
9502 ins_cost(VOLATILE_REF_COST);
9503 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9504 ins_encode %{
9505 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9506 %}
9507 ins_pipe(pipe_serial);
9508 %}
9509
9510 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9511 predicate(needs_acquiring_load_exclusive(n));
9512 match(Set prev (GetAndSetL mem newv));
9513 ins_cost(VOLATILE_REF_COST);
9514 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9515 ins_encode %{
9516 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9517 %}
9518 ins_pipe(pipe_serial);
9519 %}
9520
9521 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9522 predicate(needs_acquiring_load_exclusive(n));
9523 match(Set prev (GetAndSetN mem newv));
9524 ins_cost(VOLATILE_REF_COST);
9525 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9526 ins_encode %{
9527 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9528 %}
9529 ins_pipe(pipe_serial);
9530 %}
9531
9532 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9533 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9534 match(Set prev (GetAndSetP mem newv));
9535 ins_cost(VOLATILE_REF_COST);
9536 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9537 ins_encode %{
9538 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9539 %}
9540 ins_pipe(pipe_serial);
9541 %}
9542
9543
9544 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9545 match(Set newval (GetAndAddL mem incr));
9546 ins_cost(2 * VOLATILE_REF_COST + 1);
9547 format %{ "get_and_addL $newval, [$mem], $incr" %}
9548 ins_encode %{
9549 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9550 %}
9551 ins_pipe(pipe_serial);
9552 %}
9553
9554 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9555 predicate(n->as_LoadStore()->result_not_used());
9556 match(Set dummy (GetAndAddL mem incr));
9557 ins_cost(2 * VOLATILE_REF_COST);
9558 format %{ "get_and_addL [$mem], $incr" %}
9559 ins_encode %{
9560 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9561 %}
9562 ins_pipe(pipe_serial);
9563 %}
9564
9565 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9566 match(Set newval (GetAndAddL mem incr));
9567 ins_cost(2 * VOLATILE_REF_COST + 1);
9568 format %{ "get_and_addL $newval, [$mem], $incr" %}
9569 ins_encode %{
9570 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9571 %}
9572 ins_pipe(pipe_serial);
9573 %}
9574
9575 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9576 predicate(n->as_LoadStore()->result_not_used());
9577 match(Set dummy (GetAndAddL mem incr));
9578 ins_cost(2 * VOLATILE_REF_COST);
9579 format %{ "get_and_addL [$mem], $incr" %}
9580 ins_encode %{
9581 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9582 %}
9583 ins_pipe(pipe_serial);
9584 %}
9585
9586 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9587 match(Set newval (GetAndAddI mem incr));
9588 ins_cost(2 * VOLATILE_REF_COST + 1);
9589 format %{ "get_and_addI $newval, [$mem], $incr" %}
9590 ins_encode %{
9591 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9592 %}
9593 ins_pipe(pipe_serial);
9594 %}
9595
9596 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9597 predicate(n->as_LoadStore()->result_not_used());
9598 match(Set dummy (GetAndAddI mem incr));
9599 ins_cost(2 * VOLATILE_REF_COST);
9600 format %{ "get_and_addI [$mem], $incr" %}
9601 ins_encode %{
9602 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9603 %}
9604 ins_pipe(pipe_serial);
9605 %}
9606
9607 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9608 match(Set newval (GetAndAddI mem incr));
9609 ins_cost(2 * VOLATILE_REF_COST + 1);
9610 format %{ "get_and_addI $newval, [$mem], $incr" %}
9611 ins_encode %{
9612 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9613 %}
9614 ins_pipe(pipe_serial);
9615 %}
9616
9617 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9618 predicate(n->as_LoadStore()->result_not_used());
9619 match(Set dummy (GetAndAddI mem incr));
9620 ins_cost(2 * VOLATILE_REF_COST);
9621 format %{ "get_and_addI [$mem], $incr" %}
9622 ins_encode %{
9623 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9624 %}
9625 ins_pipe(pipe_serial);
9626 %}
9627
9628 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9629 predicate(needs_acquiring_load_exclusive(n));
9630 match(Set newval (GetAndAddL mem incr));
9631 ins_cost(VOLATILE_REF_COST + 1);
9632 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9633 ins_encode %{
9634 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9635 %}
9636 ins_pipe(pipe_serial);
9637 %}
9638
9639 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9640 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9641 match(Set dummy (GetAndAddL mem incr));
9642 ins_cost(VOLATILE_REF_COST);
9643 format %{ "get_and_addL_acq [$mem], $incr" %}
9644 ins_encode %{
9645 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9646 %}
9647 ins_pipe(pipe_serial);
9648 %}
9649
9650 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9651 predicate(needs_acquiring_load_exclusive(n));
9652 match(Set newval (GetAndAddL mem incr));
9653 ins_cost(VOLATILE_REF_COST + 1);
9654 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9655 ins_encode %{
9656 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9657 %}
9658 ins_pipe(pipe_serial);
9659 %}
9660
9661 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9662 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9663 match(Set dummy (GetAndAddL mem incr));
9664 ins_cost(VOLATILE_REF_COST);
9665 format %{ "get_and_addL_acq [$mem], $incr" %}
9666 ins_encode %{
9667 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9668 %}
9669 ins_pipe(pipe_serial);
9670 %}
9671
9672 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9673 predicate(needs_acquiring_load_exclusive(n));
9674 match(Set newval (GetAndAddI mem incr));
9675 ins_cost(VOLATILE_REF_COST + 1);
9676 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9677 ins_encode %{
9678 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9679 %}
9680 ins_pipe(pipe_serial);
9681 %}
9682
9683 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9684 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9685 match(Set dummy (GetAndAddI mem incr));
9686 ins_cost(VOLATILE_REF_COST);
9687 format %{ "get_and_addI_acq [$mem], $incr" %}
9688 ins_encode %{
9689 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9690 %}
9691 ins_pipe(pipe_serial);
9692 %}
9693
9694 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9695 predicate(needs_acquiring_load_exclusive(n));
9696 match(Set newval (GetAndAddI mem incr));
9697 ins_cost(VOLATILE_REF_COST + 1);
9698 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9699 ins_encode %{
9700 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9701 %}
9702 ins_pipe(pipe_serial);
9703 %}
9704
9705 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9706 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9707 match(Set dummy (GetAndAddI mem incr));
9708 ins_cost(VOLATILE_REF_COST);
9709 format %{ "get_and_addI_acq [$mem], $incr" %}
9710 ins_encode %{
9711 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9712 %}
9713 ins_pipe(pipe_serial);
9714 %}
9715
9716 // Manifest a CmpU result in an integer register.
9717 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9718 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9719 %{
9720 match(Set dst (CmpU3 src1 src2));
9721 effect(KILL flags);
9722
9723 ins_cost(INSN_COST * 3);
9724 format %{
9725 "cmpw $src1, $src2\n\t"
9726 "csetw $dst, ne\n\t"
9727 "cnegw $dst, lo\t# CmpU3(reg)"
9728 %}
9729 ins_encode %{
9730 __ cmpw($src1$$Register, $src2$$Register);
9731 __ csetw($dst$$Register, Assembler::NE);
9732 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9733 %}
9734
9735 ins_pipe(pipe_class_default);
9736 %}
9737
9738 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9739 %{
9740 match(Set dst (CmpU3 src1 src2));
9741 effect(KILL flags);
9742
9743 ins_cost(INSN_COST * 3);
9744 format %{
9745 "subsw zr, $src1, $src2\n\t"
9746 "csetw $dst, ne\n\t"
9747 "cnegw $dst, lo\t# CmpU3(imm)"
9748 %}
9749 ins_encode %{
9750 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9751 __ csetw($dst$$Register, Assembler::NE);
9752 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9753 %}
9754
9755 ins_pipe(pipe_class_default);
9756 %}
9757
9758 // Manifest a CmpUL result in an integer register.
9759 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9760 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9761 %{
9762 match(Set dst (CmpUL3 src1 src2));
9763 effect(KILL flags);
9764
9765 ins_cost(INSN_COST * 3);
9766 format %{
9767 "cmp $src1, $src2\n\t"
9768 "csetw $dst, ne\n\t"
9769 "cnegw $dst, lo\t# CmpUL3(reg)"
9770 %}
9771 ins_encode %{
9772 __ cmp($src1$$Register, $src2$$Register);
9773 __ csetw($dst$$Register, Assembler::NE);
9774 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9775 %}
9776
9777 ins_pipe(pipe_class_default);
9778 %}
9779
9780 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9781 %{
9782 match(Set dst (CmpUL3 src1 src2));
9783 effect(KILL flags);
9784
9785 ins_cost(INSN_COST * 3);
9786 format %{
9787 "subs zr, $src1, $src2\n\t"
9788 "csetw $dst, ne\n\t"
9789 "cnegw $dst, lo\t# CmpUL3(imm)"
9790 %}
9791 ins_encode %{
9792 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9793 __ csetw($dst$$Register, Assembler::NE);
9794 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9795 %}
9796
9797 ins_pipe(pipe_class_default);
9798 %}
9799
9800 // Manifest a CmpL result in an integer register.
9801 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9802 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9803 %{
9804 match(Set dst (CmpL3 src1 src2));
9805 effect(KILL flags);
9806
9807 ins_cost(INSN_COST * 3);
9808 format %{
9809 "cmp $src1, $src2\n\t"
9810 "csetw $dst, ne\n\t"
9811 "cnegw $dst, lt\t# CmpL3(reg)"
9812 %}
9813 ins_encode %{
9814 __ cmp($src1$$Register, $src2$$Register);
9815 __ csetw($dst$$Register, Assembler::NE);
9816 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9817 %}
9818
9819 ins_pipe(pipe_class_default);
9820 %}
9821
9822 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9823 %{
9824 match(Set dst (CmpL3 src1 src2));
9825 effect(KILL flags);
9826
9827 ins_cost(INSN_COST * 3);
9828 format %{
9829 "subs zr, $src1, $src2\n\t"
9830 "csetw $dst, ne\n\t"
9831 "cnegw $dst, lt\t# CmpL3(imm)"
9832 %}
9833 ins_encode %{
9834 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9835 __ csetw($dst$$Register, Assembler::NE);
9836 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9837 %}
9838
9839 ins_pipe(pipe_class_default);
9840 %}
9841
9842 // ============================================================================
9843 // Conditional Move Instructions
9844
9845 // n.b. we have identical rules for both a signed compare op (cmpOp)
9846 // and an unsigned compare op (cmpOpU). it would be nice if we could
9847 // define an op class which merged both inputs and use it to type the
9848 // argument to a single rule. unfortunatelyt his fails because the
9849 // opclass does not live up to the COND_INTER interface of its
9850 // component operands. When the generic code tries to negate the
9851 // operand it ends up running the generci Machoper::negate method
9852 // which throws a ShouldNotHappen. So, we have to provide two flavours
9853 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9854
9855 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9856 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9857
9858 ins_cost(INSN_COST * 2);
9859 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9860
9861 ins_encode %{
9862 __ cselw(as_Register($dst$$reg),
9863 as_Register($src2$$reg),
9864 as_Register($src1$$reg),
9865 (Assembler::Condition)$cmp$$cmpcode);
9866 %}
9867
9868 ins_pipe(icond_reg_reg);
9869 %}
9870
9871 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9872 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9873
9874 ins_cost(INSN_COST * 2);
9875 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9876
9877 ins_encode %{
9878 __ cselw(as_Register($dst$$reg),
9879 as_Register($src2$$reg),
9880 as_Register($src1$$reg),
9881 (Assembler::Condition)$cmp$$cmpcode);
9882 %}
9883
9884 ins_pipe(icond_reg_reg);
9885 %}
9886
9887 // special cases where one arg is zero
9888
9889 // n.b. this is selected in preference to the rule above because it
9890 // avoids loading constant 0 into a source register
9891
9892 // TODO
9893 // we ought only to be able to cull one of these variants as the ideal
9894 // transforms ought always to order the zero consistently (to left/right?)
9895
9896 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9897 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9898
9899 ins_cost(INSN_COST * 2);
9900 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9901
9902 ins_encode %{
9903 __ cselw(as_Register($dst$$reg),
9904 as_Register($src$$reg),
9905 zr,
9906 (Assembler::Condition)$cmp$$cmpcode);
9907 %}
9908
9909 ins_pipe(icond_reg);
9910 %}
9911
9912 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9913 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9914
9915 ins_cost(INSN_COST * 2);
9916 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9917
9918 ins_encode %{
9919 __ cselw(as_Register($dst$$reg),
9920 as_Register($src$$reg),
9921 zr,
9922 (Assembler::Condition)$cmp$$cmpcode);
9923 %}
9924
9925 ins_pipe(icond_reg);
9926 %}
9927
9928 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9929 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9930
9931 ins_cost(INSN_COST * 2);
9932 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9933
9934 ins_encode %{
9935 __ cselw(as_Register($dst$$reg),
9936 zr,
9937 as_Register($src$$reg),
9938 (Assembler::Condition)$cmp$$cmpcode);
9939 %}
9940
9941 ins_pipe(icond_reg);
9942 %}
9943
9944 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9945 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9946
9947 ins_cost(INSN_COST * 2);
9948 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9949
9950 ins_encode %{
9951 __ cselw(as_Register($dst$$reg),
9952 zr,
9953 as_Register($src$$reg),
9954 (Assembler::Condition)$cmp$$cmpcode);
9955 %}
9956
9957 ins_pipe(icond_reg);
9958 %}
9959
9960 // special case for creating a boolean 0 or 1
9961
9962 // n.b. this is selected in preference to the rule above because it
9963 // avoids loading constants 0 and 1 into a source register
9964
9965 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9966 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9967
9968 ins_cost(INSN_COST * 2);
9969 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9970
9971 ins_encode %{
9972 // equivalently
9973 // cset(as_Register($dst$$reg),
9974 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9975 __ csincw(as_Register($dst$$reg),
9976 zr,
9977 zr,
9978 (Assembler::Condition)$cmp$$cmpcode);
9979 %}
9980
9981 ins_pipe(icond_none);
9982 %}
9983
9984 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9985 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9986
9987 ins_cost(INSN_COST * 2);
9988 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9989
9990 ins_encode %{
9991 // equivalently
9992 // cset(as_Register($dst$$reg),
9993 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9994 __ csincw(as_Register($dst$$reg),
9995 zr,
9996 zr,
9997 (Assembler::Condition)$cmp$$cmpcode);
9998 %}
9999
10000 ins_pipe(icond_none);
10001 %}
10002
10003 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10004 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10005
10006 ins_cost(INSN_COST * 2);
10007 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10008
10009 ins_encode %{
10010 __ csel(as_Register($dst$$reg),
10011 as_Register($src2$$reg),
10012 as_Register($src1$$reg),
10013 (Assembler::Condition)$cmp$$cmpcode);
10014 %}
10015
10016 ins_pipe(icond_reg_reg);
10017 %}
10018
10019 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10020 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10021
10022 ins_cost(INSN_COST * 2);
10023 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10024
10025 ins_encode %{
10026 __ csel(as_Register($dst$$reg),
10027 as_Register($src2$$reg),
10028 as_Register($src1$$reg),
10029 (Assembler::Condition)$cmp$$cmpcode);
10030 %}
10031
10032 ins_pipe(icond_reg_reg);
10033 %}
10034
10035 // special cases where one arg is zero
10036
10037 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10038 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10039
10040 ins_cost(INSN_COST * 2);
10041 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10042
10043 ins_encode %{
10044 __ csel(as_Register($dst$$reg),
10045 zr,
10046 as_Register($src$$reg),
10047 (Assembler::Condition)$cmp$$cmpcode);
10048 %}
10049
10050 ins_pipe(icond_reg);
10051 %}
10052
10053 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10054 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10055
10056 ins_cost(INSN_COST * 2);
10057 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10058
10059 ins_encode %{
10060 __ csel(as_Register($dst$$reg),
10061 zr,
10062 as_Register($src$$reg),
10063 (Assembler::Condition)$cmp$$cmpcode);
10064 %}
10065
10066 ins_pipe(icond_reg);
10067 %}
10068
10069 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10070 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10071
10072 ins_cost(INSN_COST * 2);
10073 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10074
10075 ins_encode %{
10076 __ csel(as_Register($dst$$reg),
10077 as_Register($src$$reg),
10078 zr,
10079 (Assembler::Condition)$cmp$$cmpcode);
10080 %}
10081
10082 ins_pipe(icond_reg);
10083 %}
10084
10085 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10086 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10087
10088 ins_cost(INSN_COST * 2);
10089 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10090
10091 ins_encode %{
10092 __ csel(as_Register($dst$$reg),
10093 as_Register($src$$reg),
10094 zr,
10095 (Assembler::Condition)$cmp$$cmpcode);
10096 %}
10097
10098 ins_pipe(icond_reg);
10099 %}
10100
10101 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10102 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10103
10104 ins_cost(INSN_COST * 2);
10105 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10106
10107 ins_encode %{
10108 __ csel(as_Register($dst$$reg),
10109 as_Register($src2$$reg),
10110 as_Register($src1$$reg),
10111 (Assembler::Condition)$cmp$$cmpcode);
10112 %}
10113
10114 ins_pipe(icond_reg_reg);
10115 %}
10116
10117 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10118 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10119
10120 ins_cost(INSN_COST * 2);
10121 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10122
10123 ins_encode %{
10124 __ csel(as_Register($dst$$reg),
10125 as_Register($src2$$reg),
10126 as_Register($src1$$reg),
10127 (Assembler::Condition)$cmp$$cmpcode);
10128 %}
10129
10130 ins_pipe(icond_reg_reg);
10131 %}
10132
10133 // special cases where one arg is zero
10134
10135 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10136 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10137
10138 ins_cost(INSN_COST * 2);
10139 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10140
10141 ins_encode %{
10142 __ csel(as_Register($dst$$reg),
10143 zr,
10144 as_Register($src$$reg),
10145 (Assembler::Condition)$cmp$$cmpcode);
10146 %}
10147
10148 ins_pipe(icond_reg);
10149 %}
10150
10151 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10152 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10153
10154 ins_cost(INSN_COST * 2);
10155 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10156
10157 ins_encode %{
10158 __ csel(as_Register($dst$$reg),
10159 zr,
10160 as_Register($src$$reg),
10161 (Assembler::Condition)$cmp$$cmpcode);
10162 %}
10163
10164 ins_pipe(icond_reg);
10165 %}
10166
10167 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10168 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10169
10170 ins_cost(INSN_COST * 2);
10171 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10172
10173 ins_encode %{
10174 __ csel(as_Register($dst$$reg),
10175 as_Register($src$$reg),
10176 zr,
10177 (Assembler::Condition)$cmp$$cmpcode);
10178 %}
10179
10180 ins_pipe(icond_reg);
10181 %}
10182
10183 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10184 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10185
10186 ins_cost(INSN_COST * 2);
10187 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10188
10189 ins_encode %{
10190 __ csel(as_Register($dst$$reg),
10191 as_Register($src$$reg),
10192 zr,
10193 (Assembler::Condition)$cmp$$cmpcode);
10194 %}
10195
10196 ins_pipe(icond_reg);
10197 %}
10198
10199 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10200 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10201
10202 ins_cost(INSN_COST * 2);
10203 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10204
10205 ins_encode %{
10206 __ cselw(as_Register($dst$$reg),
10207 as_Register($src2$$reg),
10208 as_Register($src1$$reg),
10209 (Assembler::Condition)$cmp$$cmpcode);
10210 %}
10211
10212 ins_pipe(icond_reg_reg);
10213 %}
10214
10215 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10216 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10217
10218 ins_cost(INSN_COST * 2);
10219 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10220
10221 ins_encode %{
10222 __ cselw(as_Register($dst$$reg),
10223 as_Register($src2$$reg),
10224 as_Register($src1$$reg),
10225 (Assembler::Condition)$cmp$$cmpcode);
10226 %}
10227
10228 ins_pipe(icond_reg_reg);
10229 %}
10230
10231 // special cases where one arg is zero
10232
10233 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10234 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10235
10236 ins_cost(INSN_COST * 2);
10237 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10238
10239 ins_encode %{
10240 __ cselw(as_Register($dst$$reg),
10241 zr,
10242 as_Register($src$$reg),
10243 (Assembler::Condition)$cmp$$cmpcode);
10244 %}
10245
10246 ins_pipe(icond_reg);
10247 %}
10248
10249 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10250 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10251
10252 ins_cost(INSN_COST * 2);
10253 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10254
10255 ins_encode %{
10256 __ cselw(as_Register($dst$$reg),
10257 zr,
10258 as_Register($src$$reg),
10259 (Assembler::Condition)$cmp$$cmpcode);
10260 %}
10261
10262 ins_pipe(icond_reg);
10263 %}
10264
10265 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10266 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10267
10268 ins_cost(INSN_COST * 2);
10269 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10270
10271 ins_encode %{
10272 __ cselw(as_Register($dst$$reg),
10273 as_Register($src$$reg),
10274 zr,
10275 (Assembler::Condition)$cmp$$cmpcode);
10276 %}
10277
10278 ins_pipe(icond_reg);
10279 %}
10280
10281 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10282 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10283
10284 ins_cost(INSN_COST * 2);
10285 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10286
10287 ins_encode %{
10288 __ cselw(as_Register($dst$$reg),
10289 as_Register($src$$reg),
10290 zr,
10291 (Assembler::Condition)$cmp$$cmpcode);
10292 %}
10293
10294 ins_pipe(icond_reg);
10295 %}
10296
10297 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10298 %{
10299 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10300
10301 ins_cost(INSN_COST * 3);
10302
10303 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10304 ins_encode %{
10305 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10306 __ fcsels(as_FloatRegister($dst$$reg),
10307 as_FloatRegister($src2$$reg),
10308 as_FloatRegister($src1$$reg),
10309 cond);
10310 %}
10311
10312 ins_pipe(fp_cond_reg_reg_s);
10313 %}
10314
10315 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10316 %{
10317 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10318
10319 ins_cost(INSN_COST * 3);
10320
10321 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10322 ins_encode %{
10323 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10324 __ fcsels(as_FloatRegister($dst$$reg),
10325 as_FloatRegister($src2$$reg),
10326 as_FloatRegister($src1$$reg),
10327 cond);
10328 %}
10329
10330 ins_pipe(fp_cond_reg_reg_s);
10331 %}
10332
10333 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10334 %{
10335 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10336
10337 ins_cost(INSN_COST * 3);
10338
10339 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10340 ins_encode %{
10341 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10342 __ fcseld(as_FloatRegister($dst$$reg),
10343 as_FloatRegister($src2$$reg),
10344 as_FloatRegister($src1$$reg),
10345 cond);
10346 %}
10347
10348 ins_pipe(fp_cond_reg_reg_d);
10349 %}
10350
10351 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10352 %{
10353 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10354
10355 ins_cost(INSN_COST * 3);
10356
10357 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10358 ins_encode %{
10359 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10360 __ fcseld(as_FloatRegister($dst$$reg),
10361 as_FloatRegister($src2$$reg),
10362 as_FloatRegister($src1$$reg),
10363 cond);
10364 %}
10365
10366 ins_pipe(fp_cond_reg_reg_d);
10367 %}
10368
10369 // ============================================================================
10370 // Arithmetic Instructions
10371 //
10372
10373 // Integer Addition
10374
10375 // TODO
10376 // these currently employ operations which do not set CR and hence are
10377 // not flagged as killing CR but we would like to isolate the cases
10378 // where we want to set flags from those where we don't. need to work
10379 // out how to do that.
10380
10381 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10382 match(Set dst (AddI src1 src2));
10383
10384 ins_cost(INSN_COST);
10385 format %{ "addw $dst, $src1, $src2" %}
10386
10387 ins_encode %{
10388 __ addw(as_Register($dst$$reg),
10389 as_Register($src1$$reg),
10390 as_Register($src2$$reg));
10391 %}
10392
10393 ins_pipe(ialu_reg_reg);
10394 %}
10395
10396 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10397 match(Set dst (AddI src1 src2));
10398
10399 ins_cost(INSN_COST);
10400 format %{ "addw $dst, $src1, $src2" %}
10401
10402 // use opcode to indicate that this is an add not a sub
10403 opcode(0x0);
10404
10405 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10406
10407 ins_pipe(ialu_reg_imm);
10408 %}
10409
10410 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10411 match(Set dst (AddI (ConvL2I src1) src2));
10412
10413 ins_cost(INSN_COST);
10414 format %{ "addw $dst, $src1, $src2" %}
10415
10416 // use opcode to indicate that this is an add not a sub
10417 opcode(0x0);
10418
10419 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10420
10421 ins_pipe(ialu_reg_imm);
10422 %}
10423
10424 // Pointer Addition
10425 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10426 match(Set dst (AddP src1 src2));
10427
10428 ins_cost(INSN_COST);
10429 format %{ "add $dst, $src1, $src2\t# ptr" %}
10430
10431 ins_encode %{
10432 __ add(as_Register($dst$$reg),
10433 as_Register($src1$$reg),
10434 as_Register($src2$$reg));
10435 %}
10436
10437 ins_pipe(ialu_reg_reg);
10438 %}
10439
10440 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10441 match(Set dst (AddP src1 (ConvI2L src2)));
10442
10443 ins_cost(1.9 * INSN_COST);
10444 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10445
10446 ins_encode %{
10447 __ add(as_Register($dst$$reg),
10448 as_Register($src1$$reg),
10449 as_Register($src2$$reg), ext::sxtw);
10450 %}
10451
10452 ins_pipe(ialu_reg_reg);
10453 %}
10454
10455 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10456 match(Set dst (AddP src1 (LShiftL src2 scale)));
10457
10458 ins_cost(1.9 * INSN_COST);
10459 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10460
10461 ins_encode %{
10462 __ lea(as_Register($dst$$reg),
10463 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10464 Address::lsl($scale$$constant)));
10465 %}
10466
10467 ins_pipe(ialu_reg_reg_shift);
10468 %}
10469
10470 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10471 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10472
10473 ins_cost(1.9 * INSN_COST);
10474 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10475
10476 ins_encode %{
10477 __ lea(as_Register($dst$$reg),
10478 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10479 Address::sxtw($scale$$constant)));
10480 %}
10481
10482 ins_pipe(ialu_reg_reg_shift);
10483 %}
10484
10485 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10486 match(Set dst (LShiftL (ConvI2L src) scale));
10487
10488 ins_cost(INSN_COST);
10489 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10490
10491 ins_encode %{
10492 __ sbfiz(as_Register($dst$$reg),
10493 as_Register($src$$reg),
10494 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10495 %}
10496
10497 ins_pipe(ialu_reg_shift);
10498 %}
10499
10500 // Pointer Immediate Addition
10501 // n.b. this needs to be more expensive than using an indirect memory
10502 // operand
10503 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10504 match(Set dst (AddP src1 src2));
10505
10506 ins_cost(INSN_COST);
10507 format %{ "add $dst, $src1, $src2\t# ptr" %}
10508
10509 // use opcode to indicate that this is an add not a sub
10510 opcode(0x0);
10511
10512 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10513
10514 ins_pipe(ialu_reg_imm);
10515 %}
10516
10517 // Long Addition
10518 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10519
10520 match(Set dst (AddL src1 src2));
10521
10522 ins_cost(INSN_COST);
10523 format %{ "add $dst, $src1, $src2" %}
10524
10525 ins_encode %{
10526 __ add(as_Register($dst$$reg),
10527 as_Register($src1$$reg),
10528 as_Register($src2$$reg));
10529 %}
10530
10531 ins_pipe(ialu_reg_reg);
10532 %}
10533
10534 // No constant pool entries requiredLong Immediate Addition.
10535 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10536 match(Set dst (AddL src1 src2));
10537
10538 ins_cost(INSN_COST);
10539 format %{ "add $dst, $src1, $src2" %}
10540
10541 // use opcode to indicate that this is an add not a sub
10542 opcode(0x0);
10543
10544 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10545
10546 ins_pipe(ialu_reg_imm);
10547 %}
10548
10549 // Integer Subtraction
10550 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10551 match(Set dst (SubI src1 src2));
10552
10553 ins_cost(INSN_COST);
10554 format %{ "subw $dst, $src1, $src2" %}
10555
10556 ins_encode %{
10557 __ subw(as_Register($dst$$reg),
10558 as_Register($src1$$reg),
10559 as_Register($src2$$reg));
10560 %}
10561
10562 ins_pipe(ialu_reg_reg);
10563 %}
10564
10565 // Immediate Subtraction
10566 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10567 match(Set dst (SubI src1 src2));
10568
10569 ins_cost(INSN_COST);
10570 format %{ "subw $dst, $src1, $src2" %}
10571
10572 // use opcode to indicate that this is a sub not an add
10573 opcode(0x1);
10574
10575 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10576
10577 ins_pipe(ialu_reg_imm);
10578 %}
10579
10580 // Long Subtraction
10581 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10582
10583 match(Set dst (SubL src1 src2));
10584
10585 ins_cost(INSN_COST);
10586 format %{ "sub $dst, $src1, $src2" %}
10587
10588 ins_encode %{
10589 __ sub(as_Register($dst$$reg),
10590 as_Register($src1$$reg),
10591 as_Register($src2$$reg));
10592 %}
10593
10594 ins_pipe(ialu_reg_reg);
10595 %}
10596
10597 // No constant pool entries requiredLong Immediate Subtraction.
10598 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10599 match(Set dst (SubL src1 src2));
10600
10601 ins_cost(INSN_COST);
10602 format %{ "sub$dst, $src1, $src2" %}
10603
10604 // use opcode to indicate that this is a sub not an add
10605 opcode(0x1);
10606
10607 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10608
10609 ins_pipe(ialu_reg_imm);
10610 %}
10611
10612 // Integer Negation (special case for sub)
10613
10614 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10615 match(Set dst (SubI zero src));
10616
10617 ins_cost(INSN_COST);
10618 format %{ "negw $dst, $src\t# int" %}
10619
10620 ins_encode %{
10621 __ negw(as_Register($dst$$reg),
10622 as_Register($src$$reg));
10623 %}
10624
10625 ins_pipe(ialu_reg);
10626 %}
10627
10628 // Long Negation
10629
10630 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10631 match(Set dst (SubL zero src));
10632
10633 ins_cost(INSN_COST);
10634 format %{ "neg $dst, $src\t# long" %}
10635
10636 ins_encode %{
10637 __ neg(as_Register($dst$$reg),
10638 as_Register($src$$reg));
10639 %}
10640
10641 ins_pipe(ialu_reg);
10642 %}
10643
10644 // Integer Multiply
10645
10646 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10647 match(Set dst (MulI src1 src2));
10648
10649 ins_cost(INSN_COST * 3);
10650 format %{ "mulw $dst, $src1, $src2" %}
10651
10652 ins_encode %{
10653 __ mulw(as_Register($dst$$reg),
10654 as_Register($src1$$reg),
10655 as_Register($src2$$reg));
10656 %}
10657
10658 ins_pipe(imul_reg_reg);
10659 %}
10660
10661 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10662 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10663
10664 ins_cost(INSN_COST * 3);
10665 format %{ "smull $dst, $src1, $src2" %}
10666
10667 ins_encode %{
10668 __ smull(as_Register($dst$$reg),
10669 as_Register($src1$$reg),
10670 as_Register($src2$$reg));
10671 %}
10672
10673 ins_pipe(imul_reg_reg);
10674 %}
10675
10676 // Long Multiply
10677
10678 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10679 match(Set dst (MulL src1 src2));
10680
10681 ins_cost(INSN_COST * 5);
10682 format %{ "mul $dst, $src1, $src2" %}
10683
10684 ins_encode %{
10685 __ mul(as_Register($dst$$reg),
10686 as_Register($src1$$reg),
10687 as_Register($src2$$reg));
10688 %}
10689
10690 ins_pipe(lmul_reg_reg);
10691 %}
10692
10693 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10694 %{
10695 match(Set dst (MulHiL src1 src2));
10696
10697 ins_cost(INSN_COST * 7);
10698 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10699
10700 ins_encode %{
10701 __ smulh(as_Register($dst$$reg),
10702 as_Register($src1$$reg),
10703 as_Register($src2$$reg));
10704 %}
10705
10706 ins_pipe(lmul_reg_reg);
10707 %}
10708
10709 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10710 %{
10711 match(Set dst (UMulHiL src1 src2));
10712
10713 ins_cost(INSN_COST * 7);
10714 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10715
10716 ins_encode %{
10717 __ umulh(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 as_Register($src2$$reg));
10720 %}
10721
10722 ins_pipe(lmul_reg_reg);
10723 %}
10724
10725 // Combined Integer Multiply & Add/Sub
10726
10727 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10728 match(Set dst (AddI src3 (MulI src1 src2)));
10729
10730 ins_cost(INSN_COST * 3);
10731 format %{ "madd $dst, $src1, $src2, $src3" %}
10732
10733 ins_encode %{
10734 __ maddw(as_Register($dst$$reg),
10735 as_Register($src1$$reg),
10736 as_Register($src2$$reg),
10737 as_Register($src3$$reg));
10738 %}
10739
10740 ins_pipe(imac_reg_reg);
10741 %}
10742
10743 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10744 match(Set dst (SubI src3 (MulI src1 src2)));
10745
10746 ins_cost(INSN_COST * 3);
10747 format %{ "msub $dst, $src1, $src2, $src3" %}
10748
10749 ins_encode %{
10750 __ msubw(as_Register($dst$$reg),
10751 as_Register($src1$$reg),
10752 as_Register($src2$$reg),
10753 as_Register($src3$$reg));
10754 %}
10755
10756 ins_pipe(imac_reg_reg);
10757 %}
10758
10759 // Combined Integer Multiply & Neg
10760
10761 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10762 match(Set dst (MulI (SubI zero src1) src2));
10763
10764 ins_cost(INSN_COST * 3);
10765 format %{ "mneg $dst, $src1, $src2" %}
10766
10767 ins_encode %{
10768 __ mnegw(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg));
10771 %}
10772
10773 ins_pipe(imac_reg_reg);
10774 %}
10775
10776 // Combined Long Multiply & Add/Sub
10777
10778 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10779 match(Set dst (AddL src3 (MulL src1 src2)));
10780
10781 ins_cost(INSN_COST * 5);
10782 format %{ "madd $dst, $src1, $src2, $src3" %}
10783
10784 ins_encode %{
10785 __ madd(as_Register($dst$$reg),
10786 as_Register($src1$$reg),
10787 as_Register($src2$$reg),
10788 as_Register($src3$$reg));
10789 %}
10790
10791 ins_pipe(lmac_reg_reg);
10792 %}
10793
10794 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10795 match(Set dst (SubL src3 (MulL src1 src2)));
10796
10797 ins_cost(INSN_COST * 5);
10798 format %{ "msub $dst, $src1, $src2, $src3" %}
10799
10800 ins_encode %{
10801 __ msub(as_Register($dst$$reg),
10802 as_Register($src1$$reg),
10803 as_Register($src2$$reg),
10804 as_Register($src3$$reg));
10805 %}
10806
10807 ins_pipe(lmac_reg_reg);
10808 %}
10809
10810 // Combined Long Multiply & Neg
10811
10812 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10813 match(Set dst (MulL (SubL zero src1) src2));
10814
10815 ins_cost(INSN_COST * 5);
10816 format %{ "mneg $dst, $src1, $src2" %}
10817
10818 ins_encode %{
10819 __ mneg(as_Register($dst$$reg),
10820 as_Register($src1$$reg),
10821 as_Register($src2$$reg));
10822 %}
10823
10824 ins_pipe(lmac_reg_reg);
10825 %}
10826
10827 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10828
10829 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10830 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10831
10832 ins_cost(INSN_COST * 3);
10833 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10834
10835 ins_encode %{
10836 __ smaddl(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg),
10839 as_Register($src3$$reg));
10840 %}
10841
10842 ins_pipe(imac_reg_reg);
10843 %}
10844
10845 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10846 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10847
10848 ins_cost(INSN_COST * 3);
10849 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10850
10851 ins_encode %{
10852 __ smsubl(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg),
10855 as_Register($src3$$reg));
10856 %}
10857
10858 ins_pipe(imac_reg_reg);
10859 %}
10860
10861 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10862 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10863
10864 ins_cost(INSN_COST * 3);
10865 format %{ "smnegl $dst, $src1, $src2" %}
10866
10867 ins_encode %{
10868 __ smnegl(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg));
10871 %}
10872
10873 ins_pipe(imac_reg_reg);
10874 %}
10875
10876 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10877
10878 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10879 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10880
10881 ins_cost(INSN_COST * 5);
10882 format %{ "mulw rscratch1, $src1, $src2\n\t"
10883 "maddw $dst, $src3, $src4, rscratch1" %}
10884
10885 ins_encode %{
10886 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10887 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10888
10889 ins_pipe(imac_reg_reg);
10890 %}
10891
10892 // Integer Divide
10893
10894 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10895 match(Set dst (DivI src1 src2));
10896
10897 ins_cost(INSN_COST * 19);
10898 format %{ "sdivw $dst, $src1, $src2" %}
10899
10900 ins_encode(aarch64_enc_divw(dst, src1, src2));
10901 ins_pipe(idiv_reg_reg);
10902 %}
10903
10904 // Long Divide
10905
10906 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10907 match(Set dst (DivL src1 src2));
10908
10909 ins_cost(INSN_COST * 35);
10910 format %{ "sdiv $dst, $src1, $src2" %}
10911
10912 ins_encode(aarch64_enc_div(dst, src1, src2));
10913 ins_pipe(ldiv_reg_reg);
10914 %}
10915
10916 // Integer Remainder
10917
10918 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10919 match(Set dst (ModI src1 src2));
10920
10921 ins_cost(INSN_COST * 22);
10922 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10923 "msubw $dst, rscratch1, $src2, $src1" %}
10924
10925 ins_encode(aarch64_enc_modw(dst, src1, src2));
10926 ins_pipe(idiv_reg_reg);
10927 %}
10928
10929 // Long Remainder
10930
10931 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10932 match(Set dst (ModL src1 src2));
10933
10934 ins_cost(INSN_COST * 38);
10935 format %{ "sdiv rscratch1, $src1, $src2\n"
10936 "msub $dst, rscratch1, $src2, $src1" %}
10937
10938 ins_encode(aarch64_enc_mod(dst, src1, src2));
10939 ins_pipe(ldiv_reg_reg);
10940 %}
10941
10942 // Unsigned Integer Divide
10943
10944 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10945 match(Set dst (UDivI src1 src2));
10946
10947 ins_cost(INSN_COST * 19);
10948 format %{ "udivw $dst, $src1, $src2" %}
10949
10950 ins_encode %{
10951 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10952 %}
10953
10954 ins_pipe(idiv_reg_reg);
10955 %}
10956
10957 // Unsigned Long Divide
10958
10959 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10960 match(Set dst (UDivL src1 src2));
10961
10962 ins_cost(INSN_COST * 35);
10963 format %{ "udiv $dst, $src1, $src2" %}
10964
10965 ins_encode %{
10966 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10967 %}
10968
10969 ins_pipe(ldiv_reg_reg);
10970 %}
10971
10972 // Unsigned Integer Remainder
10973
10974 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10975 match(Set dst (UModI src1 src2));
10976
10977 ins_cost(INSN_COST * 22);
10978 format %{ "udivw rscratch1, $src1, $src2\n\t"
10979 "msubw $dst, rscratch1, $src2, $src1" %}
10980
10981 ins_encode %{
10982 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10983 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10984 %}
10985
10986 ins_pipe(idiv_reg_reg);
10987 %}
10988
10989 // Unsigned Long Remainder
10990
10991 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10992 match(Set dst (UModL src1 src2));
10993
10994 ins_cost(INSN_COST * 38);
10995 format %{ "udiv rscratch1, $src1, $src2\n"
10996 "msub $dst, rscratch1, $src2, $src1" %}
10997
10998 ins_encode %{
10999 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11000 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11001 %}
11002
11003 ins_pipe(ldiv_reg_reg);
11004 %}
11005
11006 // Integer Shifts
11007
11008 // Shift Left Register
11009 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11010 match(Set dst (LShiftI src1 src2));
11011
11012 ins_cost(INSN_COST * 2);
11013 format %{ "lslvw $dst, $src1, $src2" %}
11014
11015 ins_encode %{
11016 __ lslvw(as_Register($dst$$reg),
11017 as_Register($src1$$reg),
11018 as_Register($src2$$reg));
11019 %}
11020
11021 ins_pipe(ialu_reg_reg_vshift);
11022 %}
11023
11024 // Shift Left Immediate
11025 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11026 match(Set dst (LShiftI src1 src2));
11027
11028 ins_cost(INSN_COST);
11029 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11030
11031 ins_encode %{
11032 __ lslw(as_Register($dst$$reg),
11033 as_Register($src1$$reg),
11034 $src2$$constant & 0x1f);
11035 %}
11036
11037 ins_pipe(ialu_reg_shift);
11038 %}
11039
11040 // Shift Right Logical Register
11041 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11042 match(Set dst (URShiftI src1 src2));
11043
11044 ins_cost(INSN_COST * 2);
11045 format %{ "lsrvw $dst, $src1, $src2" %}
11046
11047 ins_encode %{
11048 __ lsrvw(as_Register($dst$$reg),
11049 as_Register($src1$$reg),
11050 as_Register($src2$$reg));
11051 %}
11052
11053 ins_pipe(ialu_reg_reg_vshift);
11054 %}
11055
11056 // Shift Right Logical Immediate
11057 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11058 match(Set dst (URShiftI src1 src2));
11059
11060 ins_cost(INSN_COST);
11061 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11062
11063 ins_encode %{
11064 __ lsrw(as_Register($dst$$reg),
11065 as_Register($src1$$reg),
11066 $src2$$constant & 0x1f);
11067 %}
11068
11069 ins_pipe(ialu_reg_shift);
11070 %}
11071
11072 // Shift Right Arithmetic Register
11073 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11074 match(Set dst (RShiftI src1 src2));
11075
11076 ins_cost(INSN_COST * 2);
11077 format %{ "asrvw $dst, $src1, $src2" %}
11078
11079 ins_encode %{
11080 __ asrvw(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg));
11083 %}
11084
11085 ins_pipe(ialu_reg_reg_vshift);
11086 %}
11087
11088 // Shift Right Arithmetic Immediate
11089 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11090 match(Set dst (RShiftI src1 src2));
11091
11092 ins_cost(INSN_COST);
11093 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11094
11095 ins_encode %{
11096 __ asrw(as_Register($dst$$reg),
11097 as_Register($src1$$reg),
11098 $src2$$constant & 0x1f);
11099 %}
11100
11101 ins_pipe(ialu_reg_shift);
11102 %}
11103
11104 // Combined Int Mask and Right Shift (using UBFM)
11105 // TODO
11106
11107 // Long Shifts
11108
11109 // Shift Left Register
11110 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11111 match(Set dst (LShiftL src1 src2));
11112
11113 ins_cost(INSN_COST * 2);
11114 format %{ "lslv $dst, $src1, $src2" %}
11115
11116 ins_encode %{
11117 __ lslv(as_Register($dst$$reg),
11118 as_Register($src1$$reg),
11119 as_Register($src2$$reg));
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_vshift);
11123 %}
11124
11125 // Shift Left Immediate
11126 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11127 match(Set dst (LShiftL src1 src2));
11128
11129 ins_cost(INSN_COST);
11130 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11131
11132 ins_encode %{
11133 __ lsl(as_Register($dst$$reg),
11134 as_Register($src1$$reg),
11135 $src2$$constant & 0x3f);
11136 %}
11137
11138 ins_pipe(ialu_reg_shift);
11139 %}
11140
11141 // Shift Right Logical Register
11142 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11143 match(Set dst (URShiftL src1 src2));
11144
11145 ins_cost(INSN_COST * 2);
11146 format %{ "lsrv $dst, $src1, $src2" %}
11147
11148 ins_encode %{
11149 __ lsrv(as_Register($dst$$reg),
11150 as_Register($src1$$reg),
11151 as_Register($src2$$reg));
11152 %}
11153
11154 ins_pipe(ialu_reg_reg_vshift);
11155 %}
11156
11157 // Shift Right Logical Immediate
11158 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11159 match(Set dst (URShiftL src1 src2));
11160
11161 ins_cost(INSN_COST);
11162 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11163
11164 ins_encode %{
11165 __ lsr(as_Register($dst$$reg),
11166 as_Register($src1$$reg),
11167 $src2$$constant & 0x3f);
11168 %}
11169
11170 ins_pipe(ialu_reg_shift);
11171 %}
11172
11173 // A special-case pattern for card table stores.
11174 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11175 match(Set dst (URShiftL (CastP2X src1) src2));
11176
11177 ins_cost(INSN_COST);
11178 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11179
11180 ins_encode %{
11181 __ lsr(as_Register($dst$$reg),
11182 as_Register($src1$$reg),
11183 $src2$$constant & 0x3f);
11184 %}
11185
11186 ins_pipe(ialu_reg_shift);
11187 %}
11188
11189 // Shift Right Arithmetic Register
11190 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11191 match(Set dst (RShiftL src1 src2));
11192
11193 ins_cost(INSN_COST * 2);
11194 format %{ "asrv $dst, $src1, $src2" %}
11195
11196 ins_encode %{
11197 __ asrv(as_Register($dst$$reg),
11198 as_Register($src1$$reg),
11199 as_Register($src2$$reg));
11200 %}
11201
11202 ins_pipe(ialu_reg_reg_vshift);
11203 %}
11204
11205 // Shift Right Arithmetic Immediate
11206 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11207 match(Set dst (RShiftL src1 src2));
11208
11209 ins_cost(INSN_COST);
11210 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11211
11212 ins_encode %{
11213 __ asr(as_Register($dst$$reg),
11214 as_Register($src1$$reg),
11215 $src2$$constant & 0x3f);
11216 %}
11217
11218 ins_pipe(ialu_reg_shift);
11219 %}
11220
11221 // BEGIN This section of the file is automatically generated. Do not edit --------------
11222 // This section is generated from aarch64_ad.m4
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 instruct regL_not_reg(iRegLNoSp dst,
11227 iRegL src1, immL_M1 m1,
11228 rFlagsReg cr) %{
11229 match(Set dst (XorL src1 m1));
11230 ins_cost(INSN_COST);
11231 format %{ "eon $dst, $src1, zr" %}
11232
11233 ins_encode %{
11234 __ eon(as_Register($dst$$reg),
11235 as_Register($src1$$reg),
11236 zr,
11237 Assembler::LSL, 0);
11238 %}
11239
11240 ins_pipe(ialu_reg);
11241 %}
11242
11243 // This pattern is automatically generated from aarch64_ad.m4
11244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11245 instruct regI_not_reg(iRegINoSp dst,
11246 iRegIorL2I src1, immI_M1 m1,
11247 rFlagsReg cr) %{
11248 match(Set dst (XorI src1 m1));
11249 ins_cost(INSN_COST);
11250 format %{ "eonw $dst, $src1, zr" %}
11251
11252 ins_encode %{
11253 __ eonw(as_Register($dst$$reg),
11254 as_Register($src1$$reg),
11255 zr,
11256 Assembler::LSL, 0);
11257 %}
11258
11259 ins_pipe(ialu_reg);
11260 %}
11261
11262 // This pattern is automatically generated from aarch64_ad.m4
11263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11264 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11265 immI0 zero, iRegIorL2I src1, immI src2) %{
11266 match(Set dst (SubI zero (URShiftI src1 src2)));
11267
11268 ins_cost(1.9 * INSN_COST);
11269 format %{ "negw $dst, $src1, LSR $src2" %}
11270
11271 ins_encode %{
11272 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11273 Assembler::LSR, $src2$$constant & 0x1f);
11274 %}
11275
11276 ins_pipe(ialu_reg_shift);
11277 %}
11278
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11282 immI0 zero, iRegIorL2I src1, immI src2) %{
11283 match(Set dst (SubI zero (RShiftI src1 src2)));
11284
11285 ins_cost(1.9 * INSN_COST);
11286 format %{ "negw $dst, $src1, ASR $src2" %}
11287
11288 ins_encode %{
11289 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11290 Assembler::ASR, $src2$$constant & 0x1f);
11291 %}
11292
11293 ins_pipe(ialu_reg_shift);
11294 %}
11295
11296 // This pattern is automatically generated from aarch64_ad.m4
11297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11298 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11299 immI0 zero, iRegIorL2I src1, immI src2) %{
11300 match(Set dst (SubI zero (LShiftI src1 src2)));
11301
11302 ins_cost(1.9 * INSN_COST);
11303 format %{ "negw $dst, $src1, LSL $src2" %}
11304
11305 ins_encode %{
11306 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11307 Assembler::LSL, $src2$$constant & 0x1f);
11308 %}
11309
11310 ins_pipe(ialu_reg_shift);
11311 %}
11312
11313 // This pattern is automatically generated from aarch64_ad.m4
11314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11315 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11316 immL0 zero, iRegL src1, immI src2) %{
11317 match(Set dst (SubL zero (URShiftL src1 src2)));
11318
11319 ins_cost(1.9 * INSN_COST);
11320 format %{ "neg $dst, $src1, LSR $src2" %}
11321
11322 ins_encode %{
11323 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11324 Assembler::LSR, $src2$$constant & 0x3f);
11325 %}
11326
11327 ins_pipe(ialu_reg_shift);
11328 %}
11329
11330 // This pattern is automatically generated from aarch64_ad.m4
11331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11332 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11333 immL0 zero, iRegL src1, immI src2) %{
11334 match(Set dst (SubL zero (RShiftL src1 src2)));
11335
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "neg $dst, $src1, ASR $src2" %}
11338
11339 ins_encode %{
11340 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11341 Assembler::ASR, $src2$$constant & 0x3f);
11342 %}
11343
11344 ins_pipe(ialu_reg_shift);
11345 %}
11346
11347 // This pattern is automatically generated from aarch64_ad.m4
11348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11349 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11350 immL0 zero, iRegL src1, immI src2) %{
11351 match(Set dst (SubL zero (LShiftL src1 src2)));
11352
11353 ins_cost(1.9 * INSN_COST);
11354 format %{ "neg $dst, $src1, LSL $src2" %}
11355
11356 ins_encode %{
11357 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11358 Assembler::LSL, $src2$$constant & 0x3f);
11359 %}
11360
11361 ins_pipe(ialu_reg_shift);
11362 %}
11363
11364 // This pattern is automatically generated from aarch64_ad.m4
11365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11366 instruct AndI_reg_not_reg(iRegINoSp dst,
11367 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11368 match(Set dst (AndI src1 (XorI src2 m1)));
11369 ins_cost(INSN_COST);
11370 format %{ "bicw $dst, $src1, $src2" %}
11371
11372 ins_encode %{
11373 __ bicw(as_Register($dst$$reg),
11374 as_Register($src1$$reg),
11375 as_Register($src2$$reg),
11376 Assembler::LSL, 0);
11377 %}
11378
11379 ins_pipe(ialu_reg_reg);
11380 %}
11381
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 instruct AndL_reg_not_reg(iRegLNoSp dst,
11385 iRegL src1, iRegL src2, immL_M1 m1) %{
11386 match(Set dst (AndL src1 (XorL src2 m1)));
11387 ins_cost(INSN_COST);
11388 format %{ "bic $dst, $src1, $src2" %}
11389
11390 ins_encode %{
11391 __ bic(as_Register($dst$$reg),
11392 as_Register($src1$$reg),
11393 as_Register($src2$$reg),
11394 Assembler::LSL, 0);
11395 %}
11396
11397 ins_pipe(ialu_reg_reg);
11398 %}
11399
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 instruct OrI_reg_not_reg(iRegINoSp dst,
11403 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11404 match(Set dst (OrI src1 (XorI src2 m1)));
11405 ins_cost(INSN_COST);
11406 format %{ "ornw $dst, $src1, $src2" %}
11407
11408 ins_encode %{
11409 __ ornw(as_Register($dst$$reg),
11410 as_Register($src1$$reg),
11411 as_Register($src2$$reg),
11412 Assembler::LSL, 0);
11413 %}
11414
11415 ins_pipe(ialu_reg_reg);
11416 %}
11417
11418 // This pattern is automatically generated from aarch64_ad.m4
11419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11420 instruct OrL_reg_not_reg(iRegLNoSp dst,
11421 iRegL src1, iRegL src2, immL_M1 m1) %{
11422 match(Set dst (OrL src1 (XorL src2 m1)));
11423 ins_cost(INSN_COST);
11424 format %{ "orn $dst, $src1, $src2" %}
11425
11426 ins_encode %{
11427 __ orn(as_Register($dst$$reg),
11428 as_Register($src1$$reg),
11429 as_Register($src2$$reg),
11430 Assembler::LSL, 0);
11431 %}
11432
11433 ins_pipe(ialu_reg_reg);
11434 %}
11435
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 instruct XorI_reg_not_reg(iRegINoSp dst,
11439 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11440 match(Set dst (XorI m1 (XorI src2 src1)));
11441 ins_cost(INSN_COST);
11442 format %{ "eonw $dst, $src1, $src2" %}
11443
11444 ins_encode %{
11445 __ eonw(as_Register($dst$$reg),
11446 as_Register($src1$$reg),
11447 as_Register($src2$$reg),
11448 Assembler::LSL, 0);
11449 %}
11450
11451 ins_pipe(ialu_reg_reg);
11452 %}
11453
11454 // This pattern is automatically generated from aarch64_ad.m4
11455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11456 instruct XorL_reg_not_reg(iRegLNoSp dst,
11457 iRegL src1, iRegL src2, immL_M1 m1) %{
11458 match(Set dst (XorL m1 (XorL src2 src1)));
11459 ins_cost(INSN_COST);
11460 format %{ "eon $dst, $src1, $src2" %}
11461
11462 ins_encode %{
11463 __ eon(as_Register($dst$$reg),
11464 as_Register($src1$$reg),
11465 as_Register($src2$$reg),
11466 Assembler::LSL, 0);
11467 %}
11468
11469 ins_pipe(ialu_reg_reg);
11470 %}
11471
11472 // This pattern is automatically generated from aarch64_ad.m4
11473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11474 // val & (-1 ^ (val >>> shift)) ==> bicw
11475 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11476 iRegIorL2I src1, iRegIorL2I src2,
11477 immI src3, immI_M1 src4) %{
11478 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11479 ins_cost(1.9 * INSN_COST);
11480 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11481
11482 ins_encode %{
11483 __ bicw(as_Register($dst$$reg),
11484 as_Register($src1$$reg),
11485 as_Register($src2$$reg),
11486 Assembler::LSR,
11487 $src3$$constant & 0x1f);
11488 %}
11489
11490 ins_pipe(ialu_reg_reg_shift);
11491 %}
11492
11493 // This pattern is automatically generated from aarch64_ad.m4
11494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11495 // val & (-1 ^ (val >>> shift)) ==> bic
11496 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11497 iRegL src1, iRegL src2,
11498 immI src3, immL_M1 src4) %{
11499 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11500 ins_cost(1.9 * INSN_COST);
11501 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11502
11503 ins_encode %{
11504 __ bic(as_Register($dst$$reg),
11505 as_Register($src1$$reg),
11506 as_Register($src2$$reg),
11507 Assembler::LSR,
11508 $src3$$constant & 0x3f);
11509 %}
11510
11511 ins_pipe(ialu_reg_reg_shift);
11512 %}
11513
11514 // This pattern is automatically generated from aarch64_ad.m4
11515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11516 // val & (-1 ^ (val >> shift)) ==> bicw
11517 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11518 iRegIorL2I src1, iRegIorL2I src2,
11519 immI src3, immI_M1 src4) %{
11520 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11521 ins_cost(1.9 * INSN_COST);
11522 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11523
11524 ins_encode %{
11525 __ bicw(as_Register($dst$$reg),
11526 as_Register($src1$$reg),
11527 as_Register($src2$$reg),
11528 Assembler::ASR,
11529 $src3$$constant & 0x1f);
11530 %}
11531
11532 ins_pipe(ialu_reg_reg_shift);
11533 %}
11534
11535 // This pattern is automatically generated from aarch64_ad.m4
11536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11537 // val & (-1 ^ (val >> shift)) ==> bic
11538 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11539 iRegL src1, iRegL src2,
11540 immI src3, immL_M1 src4) %{
11541 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11542 ins_cost(1.9 * INSN_COST);
11543 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11544
11545 ins_encode %{
11546 __ bic(as_Register($dst$$reg),
11547 as_Register($src1$$reg),
11548 as_Register($src2$$reg),
11549 Assembler::ASR,
11550 $src3$$constant & 0x3f);
11551 %}
11552
11553 ins_pipe(ialu_reg_reg_shift);
11554 %}
11555
11556 // This pattern is automatically generated from aarch64_ad.m4
11557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11558 // val & (-1 ^ (val ror shift)) ==> bicw
11559 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11560 iRegIorL2I src1, iRegIorL2I src2,
11561 immI src3, immI_M1 src4) %{
11562 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11563 ins_cost(1.9 * INSN_COST);
11564 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11565
11566 ins_encode %{
11567 __ bicw(as_Register($dst$$reg),
11568 as_Register($src1$$reg),
11569 as_Register($src2$$reg),
11570 Assembler::ROR,
11571 $src3$$constant & 0x1f);
11572 %}
11573
11574 ins_pipe(ialu_reg_reg_shift);
11575 %}
11576
11577 // This pattern is automatically generated from aarch64_ad.m4
11578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11579 // val & (-1 ^ (val ror shift)) ==> bic
11580 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11581 iRegL src1, iRegL src2,
11582 immI src3, immL_M1 src4) %{
11583 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11584 ins_cost(1.9 * INSN_COST);
11585 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11586
11587 ins_encode %{
11588 __ bic(as_Register($dst$$reg),
11589 as_Register($src1$$reg),
11590 as_Register($src2$$reg),
11591 Assembler::ROR,
11592 $src3$$constant & 0x3f);
11593 %}
11594
11595 ins_pipe(ialu_reg_reg_shift);
11596 %}
11597
11598 // This pattern is automatically generated from aarch64_ad.m4
11599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11600 // val & (-1 ^ (val << shift)) ==> bicw
11601 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11602 iRegIorL2I src1, iRegIorL2I src2,
11603 immI src3, immI_M1 src4) %{
11604 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11605 ins_cost(1.9 * INSN_COST);
11606 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11607
11608 ins_encode %{
11609 __ bicw(as_Register($dst$$reg),
11610 as_Register($src1$$reg),
11611 as_Register($src2$$reg),
11612 Assembler::LSL,
11613 $src3$$constant & 0x1f);
11614 %}
11615
11616 ins_pipe(ialu_reg_reg_shift);
11617 %}
11618
11619 // This pattern is automatically generated from aarch64_ad.m4
11620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11621 // val & (-1 ^ (val << shift)) ==> bic
11622 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11623 iRegL src1, iRegL src2,
11624 immI src3, immL_M1 src4) %{
11625 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11626 ins_cost(1.9 * INSN_COST);
11627 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11628
11629 ins_encode %{
11630 __ bic(as_Register($dst$$reg),
11631 as_Register($src1$$reg),
11632 as_Register($src2$$reg),
11633 Assembler::LSL,
11634 $src3$$constant & 0x3f);
11635 %}
11636
11637 ins_pipe(ialu_reg_reg_shift);
11638 %}
11639
11640 // This pattern is automatically generated from aarch64_ad.m4
11641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11642 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11643 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11644 iRegIorL2I src1, iRegIorL2I src2,
11645 immI src3, immI_M1 src4) %{
11646 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11647 ins_cost(1.9 * INSN_COST);
11648 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11649
11650 ins_encode %{
11651 __ eonw(as_Register($dst$$reg),
11652 as_Register($src1$$reg),
11653 as_Register($src2$$reg),
11654 Assembler::LSR,
11655 $src3$$constant & 0x1f);
11656 %}
11657
11658 ins_pipe(ialu_reg_reg_shift);
11659 %}
11660
11661 // This pattern is automatically generated from aarch64_ad.m4
11662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11663 // val ^ (-1 ^ (val >>> shift)) ==> eon
11664 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11665 iRegL src1, iRegL src2,
11666 immI src3, immL_M1 src4) %{
11667 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11668 ins_cost(1.9 * INSN_COST);
11669 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11670
11671 ins_encode %{
11672 __ eon(as_Register($dst$$reg),
11673 as_Register($src1$$reg),
11674 as_Register($src2$$reg),
11675 Assembler::LSR,
11676 $src3$$constant & 0x3f);
11677 %}
11678
11679 ins_pipe(ialu_reg_reg_shift);
11680 %}
11681
11682 // This pattern is automatically generated from aarch64_ad.m4
11683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11684 // val ^ (-1 ^ (val >> shift)) ==> eonw
11685 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11686 iRegIorL2I src1, iRegIorL2I src2,
11687 immI src3, immI_M1 src4) %{
11688 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11689 ins_cost(1.9 * INSN_COST);
11690 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11691
11692 ins_encode %{
11693 __ eonw(as_Register($dst$$reg),
11694 as_Register($src1$$reg),
11695 as_Register($src2$$reg),
11696 Assembler::ASR,
11697 $src3$$constant & 0x1f);
11698 %}
11699
11700 ins_pipe(ialu_reg_reg_shift);
11701 %}
11702
11703 // This pattern is automatically generated from aarch64_ad.m4
11704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11705 // val ^ (-1 ^ (val >> shift)) ==> eon
11706 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11707 iRegL src1, iRegL src2,
11708 immI src3, immL_M1 src4) %{
11709 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11710 ins_cost(1.9 * INSN_COST);
11711 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11712
11713 ins_encode %{
11714 __ eon(as_Register($dst$$reg),
11715 as_Register($src1$$reg),
11716 as_Register($src2$$reg),
11717 Assembler::ASR,
11718 $src3$$constant & 0x3f);
11719 %}
11720
11721 ins_pipe(ialu_reg_reg_shift);
11722 %}
11723
11724 // This pattern is automatically generated from aarch64_ad.m4
11725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11726 // val ^ (-1 ^ (val ror shift)) ==> eonw
11727 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11728 iRegIorL2I src1, iRegIorL2I src2,
11729 immI src3, immI_M1 src4) %{
11730 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11731 ins_cost(1.9 * INSN_COST);
11732 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11733
11734 ins_encode %{
11735 __ eonw(as_Register($dst$$reg),
11736 as_Register($src1$$reg),
11737 as_Register($src2$$reg),
11738 Assembler::ROR,
11739 $src3$$constant & 0x1f);
11740 %}
11741
11742 ins_pipe(ialu_reg_reg_shift);
11743 %}
11744
11745 // This pattern is automatically generated from aarch64_ad.m4
11746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11747 // val ^ (-1 ^ (val ror shift)) ==> eon
11748 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11749 iRegL src1, iRegL src2,
11750 immI src3, immL_M1 src4) %{
11751 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11752 ins_cost(1.9 * INSN_COST);
11753 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11754
11755 ins_encode %{
11756 __ eon(as_Register($dst$$reg),
11757 as_Register($src1$$reg),
11758 as_Register($src2$$reg),
11759 Assembler::ROR,
11760 $src3$$constant & 0x3f);
11761 %}
11762
11763 ins_pipe(ialu_reg_reg_shift);
11764 %}
11765
11766 // This pattern is automatically generated from aarch64_ad.m4
11767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11768 // val ^ (-1 ^ (val << shift)) ==> eonw
11769 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11770 iRegIorL2I src1, iRegIorL2I src2,
11771 immI src3, immI_M1 src4) %{
11772 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11773 ins_cost(1.9 * INSN_COST);
11774 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11775
11776 ins_encode %{
11777 __ eonw(as_Register($dst$$reg),
11778 as_Register($src1$$reg),
11779 as_Register($src2$$reg),
11780 Assembler::LSL,
11781 $src3$$constant & 0x1f);
11782 %}
11783
11784 ins_pipe(ialu_reg_reg_shift);
11785 %}
11786
11787 // This pattern is automatically generated from aarch64_ad.m4
11788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11789 // val ^ (-1 ^ (val << shift)) ==> eon
11790 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11791 iRegL src1, iRegL src2,
11792 immI src3, immL_M1 src4) %{
11793 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11794 ins_cost(1.9 * INSN_COST);
11795 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11796
11797 ins_encode %{
11798 __ eon(as_Register($dst$$reg),
11799 as_Register($src1$$reg),
11800 as_Register($src2$$reg),
11801 Assembler::LSL,
11802 $src3$$constant & 0x3f);
11803 %}
11804
11805 ins_pipe(ialu_reg_reg_shift);
11806 %}
11807
11808 // This pattern is automatically generated from aarch64_ad.m4
11809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11810 // val | (-1 ^ (val >>> shift)) ==> ornw
11811 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11812 iRegIorL2I src1, iRegIorL2I src2,
11813 immI src3, immI_M1 src4) %{
11814 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11815 ins_cost(1.9 * INSN_COST);
11816 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11817
11818 ins_encode %{
11819 __ ornw(as_Register($dst$$reg),
11820 as_Register($src1$$reg),
11821 as_Register($src2$$reg),
11822 Assembler::LSR,
11823 $src3$$constant & 0x1f);
11824 %}
11825
11826 ins_pipe(ialu_reg_reg_shift);
11827 %}
11828
11829 // This pattern is automatically generated from aarch64_ad.m4
11830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11831 // val | (-1 ^ (val >>> shift)) ==> orn
11832 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11833 iRegL src1, iRegL src2,
11834 immI src3, immL_M1 src4) %{
11835 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11836 ins_cost(1.9 * INSN_COST);
11837 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11838
11839 ins_encode %{
11840 __ orn(as_Register($dst$$reg),
11841 as_Register($src1$$reg),
11842 as_Register($src2$$reg),
11843 Assembler::LSR,
11844 $src3$$constant & 0x3f);
11845 %}
11846
11847 ins_pipe(ialu_reg_reg_shift);
11848 %}
11849
11850 // This pattern is automatically generated from aarch64_ad.m4
11851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11852 // val | (-1 ^ (val >> shift)) ==> ornw
11853 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11854 iRegIorL2I src1, iRegIorL2I src2,
11855 immI src3, immI_M1 src4) %{
11856 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11857 ins_cost(1.9 * INSN_COST);
11858 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11859
11860 ins_encode %{
11861 __ ornw(as_Register($dst$$reg),
11862 as_Register($src1$$reg),
11863 as_Register($src2$$reg),
11864 Assembler::ASR,
11865 $src3$$constant & 0x1f);
11866 %}
11867
11868 ins_pipe(ialu_reg_reg_shift);
11869 %}
11870
11871 // This pattern is automatically generated from aarch64_ad.m4
11872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11873 // val | (-1 ^ (val >> shift)) ==> orn
11874 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11875 iRegL src1, iRegL src2,
11876 immI src3, immL_M1 src4) %{
11877 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11878 ins_cost(1.9 * INSN_COST);
11879 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11880
11881 ins_encode %{
11882 __ orn(as_Register($dst$$reg),
11883 as_Register($src1$$reg),
11884 as_Register($src2$$reg),
11885 Assembler::ASR,
11886 $src3$$constant & 0x3f);
11887 %}
11888
11889 ins_pipe(ialu_reg_reg_shift);
11890 %}
11891
11892 // This pattern is automatically generated from aarch64_ad.m4
11893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11894 // val | (-1 ^ (val ror shift)) ==> ornw
11895 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11896 iRegIorL2I src1, iRegIorL2I src2,
11897 immI src3, immI_M1 src4) %{
11898 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11899 ins_cost(1.9 * INSN_COST);
11900 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11901
11902 ins_encode %{
11903 __ ornw(as_Register($dst$$reg),
11904 as_Register($src1$$reg),
11905 as_Register($src2$$reg),
11906 Assembler::ROR,
11907 $src3$$constant & 0x1f);
11908 %}
11909
11910 ins_pipe(ialu_reg_reg_shift);
11911 %}
11912
11913 // This pattern is automatically generated from aarch64_ad.m4
11914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11915 // val | (-1 ^ (val ror shift)) ==> orn
11916 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11917 iRegL src1, iRegL src2,
11918 immI src3, immL_M1 src4) %{
11919 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11920 ins_cost(1.9 * INSN_COST);
11921 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11922
11923 ins_encode %{
11924 __ orn(as_Register($dst$$reg),
11925 as_Register($src1$$reg),
11926 as_Register($src2$$reg),
11927 Assembler::ROR,
11928 $src3$$constant & 0x3f);
11929 %}
11930
11931 ins_pipe(ialu_reg_reg_shift);
11932 %}
11933
11934 // This pattern is automatically generated from aarch64_ad.m4
11935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11936 // val | (-1 ^ (val << shift)) ==> ornw
11937 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11938 iRegIorL2I src1, iRegIorL2I src2,
11939 immI src3, immI_M1 src4) %{
11940 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11941 ins_cost(1.9 * INSN_COST);
11942 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11943
11944 ins_encode %{
11945 __ ornw(as_Register($dst$$reg),
11946 as_Register($src1$$reg),
11947 as_Register($src2$$reg),
11948 Assembler::LSL,
11949 $src3$$constant & 0x1f);
11950 %}
11951
11952 ins_pipe(ialu_reg_reg_shift);
11953 %}
11954
11955 // This pattern is automatically generated from aarch64_ad.m4
11956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11957 // val | (-1 ^ (val << shift)) ==> orn
11958 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11959 iRegL src1, iRegL src2,
11960 immI src3, immL_M1 src4) %{
11961 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11962 ins_cost(1.9 * INSN_COST);
11963 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11964
11965 ins_encode %{
11966 __ orn(as_Register($dst$$reg),
11967 as_Register($src1$$reg),
11968 as_Register($src2$$reg),
11969 Assembler::LSL,
11970 $src3$$constant & 0x3f);
11971 %}
11972
11973 ins_pipe(ialu_reg_reg_shift);
11974 %}
11975
11976 // This pattern is automatically generated from aarch64_ad.m4
11977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11978 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11979 iRegIorL2I src1, iRegIorL2I src2,
11980 immI src3) %{
11981 match(Set dst (AndI src1 (URShiftI src2 src3)));
11982
11983 ins_cost(1.9 * INSN_COST);
11984 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11985
11986 ins_encode %{
11987 __ andw(as_Register($dst$$reg),
11988 as_Register($src1$$reg),
11989 as_Register($src2$$reg),
11990 Assembler::LSR,
11991 $src3$$constant & 0x1f);
11992 %}
11993
11994 ins_pipe(ialu_reg_reg_shift);
11995 %}
11996
11997 // This pattern is automatically generated from aarch64_ad.m4
11998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11999 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12000 iRegL src1, iRegL src2,
12001 immI src3) %{
12002 match(Set dst (AndL src1 (URShiftL src2 src3)));
12003
12004 ins_cost(1.9 * INSN_COST);
12005 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12006
12007 ins_encode %{
12008 __ andr(as_Register($dst$$reg),
12009 as_Register($src1$$reg),
12010 as_Register($src2$$reg),
12011 Assembler::LSR,
12012 $src3$$constant & 0x3f);
12013 %}
12014
12015 ins_pipe(ialu_reg_reg_shift);
12016 %}
12017
12018 // This pattern is automatically generated from aarch64_ad.m4
12019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12020 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12021 iRegIorL2I src1, iRegIorL2I src2,
12022 immI src3) %{
12023 match(Set dst (AndI src1 (RShiftI src2 src3)));
12024
12025 ins_cost(1.9 * INSN_COST);
12026 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12027
12028 ins_encode %{
12029 __ andw(as_Register($dst$$reg),
12030 as_Register($src1$$reg),
12031 as_Register($src2$$reg),
12032 Assembler::ASR,
12033 $src3$$constant & 0x1f);
12034 %}
12035
12036 ins_pipe(ialu_reg_reg_shift);
12037 %}
12038
12039 // This pattern is automatically generated from aarch64_ad.m4
12040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12041 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12042 iRegL src1, iRegL src2,
12043 immI src3) %{
12044 match(Set dst (AndL src1 (RShiftL src2 src3)));
12045
12046 ins_cost(1.9 * INSN_COST);
12047 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12048
12049 ins_encode %{
12050 __ andr(as_Register($dst$$reg),
12051 as_Register($src1$$reg),
12052 as_Register($src2$$reg),
12053 Assembler::ASR,
12054 $src3$$constant & 0x3f);
12055 %}
12056
12057 ins_pipe(ialu_reg_reg_shift);
12058 %}
12059
12060 // This pattern is automatically generated from aarch64_ad.m4
12061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12062 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12063 iRegIorL2I src1, iRegIorL2I src2,
12064 immI src3) %{
12065 match(Set dst (AndI src1 (LShiftI src2 src3)));
12066
12067 ins_cost(1.9 * INSN_COST);
12068 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12069
12070 ins_encode %{
12071 __ andw(as_Register($dst$$reg),
12072 as_Register($src1$$reg),
12073 as_Register($src2$$reg),
12074 Assembler::LSL,
12075 $src3$$constant & 0x1f);
12076 %}
12077
12078 ins_pipe(ialu_reg_reg_shift);
12079 %}
12080
12081 // This pattern is automatically generated from aarch64_ad.m4
12082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12083 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12084 iRegL src1, iRegL src2,
12085 immI src3) %{
12086 match(Set dst (AndL src1 (LShiftL src2 src3)));
12087
12088 ins_cost(1.9 * INSN_COST);
12089 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12090
12091 ins_encode %{
12092 __ andr(as_Register($dst$$reg),
12093 as_Register($src1$$reg),
12094 as_Register($src2$$reg),
12095 Assembler::LSL,
12096 $src3$$constant & 0x3f);
12097 %}
12098
12099 ins_pipe(ialu_reg_reg_shift);
12100 %}
12101
12102 // This pattern is automatically generated from aarch64_ad.m4
12103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12104 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12105 iRegIorL2I src1, iRegIorL2I src2,
12106 immI src3) %{
12107 match(Set dst (AndI src1 (RotateRight src2 src3)));
12108
12109 ins_cost(1.9 * INSN_COST);
12110 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12111
12112 ins_encode %{
12113 __ andw(as_Register($dst$$reg),
12114 as_Register($src1$$reg),
12115 as_Register($src2$$reg),
12116 Assembler::ROR,
12117 $src3$$constant & 0x1f);
12118 %}
12119
12120 ins_pipe(ialu_reg_reg_shift);
12121 %}
12122
12123 // This pattern is automatically generated from aarch64_ad.m4
12124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12125 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12126 iRegL src1, iRegL src2,
12127 immI src3) %{
12128 match(Set dst (AndL src1 (RotateRight src2 src3)));
12129
12130 ins_cost(1.9 * INSN_COST);
12131 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12132
12133 ins_encode %{
12134 __ andr(as_Register($dst$$reg),
12135 as_Register($src1$$reg),
12136 as_Register($src2$$reg),
12137 Assembler::ROR,
12138 $src3$$constant & 0x3f);
12139 %}
12140
12141 ins_pipe(ialu_reg_reg_shift);
12142 %}
12143
12144 // This pattern is automatically generated from aarch64_ad.m4
12145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12146 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12147 iRegIorL2I src1, iRegIorL2I src2,
12148 immI src3) %{
12149 match(Set dst (XorI src1 (URShiftI src2 src3)));
12150
12151 ins_cost(1.9 * INSN_COST);
12152 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12153
12154 ins_encode %{
12155 __ eorw(as_Register($dst$$reg),
12156 as_Register($src1$$reg),
12157 as_Register($src2$$reg),
12158 Assembler::LSR,
12159 $src3$$constant & 0x1f);
12160 %}
12161
12162 ins_pipe(ialu_reg_reg_shift);
12163 %}
12164
12165 // This pattern is automatically generated from aarch64_ad.m4
12166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12167 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12168 iRegL src1, iRegL src2,
12169 immI src3) %{
12170 match(Set dst (XorL src1 (URShiftL src2 src3)));
12171
12172 ins_cost(1.9 * INSN_COST);
12173 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12174
12175 ins_encode %{
12176 __ eor(as_Register($dst$$reg),
12177 as_Register($src1$$reg),
12178 as_Register($src2$$reg),
12179 Assembler::LSR,
12180 $src3$$constant & 0x3f);
12181 %}
12182
12183 ins_pipe(ialu_reg_reg_shift);
12184 %}
12185
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12189 iRegIorL2I src1, iRegIorL2I src2,
12190 immI src3) %{
12191 match(Set dst (XorI src1 (RShiftI src2 src3)));
12192
12193 ins_cost(1.9 * INSN_COST);
12194 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12195
12196 ins_encode %{
12197 __ eorw(as_Register($dst$$reg),
12198 as_Register($src1$$reg),
12199 as_Register($src2$$reg),
12200 Assembler::ASR,
12201 $src3$$constant & 0x1f);
12202 %}
12203
12204 ins_pipe(ialu_reg_reg_shift);
12205 %}
12206
12207 // This pattern is automatically generated from aarch64_ad.m4
12208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12209 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12210 iRegL src1, iRegL src2,
12211 immI src3) %{
12212 match(Set dst (XorL src1 (RShiftL src2 src3)));
12213
12214 ins_cost(1.9 * INSN_COST);
12215 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12216
12217 ins_encode %{
12218 __ eor(as_Register($dst$$reg),
12219 as_Register($src1$$reg),
12220 as_Register($src2$$reg),
12221 Assembler::ASR,
12222 $src3$$constant & 0x3f);
12223 %}
12224
12225 ins_pipe(ialu_reg_reg_shift);
12226 %}
12227
12228 // This pattern is automatically generated from aarch64_ad.m4
12229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12230 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12231 iRegIorL2I src1, iRegIorL2I src2,
12232 immI src3) %{
12233 match(Set dst (XorI src1 (LShiftI src2 src3)));
12234
12235 ins_cost(1.9 * INSN_COST);
12236 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12237
12238 ins_encode %{
12239 __ eorw(as_Register($dst$$reg),
12240 as_Register($src1$$reg),
12241 as_Register($src2$$reg),
12242 Assembler::LSL,
12243 $src3$$constant & 0x1f);
12244 %}
12245
12246 ins_pipe(ialu_reg_reg_shift);
12247 %}
12248
12249 // This pattern is automatically generated from aarch64_ad.m4
12250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12251 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12252 iRegL src1, iRegL src2,
12253 immI src3) %{
12254 match(Set dst (XorL src1 (LShiftL src2 src3)));
12255
12256 ins_cost(1.9 * INSN_COST);
12257 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12258
12259 ins_encode %{
12260 __ eor(as_Register($dst$$reg),
12261 as_Register($src1$$reg),
12262 as_Register($src2$$reg),
12263 Assembler::LSL,
12264 $src3$$constant & 0x3f);
12265 %}
12266
12267 ins_pipe(ialu_reg_reg_shift);
12268 %}
12269
12270 // This pattern is automatically generated from aarch64_ad.m4
12271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12272 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12273 iRegIorL2I src1, iRegIorL2I src2,
12274 immI src3) %{
12275 match(Set dst (XorI src1 (RotateRight src2 src3)));
12276
12277 ins_cost(1.9 * INSN_COST);
12278 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12279
12280 ins_encode %{
12281 __ eorw(as_Register($dst$$reg),
12282 as_Register($src1$$reg),
12283 as_Register($src2$$reg),
12284 Assembler::ROR,
12285 $src3$$constant & 0x1f);
12286 %}
12287
12288 ins_pipe(ialu_reg_reg_shift);
12289 %}
12290
12291 // This pattern is automatically generated from aarch64_ad.m4
12292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12293 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12294 iRegL src1, iRegL src2,
12295 immI src3) %{
12296 match(Set dst (XorL src1 (RotateRight src2 src3)));
12297
12298 ins_cost(1.9 * INSN_COST);
12299 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12300
12301 ins_encode %{
12302 __ eor(as_Register($dst$$reg),
12303 as_Register($src1$$reg),
12304 as_Register($src2$$reg),
12305 Assembler::ROR,
12306 $src3$$constant & 0x3f);
12307 %}
12308
12309 ins_pipe(ialu_reg_reg_shift);
12310 %}
12311
12312 // This pattern is automatically generated from aarch64_ad.m4
12313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12314 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12315 iRegIorL2I src1, iRegIorL2I src2,
12316 immI src3) %{
12317 match(Set dst (OrI src1 (URShiftI src2 src3)));
12318
12319 ins_cost(1.9 * INSN_COST);
12320 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12321
12322 ins_encode %{
12323 __ orrw(as_Register($dst$$reg),
12324 as_Register($src1$$reg),
12325 as_Register($src2$$reg),
12326 Assembler::LSR,
12327 $src3$$constant & 0x1f);
12328 %}
12329
12330 ins_pipe(ialu_reg_reg_shift);
12331 %}
12332
12333 // This pattern is automatically generated from aarch64_ad.m4
12334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12335 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12336 iRegL src1, iRegL src2,
12337 immI src3) %{
12338 match(Set dst (OrL src1 (URShiftL src2 src3)));
12339
12340 ins_cost(1.9 * INSN_COST);
12341 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12342
12343 ins_encode %{
12344 __ orr(as_Register($dst$$reg),
12345 as_Register($src1$$reg),
12346 as_Register($src2$$reg),
12347 Assembler::LSR,
12348 $src3$$constant & 0x3f);
12349 %}
12350
12351 ins_pipe(ialu_reg_reg_shift);
12352 %}
12353
12354 // This pattern is automatically generated from aarch64_ad.m4
12355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12356 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12357 iRegIorL2I src1, iRegIorL2I src2,
12358 immI src3) %{
12359 match(Set dst (OrI src1 (RShiftI src2 src3)));
12360
12361 ins_cost(1.9 * INSN_COST);
12362 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12363
12364 ins_encode %{
12365 __ orrw(as_Register($dst$$reg),
12366 as_Register($src1$$reg),
12367 as_Register($src2$$reg),
12368 Assembler::ASR,
12369 $src3$$constant & 0x1f);
12370 %}
12371
12372 ins_pipe(ialu_reg_reg_shift);
12373 %}
12374
12375 // This pattern is automatically generated from aarch64_ad.m4
12376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12377 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12378 iRegL src1, iRegL src2,
12379 immI src3) %{
12380 match(Set dst (OrL src1 (RShiftL src2 src3)));
12381
12382 ins_cost(1.9 * INSN_COST);
12383 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12384
12385 ins_encode %{
12386 __ orr(as_Register($dst$$reg),
12387 as_Register($src1$$reg),
12388 as_Register($src2$$reg),
12389 Assembler::ASR,
12390 $src3$$constant & 0x3f);
12391 %}
12392
12393 ins_pipe(ialu_reg_reg_shift);
12394 %}
12395
12396 // This pattern is automatically generated from aarch64_ad.m4
12397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12398 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12399 iRegIorL2I src1, iRegIorL2I src2,
12400 immI src3) %{
12401 match(Set dst (OrI src1 (LShiftI src2 src3)));
12402
12403 ins_cost(1.9 * INSN_COST);
12404 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12405
12406 ins_encode %{
12407 __ orrw(as_Register($dst$$reg),
12408 as_Register($src1$$reg),
12409 as_Register($src2$$reg),
12410 Assembler::LSL,
12411 $src3$$constant & 0x1f);
12412 %}
12413
12414 ins_pipe(ialu_reg_reg_shift);
12415 %}
12416
12417 // This pattern is automatically generated from aarch64_ad.m4
12418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12419 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12420 iRegL src1, iRegL src2,
12421 immI src3) %{
12422 match(Set dst (OrL src1 (LShiftL src2 src3)));
12423
12424 ins_cost(1.9 * INSN_COST);
12425 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12426
12427 ins_encode %{
12428 __ orr(as_Register($dst$$reg),
12429 as_Register($src1$$reg),
12430 as_Register($src2$$reg),
12431 Assembler::LSL,
12432 $src3$$constant & 0x3f);
12433 %}
12434
12435 ins_pipe(ialu_reg_reg_shift);
12436 %}
12437
12438 // This pattern is automatically generated from aarch64_ad.m4
12439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12440 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12441 iRegIorL2I src1, iRegIorL2I src2,
12442 immI src3) %{
12443 match(Set dst (OrI src1 (RotateRight src2 src3)));
12444
12445 ins_cost(1.9 * INSN_COST);
12446 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12447
12448 ins_encode %{
12449 __ orrw(as_Register($dst$$reg),
12450 as_Register($src1$$reg),
12451 as_Register($src2$$reg),
12452 Assembler::ROR,
12453 $src3$$constant & 0x1f);
12454 %}
12455
12456 ins_pipe(ialu_reg_reg_shift);
12457 %}
12458
12459 // This pattern is automatically generated from aarch64_ad.m4
12460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12461 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12462 iRegL src1, iRegL src2,
12463 immI src3) %{
12464 match(Set dst (OrL src1 (RotateRight src2 src3)));
12465
12466 ins_cost(1.9 * INSN_COST);
12467 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12468
12469 ins_encode %{
12470 __ orr(as_Register($dst$$reg),
12471 as_Register($src1$$reg),
12472 as_Register($src2$$reg),
12473 Assembler::ROR,
12474 $src3$$constant & 0x3f);
12475 %}
12476
12477 ins_pipe(ialu_reg_reg_shift);
12478 %}
12479
12480 // This pattern is automatically generated from aarch64_ad.m4
12481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12482 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12483 iRegIorL2I src1, iRegIorL2I src2,
12484 immI src3) %{
12485 match(Set dst (AddI src1 (URShiftI src2 src3)));
12486
12487 ins_cost(1.9 * INSN_COST);
12488 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12489
12490 ins_encode %{
12491 __ addw(as_Register($dst$$reg),
12492 as_Register($src1$$reg),
12493 as_Register($src2$$reg),
12494 Assembler::LSR,
12495 $src3$$constant & 0x1f);
12496 %}
12497
12498 ins_pipe(ialu_reg_reg_shift);
12499 %}
12500
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12504 iRegL src1, iRegL src2,
12505 immI src3) %{
12506 match(Set dst (AddL src1 (URShiftL src2 src3)));
12507
12508 ins_cost(1.9 * INSN_COST);
12509 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12510
12511 ins_encode %{
12512 __ add(as_Register($dst$$reg),
12513 as_Register($src1$$reg),
12514 as_Register($src2$$reg),
12515 Assembler::LSR,
12516 $src3$$constant & 0x3f);
12517 %}
12518
12519 ins_pipe(ialu_reg_reg_shift);
12520 %}
12521
12522 // This pattern is automatically generated from aarch64_ad.m4
12523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12524 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12525 iRegIorL2I src1, iRegIorL2I src2,
12526 immI src3) %{
12527 match(Set dst (AddI src1 (RShiftI src2 src3)));
12528
12529 ins_cost(1.9 * INSN_COST);
12530 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12531
12532 ins_encode %{
12533 __ addw(as_Register($dst$$reg),
12534 as_Register($src1$$reg),
12535 as_Register($src2$$reg),
12536 Assembler::ASR,
12537 $src3$$constant & 0x1f);
12538 %}
12539
12540 ins_pipe(ialu_reg_reg_shift);
12541 %}
12542
12543 // This pattern is automatically generated from aarch64_ad.m4
12544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12545 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12546 iRegL src1, iRegL src2,
12547 immI src3) %{
12548 match(Set dst (AddL src1 (RShiftL src2 src3)));
12549
12550 ins_cost(1.9 * INSN_COST);
12551 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12552
12553 ins_encode %{
12554 __ add(as_Register($dst$$reg),
12555 as_Register($src1$$reg),
12556 as_Register($src2$$reg),
12557 Assembler::ASR,
12558 $src3$$constant & 0x3f);
12559 %}
12560
12561 ins_pipe(ialu_reg_reg_shift);
12562 %}
12563
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12567 iRegIorL2I src1, iRegIorL2I src2,
12568 immI src3) %{
12569 match(Set dst (AddI src1 (LShiftI src2 src3)));
12570
12571 ins_cost(1.9 * INSN_COST);
12572 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12573
12574 ins_encode %{
12575 __ addw(as_Register($dst$$reg),
12576 as_Register($src1$$reg),
12577 as_Register($src2$$reg),
12578 Assembler::LSL,
12579 $src3$$constant & 0x1f);
12580 %}
12581
12582 ins_pipe(ialu_reg_reg_shift);
12583 %}
12584
12585 // This pattern is automatically generated from aarch64_ad.m4
12586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12587 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12588 iRegL src1, iRegL src2,
12589 immI src3) %{
12590 match(Set dst (AddL src1 (LShiftL src2 src3)));
12591
12592 ins_cost(1.9 * INSN_COST);
12593 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12594
12595 ins_encode %{
12596 __ add(as_Register($dst$$reg),
12597 as_Register($src1$$reg),
12598 as_Register($src2$$reg),
12599 Assembler::LSL,
12600 $src3$$constant & 0x3f);
12601 %}
12602
12603 ins_pipe(ialu_reg_reg_shift);
12604 %}
12605
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12609 iRegIorL2I src1, iRegIorL2I src2,
12610 immI src3) %{
12611 match(Set dst (SubI src1 (URShiftI src2 src3)));
12612
12613 ins_cost(1.9 * INSN_COST);
12614 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12615
12616 ins_encode %{
12617 __ subw(as_Register($dst$$reg),
12618 as_Register($src1$$reg),
12619 as_Register($src2$$reg),
12620 Assembler::LSR,
12621 $src3$$constant & 0x1f);
12622 %}
12623
12624 ins_pipe(ialu_reg_reg_shift);
12625 %}
12626
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12630 iRegL src1, iRegL src2,
12631 immI src3) %{
12632 match(Set dst (SubL src1 (URShiftL src2 src3)));
12633
12634 ins_cost(1.9 * INSN_COST);
12635 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12636
12637 ins_encode %{
12638 __ sub(as_Register($dst$$reg),
12639 as_Register($src1$$reg),
12640 as_Register($src2$$reg),
12641 Assembler::LSR,
12642 $src3$$constant & 0x3f);
12643 %}
12644
12645 ins_pipe(ialu_reg_reg_shift);
12646 %}
12647
12648 // This pattern is automatically generated from aarch64_ad.m4
12649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12650 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12651 iRegIorL2I src1, iRegIorL2I src2,
12652 immI src3) %{
12653 match(Set dst (SubI src1 (RShiftI src2 src3)));
12654
12655 ins_cost(1.9 * INSN_COST);
12656 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12657
12658 ins_encode %{
12659 __ subw(as_Register($dst$$reg),
12660 as_Register($src1$$reg),
12661 as_Register($src2$$reg),
12662 Assembler::ASR,
12663 $src3$$constant & 0x1f);
12664 %}
12665
12666 ins_pipe(ialu_reg_reg_shift);
12667 %}
12668
12669 // This pattern is automatically generated from aarch64_ad.m4
12670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12671 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12672 iRegL src1, iRegL src2,
12673 immI src3) %{
12674 match(Set dst (SubL src1 (RShiftL src2 src3)));
12675
12676 ins_cost(1.9 * INSN_COST);
12677 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12678
12679 ins_encode %{
12680 __ sub(as_Register($dst$$reg),
12681 as_Register($src1$$reg),
12682 as_Register($src2$$reg),
12683 Assembler::ASR,
12684 $src3$$constant & 0x3f);
12685 %}
12686
12687 ins_pipe(ialu_reg_reg_shift);
12688 %}
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12693 iRegIorL2I src1, iRegIorL2I src2,
12694 immI src3) %{
12695 match(Set dst (SubI src1 (LShiftI src2 src3)));
12696
12697 ins_cost(1.9 * INSN_COST);
12698 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12699
12700 ins_encode %{
12701 __ subw(as_Register($dst$$reg),
12702 as_Register($src1$$reg),
12703 as_Register($src2$$reg),
12704 Assembler::LSL,
12705 $src3$$constant & 0x1f);
12706 %}
12707
12708 ins_pipe(ialu_reg_reg_shift);
12709 %}
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12714 iRegL src1, iRegL src2,
12715 immI src3) %{
12716 match(Set dst (SubL src1 (LShiftL src2 src3)));
12717
12718 ins_cost(1.9 * INSN_COST);
12719 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12720
12721 ins_encode %{
12722 __ sub(as_Register($dst$$reg),
12723 as_Register($src1$$reg),
12724 as_Register($src2$$reg),
12725 Assembler::LSL,
12726 $src3$$constant & 0x3f);
12727 %}
12728
12729 ins_pipe(ialu_reg_reg_shift);
12730 %}
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734
12735 // Shift Left followed by Shift Right.
12736 // This idiom is used by the compiler for the i2b bytecode etc.
12737 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12738 %{
12739 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12740 ins_cost(INSN_COST * 2);
12741 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12742 ins_encode %{
12743 int lshift = $lshift_count$$constant & 63;
12744 int rshift = $rshift_count$$constant & 63;
12745 int s = 63 - lshift;
12746 int r = (rshift - lshift) & 63;
12747 __ sbfm(as_Register($dst$$reg),
12748 as_Register($src$$reg),
12749 r, s);
12750 %}
12751
12752 ins_pipe(ialu_reg_shift);
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757
12758 // Shift Left followed by Shift Right.
12759 // This idiom is used by the compiler for the i2b bytecode etc.
12760 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12761 %{
12762 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12763 ins_cost(INSN_COST * 2);
12764 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12765 ins_encode %{
12766 int lshift = $lshift_count$$constant & 31;
12767 int rshift = $rshift_count$$constant & 31;
12768 int s = 31 - lshift;
12769 int r = (rshift - lshift) & 31;
12770 __ sbfmw(as_Register($dst$$reg),
12771 as_Register($src$$reg),
12772 r, s);
12773 %}
12774
12775 ins_pipe(ialu_reg_shift);
12776 %}
12777
12778 // This pattern is automatically generated from aarch64_ad.m4
12779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12780
12781 // Shift Left followed by Shift Right.
12782 // This idiom is used by the compiler for the i2b bytecode etc.
12783 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12784 %{
12785 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12786 ins_cost(INSN_COST * 2);
12787 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12788 ins_encode %{
12789 int lshift = $lshift_count$$constant & 63;
12790 int rshift = $rshift_count$$constant & 63;
12791 int s = 63 - lshift;
12792 int r = (rshift - lshift) & 63;
12793 __ ubfm(as_Register($dst$$reg),
12794 as_Register($src$$reg),
12795 r, s);
12796 %}
12797
12798 ins_pipe(ialu_reg_shift);
12799 %}
12800
12801 // This pattern is automatically generated from aarch64_ad.m4
12802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12803
12804 // Shift Left followed by Shift Right.
12805 // This idiom is used by the compiler for the i2b bytecode etc.
12806 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12807 %{
12808 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12809 ins_cost(INSN_COST * 2);
12810 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12811 ins_encode %{
12812 int lshift = $lshift_count$$constant & 31;
12813 int rshift = $rshift_count$$constant & 31;
12814 int s = 31 - lshift;
12815 int r = (rshift - lshift) & 31;
12816 __ ubfmw(as_Register($dst$$reg),
12817 as_Register($src$$reg),
12818 r, s);
12819 %}
12820
12821 ins_pipe(ialu_reg_shift);
12822 %}
12823
12824 // Bitfield extract with shift & mask
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12829 %{
12830 match(Set dst (AndI (URShiftI src rshift) mask));
12831 // Make sure we are not going to exceed what ubfxw can do.
12832 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12833
12834 ins_cost(INSN_COST);
12835 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12836 ins_encode %{
12837 int rshift = $rshift$$constant & 31;
12838 intptr_t mask = $mask$$constant;
12839 int width = exact_log2(mask+1);
12840 __ ubfxw(as_Register($dst$$reg),
12841 as_Register($src$$reg), rshift, width);
12842 %}
12843 ins_pipe(ialu_reg_shift);
12844 %}
12845
12846 // This pattern is automatically generated from aarch64_ad.m4
12847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12848 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12849 %{
12850 match(Set dst (AndL (URShiftL src rshift) mask));
12851 // Make sure we are not going to exceed what ubfx can do.
12852 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12853
12854 ins_cost(INSN_COST);
12855 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12856 ins_encode %{
12857 int rshift = $rshift$$constant & 63;
12858 intptr_t mask = $mask$$constant;
12859 int width = exact_log2_long(mask+1);
12860 __ ubfx(as_Register($dst$$reg),
12861 as_Register($src$$reg), rshift, width);
12862 %}
12863 ins_pipe(ialu_reg_shift);
12864 %}
12865
12866
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869
12870 // We can use ubfx when extending an And with a mask when we know mask
12871 // is positive. We know that because immI_bitmask guarantees it.
12872 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12873 %{
12874 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12875 // Make sure we are not going to exceed what ubfxw can do.
12876 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12877
12878 ins_cost(INSN_COST * 2);
12879 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12880 ins_encode %{
12881 int rshift = $rshift$$constant & 31;
12882 intptr_t mask = $mask$$constant;
12883 int width = exact_log2(mask+1);
12884 __ ubfx(as_Register($dst$$reg),
12885 as_Register($src$$reg), rshift, width);
12886 %}
12887 ins_pipe(ialu_reg_shift);
12888 %}
12889
12890
12891 // This pattern is automatically generated from aarch64_ad.m4
12892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12893
12894 // We can use ubfiz when masking by a positive number and then left shifting the result.
12895 // We know that the mask is positive because immI_bitmask guarantees it.
12896 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12897 %{
12898 match(Set dst (LShiftI (AndI src mask) lshift));
12899 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12900
12901 ins_cost(INSN_COST);
12902 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12903 ins_encode %{
12904 int lshift = $lshift$$constant & 31;
12905 intptr_t mask = $mask$$constant;
12906 int width = exact_log2(mask+1);
12907 __ ubfizw(as_Register($dst$$reg),
12908 as_Register($src$$reg), lshift, width);
12909 %}
12910 ins_pipe(ialu_reg_shift);
12911 %}
12912
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915
12916 // We can use ubfiz when masking by a positive number and then left shifting the result.
12917 // We know that the mask is positive because immL_bitmask guarantees it.
12918 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12919 %{
12920 match(Set dst (LShiftL (AndL src mask) lshift));
12921 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12922
12923 ins_cost(INSN_COST);
12924 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12925 ins_encode %{
12926 int lshift = $lshift$$constant & 63;
12927 intptr_t mask = $mask$$constant;
12928 int width = exact_log2_long(mask+1);
12929 __ ubfiz(as_Register($dst$$reg),
12930 as_Register($src$$reg), lshift, width);
12931 %}
12932 ins_pipe(ialu_reg_shift);
12933 %}
12934
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937
12938 // We can use ubfiz when masking by a positive number and then left shifting the result.
12939 // We know that the mask is positive because immI_bitmask guarantees it.
12940 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12941 %{
12942 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12943 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12944
12945 ins_cost(INSN_COST);
12946 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12947 ins_encode %{
12948 int lshift = $lshift$$constant & 31;
12949 intptr_t mask = $mask$$constant;
12950 int width = exact_log2(mask+1);
12951 __ ubfizw(as_Register($dst$$reg),
12952 as_Register($src$$reg), lshift, width);
12953 %}
12954 ins_pipe(ialu_reg_shift);
12955 %}
12956
12957 // This pattern is automatically generated from aarch64_ad.m4
12958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12959
12960 // We can use ubfiz when masking by a positive number and then left shifting the result.
12961 // We know that the mask is positive because immL_bitmask guarantees it.
12962 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12963 %{
12964 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12965 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12966
12967 ins_cost(INSN_COST);
12968 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12969 ins_encode %{
12970 int lshift = $lshift$$constant & 63;
12971 intptr_t mask = $mask$$constant;
12972 int width = exact_log2_long(mask+1);
12973 __ ubfiz(as_Register($dst$$reg),
12974 as_Register($src$$reg), lshift, width);
12975 %}
12976 ins_pipe(ialu_reg_shift);
12977 %}
12978
12979
12980 // This pattern is automatically generated from aarch64_ad.m4
12981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12982
12983 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12984 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12985 %{
12986 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12987 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12988
12989 ins_cost(INSN_COST);
12990 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12991 ins_encode %{
12992 int lshift = $lshift$$constant & 63;
12993 intptr_t mask = $mask$$constant;
12994 int width = exact_log2(mask+1);
12995 __ ubfiz(as_Register($dst$$reg),
12996 as_Register($src$$reg), lshift, width);
12997 %}
12998 ins_pipe(ialu_reg_shift);
12999 %}
13000
13001 // This pattern is automatically generated from aarch64_ad.m4
13002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13003
13004 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13005 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13006 %{
13007 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13008 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13009
13010 ins_cost(INSN_COST);
13011 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13012 ins_encode %{
13013 int lshift = $lshift$$constant & 31;
13014 intptr_t mask = $mask$$constant;
13015 int width = exact_log2(mask+1);
13016 __ ubfiz(as_Register($dst$$reg),
13017 as_Register($src$$reg), lshift, width);
13018 %}
13019 ins_pipe(ialu_reg_shift);
13020 %}
13021
13022 // This pattern is automatically generated from aarch64_ad.m4
13023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13024
13025 // Can skip int2long conversions after AND with small bitmask
13026 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13027 %{
13028 match(Set dst (ConvI2L (AndI src msk)));
13029 ins_cost(INSN_COST);
13030 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13031 ins_encode %{
13032 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13033 %}
13034 ins_pipe(ialu_reg_shift);
13035 %}
13036
13037
13038 // Rotations
13039
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13043 %{
13044 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13045 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13046
13047 ins_cost(INSN_COST);
13048 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13049
13050 ins_encode %{
13051 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13052 $rshift$$constant & 63);
13053 %}
13054 ins_pipe(ialu_reg_reg_extr);
13055 %}
13056
13057
13058 // This pattern is automatically generated from aarch64_ad.m4
13059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13060 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13061 %{
13062 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13063 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13064
13065 ins_cost(INSN_COST);
13066 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13067
13068 ins_encode %{
13069 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13070 $rshift$$constant & 31);
13071 %}
13072 ins_pipe(ialu_reg_reg_extr);
13073 %}
13074
13075
13076 // This pattern is automatically generated from aarch64_ad.m4
13077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13078 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13079 %{
13080 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13081 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13082
13083 ins_cost(INSN_COST);
13084 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13085
13086 ins_encode %{
13087 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13088 $rshift$$constant & 63);
13089 %}
13090 ins_pipe(ialu_reg_reg_extr);
13091 %}
13092
13093
13094 // This pattern is automatically generated from aarch64_ad.m4
13095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13096 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13097 %{
13098 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13099 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13100
13101 ins_cost(INSN_COST);
13102 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13103
13104 ins_encode %{
13105 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13106 $rshift$$constant & 31);
13107 %}
13108 ins_pipe(ialu_reg_reg_extr);
13109 %}
13110
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13114 %{
13115 match(Set dst (RotateRight src shift));
13116
13117 ins_cost(INSN_COST);
13118 format %{ "ror $dst, $src, $shift" %}
13119
13120 ins_encode %{
13121 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13122 $shift$$constant & 0x1f);
13123 %}
13124 ins_pipe(ialu_reg_reg_vshift);
13125 %}
13126
13127 // This pattern is automatically generated from aarch64_ad.m4
13128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13129 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13130 %{
13131 match(Set dst (RotateRight src shift));
13132
13133 ins_cost(INSN_COST);
13134 format %{ "ror $dst, $src, $shift" %}
13135
13136 ins_encode %{
13137 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13138 $shift$$constant & 0x3f);
13139 %}
13140 ins_pipe(ialu_reg_reg_vshift);
13141 %}
13142
13143 // This pattern is automatically generated from aarch64_ad.m4
13144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13145 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13146 %{
13147 match(Set dst (RotateRight src shift));
13148
13149 ins_cost(INSN_COST);
13150 format %{ "ror $dst, $src, $shift" %}
13151
13152 ins_encode %{
13153 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13154 %}
13155 ins_pipe(ialu_reg_reg_vshift);
13156 %}
13157
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13161 %{
13162 match(Set dst (RotateRight src shift));
13163
13164 ins_cost(INSN_COST);
13165 format %{ "ror $dst, $src, $shift" %}
13166
13167 ins_encode %{
13168 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13169 %}
13170 ins_pipe(ialu_reg_reg_vshift);
13171 %}
13172
13173 // This pattern is automatically generated from aarch64_ad.m4
13174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13175 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13176 %{
13177 match(Set dst (RotateLeft src shift));
13178
13179 ins_cost(INSN_COST);
13180 format %{ "rol $dst, $src, $shift" %}
13181
13182 ins_encode %{
13183 __ subw(rscratch1, zr, as_Register($shift$$reg));
13184 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13185 %}
13186 ins_pipe(ialu_reg_reg_vshift);
13187 %}
13188
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13192 %{
13193 match(Set dst (RotateLeft src shift));
13194
13195 ins_cost(INSN_COST);
13196 format %{ "rol $dst, $src, $shift" %}
13197
13198 ins_encode %{
13199 __ subw(rscratch1, zr, as_Register($shift$$reg));
13200 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13201 %}
13202 ins_pipe(ialu_reg_reg_vshift);
13203 %}
13204
13205
13206 // Add/subtract (extended)
13207
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13211 %{
13212 match(Set dst (AddL src1 (ConvI2L src2)));
13213 ins_cost(INSN_COST);
13214 format %{ "add $dst, $src1, $src2, sxtw" %}
13215
13216 ins_encode %{
13217 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13218 as_Register($src2$$reg), ext::sxtw);
13219 %}
13220 ins_pipe(ialu_reg_reg);
13221 %}
13222
13223 // This pattern is automatically generated from aarch64_ad.m4
13224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13225 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13226 %{
13227 match(Set dst (SubL src1 (ConvI2L src2)));
13228 ins_cost(INSN_COST);
13229 format %{ "sub $dst, $src1, $src2, sxtw" %}
13230
13231 ins_encode %{
13232 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13233 as_Register($src2$$reg), ext::sxtw);
13234 %}
13235 ins_pipe(ialu_reg_reg);
13236 %}
13237
13238 // This pattern is automatically generated from aarch64_ad.m4
13239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13240 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13241 %{
13242 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13243 ins_cost(INSN_COST);
13244 format %{ "add $dst, $src1, $src2, sxth" %}
13245
13246 ins_encode %{
13247 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13248 as_Register($src2$$reg), ext::sxth);
13249 %}
13250 ins_pipe(ialu_reg_reg);
13251 %}
13252
13253 // This pattern is automatically generated from aarch64_ad.m4
13254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13255 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13256 %{
13257 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13258 ins_cost(INSN_COST);
13259 format %{ "add $dst, $src1, $src2, sxtb" %}
13260
13261 ins_encode %{
13262 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13263 as_Register($src2$$reg), ext::sxtb);
13264 %}
13265 ins_pipe(ialu_reg_reg);
13266 %}
13267
13268 // This pattern is automatically generated from aarch64_ad.m4
13269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13270 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13271 %{
13272 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13273 ins_cost(INSN_COST);
13274 format %{ "add $dst, $src1, $src2, uxtb" %}
13275
13276 ins_encode %{
13277 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13278 as_Register($src2$$reg), ext::uxtb);
13279 %}
13280 ins_pipe(ialu_reg_reg);
13281 %}
13282
13283 // This pattern is automatically generated from aarch64_ad.m4
13284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13285 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13286 %{
13287 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13288 ins_cost(INSN_COST);
13289 format %{ "add $dst, $src1, $src2, sxth" %}
13290
13291 ins_encode %{
13292 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13293 as_Register($src2$$reg), ext::sxth);
13294 %}
13295 ins_pipe(ialu_reg_reg);
13296 %}
13297
13298 // This pattern is automatically generated from aarch64_ad.m4
13299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13300 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13301 %{
13302 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13303 ins_cost(INSN_COST);
13304 format %{ "add $dst, $src1, $src2, sxtw" %}
13305
13306 ins_encode %{
13307 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13308 as_Register($src2$$reg), ext::sxtw);
13309 %}
13310 ins_pipe(ialu_reg_reg);
13311 %}
13312
13313 // This pattern is automatically generated from aarch64_ad.m4
13314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13315 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13316 %{
13317 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13318 ins_cost(INSN_COST);
13319 format %{ "add $dst, $src1, $src2, sxtb" %}
13320
13321 ins_encode %{
13322 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13323 as_Register($src2$$reg), ext::sxtb);
13324 %}
13325 ins_pipe(ialu_reg_reg);
13326 %}
13327
13328 // This pattern is automatically generated from aarch64_ad.m4
13329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13330 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13331 %{
13332 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13333 ins_cost(INSN_COST);
13334 format %{ "add $dst, $src1, $src2, uxtb" %}
13335
13336 ins_encode %{
13337 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13338 as_Register($src2$$reg), ext::uxtb);
13339 %}
13340 ins_pipe(ialu_reg_reg);
13341 %}
13342
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13346 %{
13347 match(Set dst (AddI src1 (AndI src2 mask)));
13348 ins_cost(INSN_COST);
13349 format %{ "addw $dst, $src1, $src2, uxtb" %}
13350
13351 ins_encode %{
13352 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13353 as_Register($src2$$reg), ext::uxtb);
13354 %}
13355 ins_pipe(ialu_reg_reg);
13356 %}
13357
13358 // This pattern is automatically generated from aarch64_ad.m4
13359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13360 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13361 %{
13362 match(Set dst (AddI src1 (AndI src2 mask)));
13363 ins_cost(INSN_COST);
13364 format %{ "addw $dst, $src1, $src2, uxth" %}
13365
13366 ins_encode %{
13367 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13368 as_Register($src2$$reg), ext::uxth);
13369 %}
13370 ins_pipe(ialu_reg_reg);
13371 %}
13372
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13376 %{
13377 match(Set dst (AddL src1 (AndL src2 mask)));
13378 ins_cost(INSN_COST);
13379 format %{ "add $dst, $src1, $src2, uxtb" %}
13380
13381 ins_encode %{
13382 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13383 as_Register($src2$$reg), ext::uxtb);
13384 %}
13385 ins_pipe(ialu_reg_reg);
13386 %}
13387
13388 // This pattern is automatically generated from aarch64_ad.m4
13389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13390 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13391 %{
13392 match(Set dst (AddL src1 (AndL src2 mask)));
13393 ins_cost(INSN_COST);
13394 format %{ "add $dst, $src1, $src2, uxth" %}
13395
13396 ins_encode %{
13397 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13398 as_Register($src2$$reg), ext::uxth);
13399 %}
13400 ins_pipe(ialu_reg_reg);
13401 %}
13402
13403 // This pattern is automatically generated from aarch64_ad.m4
13404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13405 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13406 %{
13407 match(Set dst (AddL src1 (AndL src2 mask)));
13408 ins_cost(INSN_COST);
13409 format %{ "add $dst, $src1, $src2, uxtw" %}
13410
13411 ins_encode %{
13412 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13413 as_Register($src2$$reg), ext::uxtw);
13414 %}
13415 ins_pipe(ialu_reg_reg);
13416 %}
13417
13418 // This pattern is automatically generated from aarch64_ad.m4
13419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13420 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13421 %{
13422 match(Set dst (SubI src1 (AndI src2 mask)));
13423 ins_cost(INSN_COST);
13424 format %{ "subw $dst, $src1, $src2, uxtb" %}
13425
13426 ins_encode %{
13427 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13428 as_Register($src2$$reg), ext::uxtb);
13429 %}
13430 ins_pipe(ialu_reg_reg);
13431 %}
13432
13433 // This pattern is automatically generated from aarch64_ad.m4
13434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13435 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13436 %{
13437 match(Set dst (SubI src1 (AndI src2 mask)));
13438 ins_cost(INSN_COST);
13439 format %{ "subw $dst, $src1, $src2, uxth" %}
13440
13441 ins_encode %{
13442 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13443 as_Register($src2$$reg), ext::uxth);
13444 %}
13445 ins_pipe(ialu_reg_reg);
13446 %}
13447
13448 // This pattern is automatically generated from aarch64_ad.m4
13449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13450 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13451 %{
13452 match(Set dst (SubL src1 (AndL src2 mask)));
13453 ins_cost(INSN_COST);
13454 format %{ "sub $dst, $src1, $src2, uxtb" %}
13455
13456 ins_encode %{
13457 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13458 as_Register($src2$$reg), ext::uxtb);
13459 %}
13460 ins_pipe(ialu_reg_reg);
13461 %}
13462
13463 // This pattern is automatically generated from aarch64_ad.m4
13464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13465 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13466 %{
13467 match(Set dst (SubL src1 (AndL src2 mask)));
13468 ins_cost(INSN_COST);
13469 format %{ "sub $dst, $src1, $src2, uxth" %}
13470
13471 ins_encode %{
13472 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13473 as_Register($src2$$reg), ext::uxth);
13474 %}
13475 ins_pipe(ialu_reg_reg);
13476 %}
13477
13478 // This pattern is automatically generated from aarch64_ad.m4
13479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13480 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13481 %{
13482 match(Set dst (SubL src1 (AndL src2 mask)));
13483 ins_cost(INSN_COST);
13484 format %{ "sub $dst, $src1, $src2, uxtw" %}
13485
13486 ins_encode %{
13487 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13488 as_Register($src2$$reg), ext::uxtw);
13489 %}
13490 ins_pipe(ialu_reg_reg);
13491 %}
13492
13493
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13497 %{
13498 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13499 ins_cost(1.9 * INSN_COST);
13500 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13501
13502 ins_encode %{
13503 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13504 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13505 %}
13506 ins_pipe(ialu_reg_reg_shift);
13507 %}
13508
13509 // This pattern is automatically generated from aarch64_ad.m4
13510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13511 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13512 %{
13513 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13514 ins_cost(1.9 * INSN_COST);
13515 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13516
13517 ins_encode %{
13518 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13519 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13520 %}
13521 ins_pipe(ialu_reg_reg_shift);
13522 %}
13523
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13527 %{
13528 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13529 ins_cost(1.9 * INSN_COST);
13530 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13531
13532 ins_encode %{
13533 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13534 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13535 %}
13536 ins_pipe(ialu_reg_reg_shift);
13537 %}
13538
13539 // This pattern is automatically generated from aarch64_ad.m4
13540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13541 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13542 %{
13543 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13544 ins_cost(1.9 * INSN_COST);
13545 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13546
13547 ins_encode %{
13548 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13549 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13550 %}
13551 ins_pipe(ialu_reg_reg_shift);
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13557 %{
13558 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13559 ins_cost(1.9 * INSN_COST);
13560 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13561
13562 ins_encode %{
13563 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13564 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13565 %}
13566 ins_pipe(ialu_reg_reg_shift);
13567 %}
13568
13569 // This pattern is automatically generated from aarch64_ad.m4
13570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13571 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13572 %{
13573 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13574 ins_cost(1.9 * INSN_COST);
13575 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13576
13577 ins_encode %{
13578 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13579 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13580 %}
13581 ins_pipe(ialu_reg_reg_shift);
13582 %}
13583
13584 // This pattern is automatically generated from aarch64_ad.m4
13585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13586 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13587 %{
13588 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13589 ins_cost(1.9 * INSN_COST);
13590 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13591
13592 ins_encode %{
13593 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13594 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13595 %}
13596 ins_pipe(ialu_reg_reg_shift);
13597 %}
13598
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13602 %{
13603 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13604 ins_cost(1.9 * INSN_COST);
13605 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13606
13607 ins_encode %{
13608 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13609 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13610 %}
13611 ins_pipe(ialu_reg_reg_shift);
13612 %}
13613
13614 // This pattern is automatically generated from aarch64_ad.m4
13615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13616 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13617 %{
13618 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13619 ins_cost(1.9 * INSN_COST);
13620 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13621
13622 ins_encode %{
13623 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13624 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13625 %}
13626 ins_pipe(ialu_reg_reg_shift);
13627 %}
13628
13629 // This pattern is automatically generated from aarch64_ad.m4
13630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13631 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13632 %{
13633 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13634 ins_cost(1.9 * INSN_COST);
13635 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13636
13637 ins_encode %{
13638 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13639 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13640 %}
13641 ins_pipe(ialu_reg_reg_shift);
13642 %}
13643
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13647 %{
13648 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13649 ins_cost(1.9 * INSN_COST);
13650 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13651
13652 ins_encode %{
13653 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13654 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13655 %}
13656 ins_pipe(ialu_reg_reg_shift);
13657 %}
13658
13659 // This pattern is automatically generated from aarch64_ad.m4
13660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13661 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13662 %{
13663 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13664 ins_cost(1.9 * INSN_COST);
13665 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13666
13667 ins_encode %{
13668 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13669 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13670 %}
13671 ins_pipe(ialu_reg_reg_shift);
13672 %}
13673
13674 // This pattern is automatically generated from aarch64_ad.m4
13675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13676 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13677 %{
13678 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13679 ins_cost(1.9 * INSN_COST);
13680 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13681
13682 ins_encode %{
13683 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13684 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13685 %}
13686 ins_pipe(ialu_reg_reg_shift);
13687 %}
13688
13689 // This pattern is automatically generated from aarch64_ad.m4
13690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13691 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13692 %{
13693 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13694 ins_cost(1.9 * INSN_COST);
13695 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13696
13697 ins_encode %{
13698 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13699 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13700 %}
13701 ins_pipe(ialu_reg_reg_shift);
13702 %}
13703
13704 // This pattern is automatically generated from aarch64_ad.m4
13705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13706 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13707 %{
13708 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13709 ins_cost(1.9 * INSN_COST);
13710 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13711
13712 ins_encode %{
13713 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13714 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13715 %}
13716 ins_pipe(ialu_reg_reg_shift);
13717 %}
13718
13719 // This pattern is automatically generated from aarch64_ad.m4
13720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13721 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13722 %{
13723 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13724 ins_cost(1.9 * INSN_COST);
13725 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13726
13727 ins_encode %{
13728 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13729 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13730 %}
13731 ins_pipe(ialu_reg_reg_shift);
13732 %}
13733
13734 // This pattern is automatically generated from aarch64_ad.m4
13735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13736 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13737 %{
13738 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13739 ins_cost(1.9 * INSN_COST);
13740 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13741
13742 ins_encode %{
13743 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13744 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13745 %}
13746 ins_pipe(ialu_reg_reg_shift);
13747 %}
13748
13749 // This pattern is automatically generated from aarch64_ad.m4
13750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13751 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13752 %{
13753 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13754 ins_cost(1.9 * INSN_COST);
13755 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13756
13757 ins_encode %{
13758 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13759 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13760 %}
13761 ins_pipe(ialu_reg_reg_shift);
13762 %}
13763
13764 // This pattern is automatically generated from aarch64_ad.m4
13765 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13766 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13767 %{
13768 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13769 ins_cost(1.9 * INSN_COST);
13770 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13771
13772 ins_encode %{
13773 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13774 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13775 %}
13776 ins_pipe(ialu_reg_reg_shift);
13777 %}
13778
13779 // This pattern is automatically generated from aarch64_ad.m4
13780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13781 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13782 %{
13783 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13784 ins_cost(1.9 * INSN_COST);
13785 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13786
13787 ins_encode %{
13788 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13789 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13790 %}
13791 ins_pipe(ialu_reg_reg_shift);
13792 %}
13793
13794 // This pattern is automatically generated from aarch64_ad.m4
13795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13796 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13797 %{
13798 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13799 ins_cost(1.9 * INSN_COST);
13800 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13801
13802 ins_encode %{
13803 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13804 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13805 %}
13806 ins_pipe(ialu_reg_reg_shift);
13807 %}
13808
13809 // This pattern is automatically generated from aarch64_ad.m4
13810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13811 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13812 %{
13813 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13814 ins_cost(1.9 * INSN_COST);
13815 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13816
13817 ins_encode %{
13818 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13819 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13820 %}
13821 ins_pipe(ialu_reg_reg_shift);
13822 %}
13823
13824 // This pattern is automatically generated from aarch64_ad.m4
13825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13826 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13827 %{
13828 effect(DEF dst, USE src1, USE src2, USE cr);
13829 ins_cost(INSN_COST * 2);
13830 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13831
13832 ins_encode %{
13833 __ cselw($dst$$Register,
13834 $src1$$Register,
13835 $src2$$Register,
13836 Assembler::LT);
13837 %}
13838 ins_pipe(icond_reg_reg);
13839 %}
13840
13841 // This pattern is automatically generated from aarch64_ad.m4
13842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13843 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13844 %{
13845 effect(DEF dst, USE src1, USE src2, USE cr);
13846 ins_cost(INSN_COST * 2);
13847 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13848
13849 ins_encode %{
13850 __ cselw($dst$$Register,
13851 $src1$$Register,
13852 $src2$$Register,
13853 Assembler::GT);
13854 %}
13855 ins_pipe(icond_reg_reg);
13856 %}
13857
13858 // This pattern is automatically generated from aarch64_ad.m4
13859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13860 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13861 %{
13862 effect(DEF dst, USE src1, USE cr);
13863 ins_cost(INSN_COST * 2);
13864 format %{ "cselw $dst, $src1, zr lt\t" %}
13865
13866 ins_encode %{
13867 __ cselw($dst$$Register,
13868 $src1$$Register,
13869 zr,
13870 Assembler::LT);
13871 %}
13872 ins_pipe(icond_reg);
13873 %}
13874
13875 // This pattern is automatically generated from aarch64_ad.m4
13876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13877 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13878 %{
13879 effect(DEF dst, USE src1, USE cr);
13880 ins_cost(INSN_COST * 2);
13881 format %{ "cselw $dst, $src1, zr gt\t" %}
13882
13883 ins_encode %{
13884 __ cselw($dst$$Register,
13885 $src1$$Register,
13886 zr,
13887 Assembler::GT);
13888 %}
13889 ins_pipe(icond_reg);
13890 %}
13891
13892 // This pattern is automatically generated from aarch64_ad.m4
13893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13894 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13895 %{
13896 effect(DEF dst, USE src1, USE cr);
13897 ins_cost(INSN_COST * 2);
13898 format %{ "csincw $dst, $src1, zr le\t" %}
13899
13900 ins_encode %{
13901 __ csincw($dst$$Register,
13902 $src1$$Register,
13903 zr,
13904 Assembler::LE);
13905 %}
13906 ins_pipe(icond_reg);
13907 %}
13908
13909 // This pattern is automatically generated from aarch64_ad.m4
13910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13911 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13912 %{
13913 effect(DEF dst, USE src1, USE cr);
13914 ins_cost(INSN_COST * 2);
13915 format %{ "csincw $dst, $src1, zr gt\t" %}
13916
13917 ins_encode %{
13918 __ csincw($dst$$Register,
13919 $src1$$Register,
13920 zr,
13921 Assembler::GT);
13922 %}
13923 ins_pipe(icond_reg);
13924 %}
13925
13926 // This pattern is automatically generated from aarch64_ad.m4
13927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13928 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13929 %{
13930 effect(DEF dst, USE src1, USE cr);
13931 ins_cost(INSN_COST * 2);
13932 format %{ "csinvw $dst, $src1, zr lt\t" %}
13933
13934 ins_encode %{
13935 __ csinvw($dst$$Register,
13936 $src1$$Register,
13937 zr,
13938 Assembler::LT);
13939 %}
13940 ins_pipe(icond_reg);
13941 %}
13942
13943 // This pattern is automatically generated from aarch64_ad.m4
13944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13945 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13946 %{
13947 effect(DEF dst, USE src1, USE cr);
13948 ins_cost(INSN_COST * 2);
13949 format %{ "csinvw $dst, $src1, zr ge\t" %}
13950
13951 ins_encode %{
13952 __ csinvw($dst$$Register,
13953 $src1$$Register,
13954 zr,
13955 Assembler::GE);
13956 %}
13957 ins_pipe(icond_reg);
13958 %}
13959
13960 // This pattern is automatically generated from aarch64_ad.m4
13961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13962 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13963 %{
13964 match(Set dst (MinI src imm));
13965 ins_cost(INSN_COST * 3);
13966 expand %{
13967 rFlagsReg cr;
13968 compI_reg_imm0(cr, src);
13969 cmovI_reg_imm0_lt(dst, src, cr);
13970 %}
13971 %}
13972
13973 // This pattern is automatically generated from aarch64_ad.m4
13974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13975 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13976 %{
13977 match(Set dst (MinI imm src));
13978 ins_cost(INSN_COST * 3);
13979 expand %{
13980 rFlagsReg cr;
13981 compI_reg_imm0(cr, src);
13982 cmovI_reg_imm0_lt(dst, src, cr);
13983 %}
13984 %}
13985
13986 // This pattern is automatically generated from aarch64_ad.m4
13987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13988 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13989 %{
13990 match(Set dst (MinI src imm));
13991 ins_cost(INSN_COST * 3);
13992 expand %{
13993 rFlagsReg cr;
13994 compI_reg_imm0(cr, src);
13995 cmovI_reg_imm1_le(dst, src, cr);
13996 %}
13997 %}
13998
13999 // This pattern is automatically generated from aarch64_ad.m4
14000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14001 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14002 %{
14003 match(Set dst (MinI imm src));
14004 ins_cost(INSN_COST * 3);
14005 expand %{
14006 rFlagsReg cr;
14007 compI_reg_imm0(cr, src);
14008 cmovI_reg_imm1_le(dst, src, cr);
14009 %}
14010 %}
14011
14012 // This pattern is automatically generated from aarch64_ad.m4
14013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14014 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14015 %{
14016 match(Set dst (MinI src imm));
14017 ins_cost(INSN_COST * 3);
14018 expand %{
14019 rFlagsReg cr;
14020 compI_reg_imm0(cr, src);
14021 cmovI_reg_immM1_lt(dst, src, cr);
14022 %}
14023 %}
14024
14025 // This pattern is automatically generated from aarch64_ad.m4
14026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14027 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14028 %{
14029 match(Set dst (MinI imm src));
14030 ins_cost(INSN_COST * 3);
14031 expand %{
14032 rFlagsReg cr;
14033 compI_reg_imm0(cr, src);
14034 cmovI_reg_immM1_lt(dst, src, cr);
14035 %}
14036 %}
14037
14038 // This pattern is automatically generated from aarch64_ad.m4
14039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14040 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14041 %{
14042 match(Set dst (MaxI src imm));
14043 ins_cost(INSN_COST * 3);
14044 expand %{
14045 rFlagsReg cr;
14046 compI_reg_imm0(cr, src);
14047 cmovI_reg_imm0_gt(dst, src, cr);
14048 %}
14049 %}
14050
14051 // This pattern is automatically generated from aarch64_ad.m4
14052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14053 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14054 %{
14055 match(Set dst (MaxI imm src));
14056 ins_cost(INSN_COST * 3);
14057 expand %{
14058 rFlagsReg cr;
14059 compI_reg_imm0(cr, src);
14060 cmovI_reg_imm0_gt(dst, src, cr);
14061 %}
14062 %}
14063
14064 // This pattern is automatically generated from aarch64_ad.m4
14065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14066 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14067 %{
14068 match(Set dst (MaxI src imm));
14069 ins_cost(INSN_COST * 3);
14070 expand %{
14071 rFlagsReg cr;
14072 compI_reg_imm0(cr, src);
14073 cmovI_reg_imm1_gt(dst, src, cr);
14074 %}
14075 %}
14076
14077 // This pattern is automatically generated from aarch64_ad.m4
14078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14079 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14080 %{
14081 match(Set dst (MaxI imm src));
14082 ins_cost(INSN_COST * 3);
14083 expand %{
14084 rFlagsReg cr;
14085 compI_reg_imm0(cr, src);
14086 cmovI_reg_imm1_gt(dst, src, cr);
14087 %}
14088 %}
14089
14090 // This pattern is automatically generated from aarch64_ad.m4
14091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14092 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14093 %{
14094 match(Set dst (MaxI src imm));
14095 ins_cost(INSN_COST * 3);
14096 expand %{
14097 rFlagsReg cr;
14098 compI_reg_imm0(cr, src);
14099 cmovI_reg_immM1_ge(dst, src, cr);
14100 %}
14101 %}
14102
14103 // This pattern is automatically generated from aarch64_ad.m4
14104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14105 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14106 %{
14107 match(Set dst (MaxI imm src));
14108 ins_cost(INSN_COST * 3);
14109 expand %{
14110 rFlagsReg cr;
14111 compI_reg_imm0(cr, src);
14112 cmovI_reg_immM1_ge(dst, src, cr);
14113 %}
14114 %}
14115
14116
14117
14118 // END This section of the file is automatically generated. Do not edit --------------
14119
14120
14121 // ============================================================================
14122 // Floating Point Arithmetic Instructions
14123
14124 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14125 match(Set dst (AddF src1 src2));
14126
14127 ins_cost(INSN_COST * 5);
14128 format %{ "fadds $dst, $src1, $src2" %}
14129
14130 ins_encode %{
14131 __ fadds(as_FloatRegister($dst$$reg),
14132 as_FloatRegister($src1$$reg),
14133 as_FloatRegister($src2$$reg));
14134 %}
14135
14136 ins_pipe(fp_dop_reg_reg_s);
14137 %}
14138
14139 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14140 match(Set dst (AddD src1 src2));
14141
14142 ins_cost(INSN_COST * 5);
14143 format %{ "faddd $dst, $src1, $src2" %}
14144
14145 ins_encode %{
14146 __ faddd(as_FloatRegister($dst$$reg),
14147 as_FloatRegister($src1$$reg),
14148 as_FloatRegister($src2$$reg));
14149 %}
14150
14151 ins_pipe(fp_dop_reg_reg_d);
14152 %}
14153
14154 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14155 match(Set dst (SubF src1 src2));
14156
14157 ins_cost(INSN_COST * 5);
14158 format %{ "fsubs $dst, $src1, $src2" %}
14159
14160 ins_encode %{
14161 __ fsubs(as_FloatRegister($dst$$reg),
14162 as_FloatRegister($src1$$reg),
14163 as_FloatRegister($src2$$reg));
14164 %}
14165
14166 ins_pipe(fp_dop_reg_reg_s);
14167 %}
14168
14169 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14170 match(Set dst (SubD src1 src2));
14171
14172 ins_cost(INSN_COST * 5);
14173 format %{ "fsubd $dst, $src1, $src2" %}
14174
14175 ins_encode %{
14176 __ fsubd(as_FloatRegister($dst$$reg),
14177 as_FloatRegister($src1$$reg),
14178 as_FloatRegister($src2$$reg));
14179 %}
14180
14181 ins_pipe(fp_dop_reg_reg_d);
14182 %}
14183
14184 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14185 match(Set dst (MulF src1 src2));
14186
14187 ins_cost(INSN_COST * 6);
14188 format %{ "fmuls $dst, $src1, $src2" %}
14189
14190 ins_encode %{
14191 __ fmuls(as_FloatRegister($dst$$reg),
14192 as_FloatRegister($src1$$reg),
14193 as_FloatRegister($src2$$reg));
14194 %}
14195
14196 ins_pipe(fp_dop_reg_reg_s);
14197 %}
14198
14199 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14200 match(Set dst (MulD src1 src2));
14201
14202 ins_cost(INSN_COST * 6);
14203 format %{ "fmuld $dst, $src1, $src2" %}
14204
14205 ins_encode %{
14206 __ fmuld(as_FloatRegister($dst$$reg),
14207 as_FloatRegister($src1$$reg),
14208 as_FloatRegister($src2$$reg));
14209 %}
14210
14211 ins_pipe(fp_dop_reg_reg_d);
14212 %}
14213
14214 // src1 * src2 + src3
14215 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14216 predicate(UseFMA);
14217 match(Set dst (FmaF src3 (Binary src1 src2)));
14218
14219 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14220
14221 ins_encode %{
14222 __ fmadds(as_FloatRegister($dst$$reg),
14223 as_FloatRegister($src1$$reg),
14224 as_FloatRegister($src2$$reg),
14225 as_FloatRegister($src3$$reg));
14226 %}
14227
14228 ins_pipe(pipe_class_default);
14229 %}
14230
14231 // src1 * src2 + src3
14232 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14233 predicate(UseFMA);
14234 match(Set dst (FmaD src3 (Binary src1 src2)));
14235
14236 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14237
14238 ins_encode %{
14239 __ fmaddd(as_FloatRegister($dst$$reg),
14240 as_FloatRegister($src1$$reg),
14241 as_FloatRegister($src2$$reg),
14242 as_FloatRegister($src3$$reg));
14243 %}
14244
14245 ins_pipe(pipe_class_default);
14246 %}
14247
14248 // -src1 * src2 + src3
14249 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14250 predicate(UseFMA);
14251 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14252 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14253
14254 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14255
14256 ins_encode %{
14257 __ fmsubs(as_FloatRegister($dst$$reg),
14258 as_FloatRegister($src1$$reg),
14259 as_FloatRegister($src2$$reg),
14260 as_FloatRegister($src3$$reg));
14261 %}
14262
14263 ins_pipe(pipe_class_default);
14264 %}
14265
14266 // -src1 * src2 + src3
14267 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14268 predicate(UseFMA);
14269 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14270 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14271
14272 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14273
14274 ins_encode %{
14275 __ fmsubd(as_FloatRegister($dst$$reg),
14276 as_FloatRegister($src1$$reg),
14277 as_FloatRegister($src2$$reg),
14278 as_FloatRegister($src3$$reg));
14279 %}
14280
14281 ins_pipe(pipe_class_default);
14282 %}
14283
14284 // -src1 * src2 - src3
14285 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14286 predicate(UseFMA);
14287 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14288 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14289
14290 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14291
14292 ins_encode %{
14293 __ fnmadds(as_FloatRegister($dst$$reg),
14294 as_FloatRegister($src1$$reg),
14295 as_FloatRegister($src2$$reg),
14296 as_FloatRegister($src3$$reg));
14297 %}
14298
14299 ins_pipe(pipe_class_default);
14300 %}
14301
14302 // -src1 * src2 - src3
14303 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14304 predicate(UseFMA);
14305 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14306 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14307
14308 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14309
14310 ins_encode %{
14311 __ fnmaddd(as_FloatRegister($dst$$reg),
14312 as_FloatRegister($src1$$reg),
14313 as_FloatRegister($src2$$reg),
14314 as_FloatRegister($src3$$reg));
14315 %}
14316
14317 ins_pipe(pipe_class_default);
14318 %}
14319
14320 // src1 * src2 - src3
14321 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14322 predicate(UseFMA);
14323 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14324
14325 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14326
14327 ins_encode %{
14328 __ fnmsubs(as_FloatRegister($dst$$reg),
14329 as_FloatRegister($src1$$reg),
14330 as_FloatRegister($src2$$reg),
14331 as_FloatRegister($src3$$reg));
14332 %}
14333
14334 ins_pipe(pipe_class_default);
14335 %}
14336
14337 // src1 * src2 - src3
14338 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14339 predicate(UseFMA);
14340 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14341
14342 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14343
14344 ins_encode %{
14345 // n.b. insn name should be fnmsubd
14346 __ fnmsub(as_FloatRegister($dst$$reg),
14347 as_FloatRegister($src1$$reg),
14348 as_FloatRegister($src2$$reg),
14349 as_FloatRegister($src3$$reg));
14350 %}
14351
14352 ins_pipe(pipe_class_default);
14353 %}
14354
14355
14356 // Math.max(FF)F
14357 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14358 match(Set dst (MaxF src1 src2));
14359
14360 format %{ "fmaxs $dst, $src1, $src2" %}
14361 ins_encode %{
14362 __ fmaxs(as_FloatRegister($dst$$reg),
14363 as_FloatRegister($src1$$reg),
14364 as_FloatRegister($src2$$reg));
14365 %}
14366
14367 ins_pipe(fp_dop_reg_reg_s);
14368 %}
14369
14370 // Math.min(FF)F
14371 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14372 match(Set dst (MinF src1 src2));
14373
14374 format %{ "fmins $dst, $src1, $src2" %}
14375 ins_encode %{
14376 __ fmins(as_FloatRegister($dst$$reg),
14377 as_FloatRegister($src1$$reg),
14378 as_FloatRegister($src2$$reg));
14379 %}
14380
14381 ins_pipe(fp_dop_reg_reg_s);
14382 %}
14383
14384 // Math.max(DD)D
14385 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14386 match(Set dst (MaxD src1 src2));
14387
14388 format %{ "fmaxd $dst, $src1, $src2" %}
14389 ins_encode %{
14390 __ fmaxd(as_FloatRegister($dst$$reg),
14391 as_FloatRegister($src1$$reg),
14392 as_FloatRegister($src2$$reg));
14393 %}
14394
14395 ins_pipe(fp_dop_reg_reg_d);
14396 %}
14397
14398 // Math.min(DD)D
14399 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14400 match(Set dst (MinD src1 src2));
14401
14402 format %{ "fmind $dst, $src1, $src2" %}
14403 ins_encode %{
14404 __ fmind(as_FloatRegister($dst$$reg),
14405 as_FloatRegister($src1$$reg),
14406 as_FloatRegister($src2$$reg));
14407 %}
14408
14409 ins_pipe(fp_dop_reg_reg_d);
14410 %}
14411
14412
14413 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14414 match(Set dst (DivF src1 src2));
14415
14416 ins_cost(INSN_COST * 18);
14417 format %{ "fdivs $dst, $src1, $src2" %}
14418
14419 ins_encode %{
14420 __ fdivs(as_FloatRegister($dst$$reg),
14421 as_FloatRegister($src1$$reg),
14422 as_FloatRegister($src2$$reg));
14423 %}
14424
14425 ins_pipe(fp_div_s);
14426 %}
14427
14428 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14429 match(Set dst (DivD src1 src2));
14430
14431 ins_cost(INSN_COST * 32);
14432 format %{ "fdivd $dst, $src1, $src2" %}
14433
14434 ins_encode %{
14435 __ fdivd(as_FloatRegister($dst$$reg),
14436 as_FloatRegister($src1$$reg),
14437 as_FloatRegister($src2$$reg));
14438 %}
14439
14440 ins_pipe(fp_div_d);
14441 %}
14442
14443 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14444 match(Set dst (NegF src));
14445
14446 ins_cost(INSN_COST * 3);
14447 format %{ "fneg $dst, $src" %}
14448
14449 ins_encode %{
14450 __ fnegs(as_FloatRegister($dst$$reg),
14451 as_FloatRegister($src$$reg));
14452 %}
14453
14454 ins_pipe(fp_uop_s);
14455 %}
14456
14457 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14458 match(Set dst (NegD src));
14459
14460 ins_cost(INSN_COST * 3);
14461 format %{ "fnegd $dst, $src" %}
14462
14463 ins_encode %{
14464 __ fnegd(as_FloatRegister($dst$$reg),
14465 as_FloatRegister($src$$reg));
14466 %}
14467
14468 ins_pipe(fp_uop_d);
14469 %}
14470
14471 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14472 %{
14473 match(Set dst (AbsI src));
14474
14475 effect(KILL cr);
14476 ins_cost(INSN_COST * 2);
14477 format %{ "cmpw $src, zr\n\t"
14478 "cnegw $dst, $src, Assembler::LT\t# int abs"
14479 %}
14480
14481 ins_encode %{
14482 __ cmpw(as_Register($src$$reg), zr);
14483 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14484 %}
14485 ins_pipe(pipe_class_default);
14486 %}
14487
14488 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14489 %{
14490 match(Set dst (AbsL src));
14491
14492 effect(KILL cr);
14493 ins_cost(INSN_COST * 2);
14494 format %{ "cmp $src, zr\n\t"
14495 "cneg $dst, $src, Assembler::LT\t# long abs"
14496 %}
14497
14498 ins_encode %{
14499 __ cmp(as_Register($src$$reg), zr);
14500 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14501 %}
14502 ins_pipe(pipe_class_default);
14503 %}
14504
14505 instruct absF_reg(vRegF dst, vRegF src) %{
14506 match(Set dst (AbsF src));
14507
14508 ins_cost(INSN_COST * 3);
14509 format %{ "fabss $dst, $src" %}
14510 ins_encode %{
14511 __ fabss(as_FloatRegister($dst$$reg),
14512 as_FloatRegister($src$$reg));
14513 %}
14514
14515 ins_pipe(fp_uop_s);
14516 %}
14517
14518 instruct absD_reg(vRegD dst, vRegD src) %{
14519 match(Set dst (AbsD src));
14520
14521 ins_cost(INSN_COST * 3);
14522 format %{ "fabsd $dst, $src" %}
14523 ins_encode %{
14524 __ fabsd(as_FloatRegister($dst$$reg),
14525 as_FloatRegister($src$$reg));
14526 %}
14527
14528 ins_pipe(fp_uop_d);
14529 %}
14530
14531 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14532 match(Set dst (AbsF (SubF src1 src2)));
14533
14534 ins_cost(INSN_COST * 3);
14535 format %{ "fabds $dst, $src1, $src2" %}
14536 ins_encode %{
14537 __ fabds(as_FloatRegister($dst$$reg),
14538 as_FloatRegister($src1$$reg),
14539 as_FloatRegister($src2$$reg));
14540 %}
14541
14542 ins_pipe(fp_uop_s);
14543 %}
14544
14545 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14546 match(Set dst (AbsD (SubD src1 src2)));
14547
14548 ins_cost(INSN_COST * 3);
14549 format %{ "fabdd $dst, $src1, $src2" %}
14550 ins_encode %{
14551 __ fabdd(as_FloatRegister($dst$$reg),
14552 as_FloatRegister($src1$$reg),
14553 as_FloatRegister($src2$$reg));
14554 %}
14555
14556 ins_pipe(fp_uop_d);
14557 %}
14558
14559 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14560 match(Set dst (SqrtD src));
14561
14562 ins_cost(INSN_COST * 50);
14563 format %{ "fsqrtd $dst, $src" %}
14564 ins_encode %{
14565 __ fsqrtd(as_FloatRegister($dst$$reg),
14566 as_FloatRegister($src$$reg));
14567 %}
14568
14569 ins_pipe(fp_div_s);
14570 %}
14571
14572 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14573 match(Set dst (SqrtF src));
14574
14575 ins_cost(INSN_COST * 50);
14576 format %{ "fsqrts $dst, $src" %}
14577 ins_encode %{
14578 __ fsqrts(as_FloatRegister($dst$$reg),
14579 as_FloatRegister($src$$reg));
14580 %}
14581
14582 ins_pipe(fp_div_d);
14583 %}
14584
14585 // Math.rint, floor, ceil
14586 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14587 match(Set dst (RoundDoubleMode src rmode));
14588 format %{ "frint $dst, $src, $rmode" %}
14589 ins_encode %{
14590 switch ($rmode$$constant) {
14591 case RoundDoubleModeNode::rmode_rint:
14592 __ frintnd(as_FloatRegister($dst$$reg),
14593 as_FloatRegister($src$$reg));
14594 break;
14595 case RoundDoubleModeNode::rmode_floor:
14596 __ frintmd(as_FloatRegister($dst$$reg),
14597 as_FloatRegister($src$$reg));
14598 break;
14599 case RoundDoubleModeNode::rmode_ceil:
14600 __ frintpd(as_FloatRegister($dst$$reg),
14601 as_FloatRegister($src$$reg));
14602 break;
14603 }
14604 %}
14605 ins_pipe(fp_uop_d);
14606 %}
14607
14608 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14609 match(Set dst (CopySignD src1 (Binary src2 zero)));
14610 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14611 format %{ "CopySignD $dst $src1 $src2" %}
14612 ins_encode %{
14613 FloatRegister dst = as_FloatRegister($dst$$reg),
14614 src1 = as_FloatRegister($src1$$reg),
14615 src2 = as_FloatRegister($src2$$reg),
14616 zero = as_FloatRegister($zero$$reg);
14617 __ fnegd(dst, zero);
14618 __ bsl(dst, __ T8B, src2, src1);
14619 %}
14620 ins_pipe(fp_uop_d);
14621 %}
14622
14623 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14624 match(Set dst (CopySignF src1 src2));
14625 effect(TEMP_DEF dst, USE src1, USE src2);
14626 format %{ "CopySignF $dst $src1 $src2" %}
14627 ins_encode %{
14628 FloatRegister dst = as_FloatRegister($dst$$reg),
14629 src1 = as_FloatRegister($src1$$reg),
14630 src2 = as_FloatRegister($src2$$reg);
14631 __ movi(dst, __ T2S, 0x80, 24);
14632 __ bsl(dst, __ T8B, src2, src1);
14633 %}
14634 ins_pipe(fp_uop_d);
14635 %}
14636
14637 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14638 match(Set dst (SignumD src (Binary zero one)));
14639 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14640 format %{ "signumD $dst, $src" %}
14641 ins_encode %{
14642 FloatRegister src = as_FloatRegister($src$$reg),
14643 dst = as_FloatRegister($dst$$reg),
14644 zero = as_FloatRegister($zero$$reg),
14645 one = as_FloatRegister($one$$reg);
14646 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14647 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14648 // Bit selection instruction gets bit from "one" for each enabled bit in
14649 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14650 // NaN the whole "src" will be copied because "dst" is zero. For all other
14651 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14652 // from "src", and all other bits are copied from 1.0.
14653 __ bsl(dst, __ T8B, one, src);
14654 %}
14655 ins_pipe(fp_uop_d);
14656 %}
14657
14658 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14659 match(Set dst (SignumF src (Binary zero one)));
14660 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14661 format %{ "signumF $dst, $src" %}
14662 ins_encode %{
14663 FloatRegister src = as_FloatRegister($src$$reg),
14664 dst = as_FloatRegister($dst$$reg),
14665 zero = as_FloatRegister($zero$$reg),
14666 one = as_FloatRegister($one$$reg);
14667 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14668 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14669 // Bit selection instruction gets bit from "one" for each enabled bit in
14670 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14671 // NaN the whole "src" will be copied because "dst" is zero. For all other
14672 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14673 // from "src", and all other bits are copied from 1.0.
14674 __ bsl(dst, __ T8B, one, src);
14675 %}
14676 ins_pipe(fp_uop_d);
14677 %}
14678
14679 instruct onspinwait() %{
14680 match(OnSpinWait);
14681 ins_cost(INSN_COST);
14682
14683 format %{ "onspinwait" %}
14684
14685 ins_encode %{
14686 __ spin_wait();
14687 %}
14688 ins_pipe(pipe_class_empty);
14689 %}
14690
14691 // ============================================================================
14692 // Logical Instructions
14693
14694 // Integer Logical Instructions
14695
14696 // And Instructions
14697
14698
14699 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14700 match(Set dst (AndI src1 src2));
14701
14702 format %{ "andw $dst, $src1, $src2\t# int" %}
14703
14704 ins_cost(INSN_COST);
14705 ins_encode %{
14706 __ andw(as_Register($dst$$reg),
14707 as_Register($src1$$reg),
14708 as_Register($src2$$reg));
14709 %}
14710
14711 ins_pipe(ialu_reg_reg);
14712 %}
14713
14714 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14715 match(Set dst (AndI src1 src2));
14716
14717 format %{ "andsw $dst, $src1, $src2\t# int" %}
14718
14719 ins_cost(INSN_COST);
14720 ins_encode %{
14721 __ andw(as_Register($dst$$reg),
14722 as_Register($src1$$reg),
14723 (uint64_t)($src2$$constant));
14724 %}
14725
14726 ins_pipe(ialu_reg_imm);
14727 %}
14728
14729 // Or Instructions
14730
14731 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14732 match(Set dst (OrI src1 src2));
14733
14734 format %{ "orrw $dst, $src1, $src2\t# int" %}
14735
14736 ins_cost(INSN_COST);
14737 ins_encode %{
14738 __ orrw(as_Register($dst$$reg),
14739 as_Register($src1$$reg),
14740 as_Register($src2$$reg));
14741 %}
14742
14743 ins_pipe(ialu_reg_reg);
14744 %}
14745
14746 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14747 match(Set dst (OrI src1 src2));
14748
14749 format %{ "orrw $dst, $src1, $src2\t# int" %}
14750
14751 ins_cost(INSN_COST);
14752 ins_encode %{
14753 __ orrw(as_Register($dst$$reg),
14754 as_Register($src1$$reg),
14755 (uint64_t)($src2$$constant));
14756 %}
14757
14758 ins_pipe(ialu_reg_imm);
14759 %}
14760
14761 // Xor Instructions
14762
14763 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14764 match(Set dst (XorI src1 src2));
14765
14766 format %{ "eorw $dst, $src1, $src2\t# int" %}
14767
14768 ins_cost(INSN_COST);
14769 ins_encode %{
14770 __ eorw(as_Register($dst$$reg),
14771 as_Register($src1$$reg),
14772 as_Register($src2$$reg));
14773 %}
14774
14775 ins_pipe(ialu_reg_reg);
14776 %}
14777
14778 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14779 match(Set dst (XorI src1 src2));
14780
14781 format %{ "eorw $dst, $src1, $src2\t# int" %}
14782
14783 ins_cost(INSN_COST);
14784 ins_encode %{
14785 __ eorw(as_Register($dst$$reg),
14786 as_Register($src1$$reg),
14787 (uint64_t)($src2$$constant));
14788 %}
14789
14790 ins_pipe(ialu_reg_imm);
14791 %}
14792
14793 // Long Logical Instructions
14794 // TODO
14795
14796 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14797 match(Set dst (AndL src1 src2));
14798
14799 format %{ "and $dst, $src1, $src2\t# int" %}
14800
14801 ins_cost(INSN_COST);
14802 ins_encode %{
14803 __ andr(as_Register($dst$$reg),
14804 as_Register($src1$$reg),
14805 as_Register($src2$$reg));
14806 %}
14807
14808 ins_pipe(ialu_reg_reg);
14809 %}
14810
14811 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14812 match(Set dst (AndL src1 src2));
14813
14814 format %{ "and $dst, $src1, $src2\t# int" %}
14815
14816 ins_cost(INSN_COST);
14817 ins_encode %{
14818 __ andr(as_Register($dst$$reg),
14819 as_Register($src1$$reg),
14820 (uint64_t)($src2$$constant));
14821 %}
14822
14823 ins_pipe(ialu_reg_imm);
14824 %}
14825
14826 // Or Instructions
14827
14828 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14829 match(Set dst (OrL src1 src2));
14830
14831 format %{ "orr $dst, $src1, $src2\t# int" %}
14832
14833 ins_cost(INSN_COST);
14834 ins_encode %{
14835 __ orr(as_Register($dst$$reg),
14836 as_Register($src1$$reg),
14837 as_Register($src2$$reg));
14838 %}
14839
14840 ins_pipe(ialu_reg_reg);
14841 %}
14842
14843 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14844 match(Set dst (OrL src1 src2));
14845
14846 format %{ "orr $dst, $src1, $src2\t# int" %}
14847
14848 ins_cost(INSN_COST);
14849 ins_encode %{
14850 __ orr(as_Register($dst$$reg),
14851 as_Register($src1$$reg),
14852 (uint64_t)($src2$$constant));
14853 %}
14854
14855 ins_pipe(ialu_reg_imm);
14856 %}
14857
14858 // Xor Instructions
14859
14860 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14861 match(Set dst (XorL src1 src2));
14862
14863 format %{ "eor $dst, $src1, $src2\t# int" %}
14864
14865 ins_cost(INSN_COST);
14866 ins_encode %{
14867 __ eor(as_Register($dst$$reg),
14868 as_Register($src1$$reg),
14869 as_Register($src2$$reg));
14870 %}
14871
14872 ins_pipe(ialu_reg_reg);
14873 %}
14874
14875 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14876 match(Set dst (XorL src1 src2));
14877
14878 ins_cost(INSN_COST);
14879 format %{ "eor $dst, $src1, $src2\t# int" %}
14880
14881 ins_encode %{
14882 __ eor(as_Register($dst$$reg),
14883 as_Register($src1$$reg),
14884 (uint64_t)($src2$$constant));
14885 %}
14886
14887 ins_pipe(ialu_reg_imm);
14888 %}
14889
14890 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14891 %{
14892 match(Set dst (ConvI2L src));
14893
14894 ins_cost(INSN_COST);
14895 format %{ "sxtw $dst, $src\t# i2l" %}
14896 ins_encode %{
14897 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14898 %}
14899 ins_pipe(ialu_reg_shift);
14900 %}
14901
14902 // this pattern occurs in bigmath arithmetic
14903 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14904 %{
14905 match(Set dst (AndL (ConvI2L src) mask));
14906
14907 ins_cost(INSN_COST);
14908 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14909 ins_encode %{
14910 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14911 %}
14912
14913 ins_pipe(ialu_reg_shift);
14914 %}
14915
14916 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14917 match(Set dst (ConvL2I src));
14918
14919 ins_cost(INSN_COST);
14920 format %{ "movw $dst, $src \t// l2i" %}
14921
14922 ins_encode %{
14923 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14924 %}
14925
14926 ins_pipe(ialu_reg);
14927 %}
14928
14929 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14930 %{
14931 match(Set dst (Conv2B src));
14932 effect(KILL cr);
14933
14934 format %{
14935 "cmpw $src, zr\n\t"
14936 "cset $dst, ne"
14937 %}
14938
14939 ins_encode %{
14940 __ cmpw(as_Register($src$$reg), zr);
14941 __ cset(as_Register($dst$$reg), Assembler::NE);
14942 %}
14943
14944 ins_pipe(ialu_reg);
14945 %}
14946
14947 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14948 %{
14949 match(Set dst (Conv2B src));
14950 effect(KILL cr);
14951
14952 format %{
14953 "cmp $src, zr\n\t"
14954 "cset $dst, ne"
14955 %}
14956
14957 ins_encode %{
14958 __ cmp(as_Register($src$$reg), zr);
14959 __ cset(as_Register($dst$$reg), Assembler::NE);
14960 %}
14961
14962 ins_pipe(ialu_reg);
14963 %}
14964
14965 instruct convD2F_reg(vRegF dst, vRegD src) %{
14966 match(Set dst (ConvD2F src));
14967
14968 ins_cost(INSN_COST * 5);
14969 format %{ "fcvtd $dst, $src \t// d2f" %}
14970
14971 ins_encode %{
14972 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14973 %}
14974
14975 ins_pipe(fp_d2f);
14976 %}
14977
14978 instruct convF2D_reg(vRegD dst, vRegF src) %{
14979 match(Set dst (ConvF2D src));
14980
14981 ins_cost(INSN_COST * 5);
14982 format %{ "fcvts $dst, $src \t// f2d" %}
14983
14984 ins_encode %{
14985 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14986 %}
14987
14988 ins_pipe(fp_f2d);
14989 %}
14990
14991 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14992 match(Set dst (ConvF2I src));
14993
14994 ins_cost(INSN_COST * 5);
14995 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14996
14997 ins_encode %{
14998 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14999 %}
15000
15001 ins_pipe(fp_f2i);
15002 %}
15003
15004 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15005 match(Set dst (ConvF2L src));
15006
15007 ins_cost(INSN_COST * 5);
15008 format %{ "fcvtzs $dst, $src \t// f2l" %}
15009
15010 ins_encode %{
15011 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15012 %}
15013
15014 ins_pipe(fp_f2l);
15015 %}
15016
15017 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15018 match(Set dst (ConvF2HF src));
15019 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15020 "smov $dst, $tmp\t# move result from $tmp to $dst"
15021 %}
15022 effect(TEMP tmp);
15023 ins_encode %{
15024 __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
15025 __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
15026 %}
15027 ins_pipe(pipe_slow);
15028 %}
15029
15030 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15031 match(Set dst (ConvHF2F src));
15032 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15033 "fcvt $dst, $tmp\t# convert half to single precision"
15034 %}
15035 effect(TEMP tmp);
15036 ins_encode %{
15037 __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
15038 __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
15039 %}
15040 ins_pipe(pipe_slow);
15041 %}
15042
15043 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15044 match(Set dst (ConvI2F src));
15045
15046 ins_cost(INSN_COST * 5);
15047 format %{ "scvtfws $dst, $src \t// i2f" %}
15048
15049 ins_encode %{
15050 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15051 %}
15052
15053 ins_pipe(fp_i2f);
15054 %}
15055
15056 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15057 match(Set dst (ConvL2F src));
15058
15059 ins_cost(INSN_COST * 5);
15060 format %{ "scvtfs $dst, $src \t// l2f" %}
15061
15062 ins_encode %{
15063 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15064 %}
15065
15066 ins_pipe(fp_l2f);
15067 %}
15068
15069 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15070 match(Set dst (ConvD2I src));
15071
15072 ins_cost(INSN_COST * 5);
15073 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15074
15075 ins_encode %{
15076 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15077 %}
15078
15079 ins_pipe(fp_d2i);
15080 %}
15081
15082 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15083 match(Set dst (ConvD2L src));
15084
15085 ins_cost(INSN_COST * 5);
15086 format %{ "fcvtzd $dst, $src \t// d2l" %}
15087
15088 ins_encode %{
15089 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15090 %}
15091
15092 ins_pipe(fp_d2l);
15093 %}
15094
15095 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15096 match(Set dst (ConvI2D src));
15097
15098 ins_cost(INSN_COST * 5);
15099 format %{ "scvtfwd $dst, $src \t// i2d" %}
15100
15101 ins_encode %{
15102 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15103 %}
15104
15105 ins_pipe(fp_i2d);
15106 %}
15107
15108 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15109 match(Set dst (ConvL2D src));
15110
15111 ins_cost(INSN_COST * 5);
15112 format %{ "scvtfd $dst, $src \t// l2d" %}
15113
15114 ins_encode %{
15115 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15116 %}
15117
15118 ins_pipe(fp_l2d);
15119 %}
15120
15121 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15122 %{
15123 match(Set dst (RoundD src));
15124 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15125 format %{ "java_round_double $dst,$src"%}
15126 ins_encode %{
15127 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15128 as_FloatRegister($ftmp$$reg));
15129 %}
15130 ins_pipe(pipe_slow);
15131 %}
15132
15133 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15134 %{
15135 match(Set dst (RoundF src));
15136 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15137 format %{ "java_round_float $dst,$src"%}
15138 ins_encode %{
15139 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15140 as_FloatRegister($ftmp$$reg));
15141 %}
15142 ins_pipe(pipe_slow);
15143 %}
15144
15145 // stack <-> reg and reg <-> reg shuffles with no conversion
15146
15147 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15148
15149 match(Set dst (MoveF2I src));
15150
15151 effect(DEF dst, USE src);
15152
15153 ins_cost(4 * INSN_COST);
15154
15155 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15156
15157 ins_encode %{
15158 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15159 %}
15160
15161 ins_pipe(iload_reg_reg);
15162
15163 %}
15164
15165 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15166
15167 match(Set dst (MoveI2F src));
15168
15169 effect(DEF dst, USE src);
15170
15171 ins_cost(4 * INSN_COST);
15172
15173 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15174
15175 ins_encode %{
15176 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15177 %}
15178
15179 ins_pipe(pipe_class_memory);
15180
15181 %}
15182
15183 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15184
15185 match(Set dst (MoveD2L src));
15186
15187 effect(DEF dst, USE src);
15188
15189 ins_cost(4 * INSN_COST);
15190
15191 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15192
15193 ins_encode %{
15194 __ ldr($dst$$Register, Address(sp, $src$$disp));
15195 %}
15196
15197 ins_pipe(iload_reg_reg);
15198
15199 %}
15200
15201 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15202
15203 match(Set dst (MoveL2D src));
15204
15205 effect(DEF dst, USE src);
15206
15207 ins_cost(4 * INSN_COST);
15208
15209 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15210
15211 ins_encode %{
15212 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15213 %}
15214
15215 ins_pipe(pipe_class_memory);
15216
15217 %}
15218
15219 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15220
15221 match(Set dst (MoveF2I src));
15222
15223 effect(DEF dst, USE src);
15224
15225 ins_cost(INSN_COST);
15226
15227 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15228
15229 ins_encode %{
15230 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15231 %}
15232
15233 ins_pipe(pipe_class_memory);
15234
15235 %}
15236
15237 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15238
15239 match(Set dst (MoveI2F src));
15240
15241 effect(DEF dst, USE src);
15242
15243 ins_cost(INSN_COST);
15244
15245 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15246
15247 ins_encode %{
15248 __ strw($src$$Register, Address(sp, $dst$$disp));
15249 %}
15250
15251 ins_pipe(istore_reg_reg);
15252
15253 %}
15254
15255 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15256
15257 match(Set dst (MoveD2L src));
15258
15259 effect(DEF dst, USE src);
15260
15261 ins_cost(INSN_COST);
15262
15263 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15264
15265 ins_encode %{
15266 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15267 %}
15268
15269 ins_pipe(pipe_class_memory);
15270
15271 %}
15272
15273 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15274
15275 match(Set dst (MoveL2D src));
15276
15277 effect(DEF dst, USE src);
15278
15279 ins_cost(INSN_COST);
15280
15281 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15282
15283 ins_encode %{
15284 __ str($src$$Register, Address(sp, $dst$$disp));
15285 %}
15286
15287 ins_pipe(istore_reg_reg);
15288
15289 %}
15290
15291 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15292
15293 match(Set dst (MoveF2I src));
15294
15295 effect(DEF dst, USE src);
15296
15297 ins_cost(INSN_COST);
15298
15299 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15300
15301 ins_encode %{
15302 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15303 %}
15304
15305 ins_pipe(fp_f2i);
15306
15307 %}
15308
15309 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15310
15311 match(Set dst (MoveI2F src));
15312
15313 effect(DEF dst, USE src);
15314
15315 ins_cost(INSN_COST);
15316
15317 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15318
15319 ins_encode %{
15320 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15321 %}
15322
15323 ins_pipe(fp_i2f);
15324
15325 %}
15326
15327 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15328
15329 match(Set dst (MoveD2L src));
15330
15331 effect(DEF dst, USE src);
15332
15333 ins_cost(INSN_COST);
15334
15335 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15336
15337 ins_encode %{
15338 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15339 %}
15340
15341 ins_pipe(fp_d2l);
15342
15343 %}
15344
15345 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15346
15347 match(Set dst (MoveL2D src));
15348
15349 effect(DEF dst, USE src);
15350
15351 ins_cost(INSN_COST);
15352
15353 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15354
15355 ins_encode %{
15356 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15357 %}
15358
15359 ins_pipe(fp_l2d);
15360
15361 %}
15362
15363 // ============================================================================
15364 // clearing of an array
15365
15366 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15367 %{
15368 match(Set dummy (ClearArray cnt base));
15369 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15370
15371 ins_cost(4 * INSN_COST);
15372 format %{ "ClearArray $cnt, $base" %}
15373
15374 ins_encode %{
15375 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15376 if (tpc == NULL) {
15377 ciEnv::current()->record_failure("CodeCache is full");
15378 return;
15379 }
15380 %}
15381
15382 ins_pipe(pipe_class_memory);
15383 %}
15384
15385 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15386 %{
15387 predicate((uint64_t)n->in(2)->get_long()
15388 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15389 match(Set dummy (ClearArray cnt base));
15390 effect(TEMP temp, USE_KILL base, KILL cr);
15391
15392 ins_cost(4 * INSN_COST);
15393 format %{ "ClearArray $cnt, $base" %}
15394
15395 ins_encode %{
15396 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15397 if (tpc == NULL) {
15398 ciEnv::current()->record_failure("CodeCache is full");
15399 return;
15400 }
15401 %}
15402
15403 ins_pipe(pipe_class_memory);
15404 %}
15405
15406 // ============================================================================
15407 // Overflow Math Instructions
15408
15409 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15410 %{
15411 match(Set cr (OverflowAddI op1 op2));
15412
15413 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15414 ins_cost(INSN_COST);
15415 ins_encode %{
15416 __ cmnw($op1$$Register, $op2$$Register);
15417 %}
15418
15419 ins_pipe(icmp_reg_reg);
15420 %}
15421
15422 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15423 %{
15424 match(Set cr (OverflowAddI op1 op2));
15425
15426 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15427 ins_cost(INSN_COST);
15428 ins_encode %{
15429 __ cmnw($op1$$Register, $op2$$constant);
15430 %}
15431
15432 ins_pipe(icmp_reg_imm);
15433 %}
15434
15435 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15436 %{
15437 match(Set cr (OverflowAddL op1 op2));
15438
15439 format %{ "cmn $op1, $op2\t# overflow check long" %}
15440 ins_cost(INSN_COST);
15441 ins_encode %{
15442 __ cmn($op1$$Register, $op2$$Register);
15443 %}
15444
15445 ins_pipe(icmp_reg_reg);
15446 %}
15447
15448 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15449 %{
15450 match(Set cr (OverflowAddL op1 op2));
15451
15452 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15453 ins_cost(INSN_COST);
15454 ins_encode %{
15455 __ adds(zr, $op1$$Register, $op2$$constant);
15456 %}
15457
15458 ins_pipe(icmp_reg_imm);
15459 %}
15460
15461 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15462 %{
15463 match(Set cr (OverflowSubI op1 op2));
15464
15465 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15466 ins_cost(INSN_COST);
15467 ins_encode %{
15468 __ cmpw($op1$$Register, $op2$$Register);
15469 %}
15470
15471 ins_pipe(icmp_reg_reg);
15472 %}
15473
15474 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15475 %{
15476 match(Set cr (OverflowSubI op1 op2));
15477
15478 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15479 ins_cost(INSN_COST);
15480 ins_encode %{
15481 __ cmpw($op1$$Register, $op2$$constant);
15482 %}
15483
15484 ins_pipe(icmp_reg_imm);
15485 %}
15486
15487 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15488 %{
15489 match(Set cr (OverflowSubL op1 op2));
15490
15491 format %{ "cmp $op1, $op2\t# overflow check long" %}
15492 ins_cost(INSN_COST);
15493 ins_encode %{
15494 __ cmp($op1$$Register, $op2$$Register);
15495 %}
15496
15497 ins_pipe(icmp_reg_reg);
15498 %}
15499
15500 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15501 %{
15502 match(Set cr (OverflowSubL op1 op2));
15503
15504 format %{ "cmp $op1, $op2\t# overflow check long" %}
15505 ins_cost(INSN_COST);
15506 ins_encode %{
15507 __ subs(zr, $op1$$Register, $op2$$constant);
15508 %}
15509
15510 ins_pipe(icmp_reg_imm);
15511 %}
15512
15513 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15514 %{
15515 match(Set cr (OverflowSubI zero op1));
15516
15517 format %{ "cmpw zr, $op1\t# overflow check int" %}
15518 ins_cost(INSN_COST);
15519 ins_encode %{
15520 __ cmpw(zr, $op1$$Register);
15521 %}
15522
15523 ins_pipe(icmp_reg_imm);
15524 %}
15525
15526 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15527 %{
15528 match(Set cr (OverflowSubL zero op1));
15529
15530 format %{ "cmp zr, $op1\t# overflow check long" %}
15531 ins_cost(INSN_COST);
15532 ins_encode %{
15533 __ cmp(zr, $op1$$Register);
15534 %}
15535
15536 ins_pipe(icmp_reg_imm);
15537 %}
15538
15539 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15540 %{
15541 match(Set cr (OverflowMulI op1 op2));
15542
15543 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15544 "cmp rscratch1, rscratch1, sxtw\n\t"
15545 "movw rscratch1, #0x80000000\n\t"
15546 "cselw rscratch1, rscratch1, zr, NE\n\t"
15547 "cmpw rscratch1, #1" %}
15548 ins_cost(5 * INSN_COST);
15549 ins_encode %{
15550 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15551 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15552 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15553 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15554 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15555 %}
15556
15557 ins_pipe(pipe_slow);
15558 %}
15559
15560 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15561 %{
15562 match(If cmp (OverflowMulI op1 op2));
15563 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15564 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15565 effect(USE labl, KILL cr);
15566
15567 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15568 "cmp rscratch1, rscratch1, sxtw\n\t"
15569 "b$cmp $labl" %}
15570 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15571 ins_encode %{
15572 Label* L = $labl$$label;
15573 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15574 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15575 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15576 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15577 %}
15578
15579 ins_pipe(pipe_serial);
15580 %}
15581
15582 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15583 %{
15584 match(Set cr (OverflowMulL op1 op2));
15585
15586 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15587 "smulh rscratch2, $op1, $op2\n\t"
15588 "cmp rscratch2, rscratch1, ASR #63\n\t"
15589 "movw rscratch1, #0x80000000\n\t"
15590 "cselw rscratch1, rscratch1, zr, NE\n\t"
15591 "cmpw rscratch1, #1" %}
15592 ins_cost(6 * INSN_COST);
15593 ins_encode %{
15594 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15595 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15596 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15597 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15598 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15599 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15600 %}
15601
15602 ins_pipe(pipe_slow);
15603 %}
15604
15605 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15606 %{
15607 match(If cmp (OverflowMulL op1 op2));
15608 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15609 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15610 effect(USE labl, KILL cr);
15611
15612 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15613 "smulh rscratch2, $op1, $op2\n\t"
15614 "cmp rscratch2, rscratch1, ASR #63\n\t"
15615 "b$cmp $labl" %}
15616 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15617 ins_encode %{
15618 Label* L = $labl$$label;
15619 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15620 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15621 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15622 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15623 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15624 %}
15625
15626 ins_pipe(pipe_serial);
15627 %}
15628
15629 // ============================================================================
15630 // Compare Instructions
15631
15632 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15633 %{
15634 match(Set cr (CmpI op1 op2));
15635
15636 effect(DEF cr, USE op1, USE op2);
15637
15638 ins_cost(INSN_COST);
15639 format %{ "cmpw $op1, $op2" %}
15640
15641 ins_encode(aarch64_enc_cmpw(op1, op2));
15642
15643 ins_pipe(icmp_reg_reg);
15644 %}
15645
15646 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15647 %{
15648 match(Set cr (CmpI op1 zero));
15649
15650 effect(DEF cr, USE op1);
15651
15652 ins_cost(INSN_COST);
15653 format %{ "cmpw $op1, 0" %}
15654
15655 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15656
15657 ins_pipe(icmp_reg_imm);
15658 %}
15659
15660 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15661 %{
15662 match(Set cr (CmpI op1 op2));
15663
15664 effect(DEF cr, USE op1);
15665
15666 ins_cost(INSN_COST);
15667 format %{ "cmpw $op1, $op2" %}
15668
15669 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15670
15671 ins_pipe(icmp_reg_imm);
15672 %}
15673
15674 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15675 %{
15676 match(Set cr (CmpI op1 op2));
15677
15678 effect(DEF cr, USE op1);
15679
15680 ins_cost(INSN_COST * 2);
15681 format %{ "cmpw $op1, $op2" %}
15682
15683 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15684
15685 ins_pipe(icmp_reg_imm);
15686 %}
15687
15688 // Unsigned compare Instructions; really, same as signed compare
15689 // except it should only be used to feed an If or a CMovI which takes a
15690 // cmpOpU.
15691
15692 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15693 %{
15694 match(Set cr (CmpU op1 op2));
15695
15696 effect(DEF cr, USE op1, USE op2);
15697
15698 ins_cost(INSN_COST);
15699 format %{ "cmpw $op1, $op2\t# unsigned" %}
15700
15701 ins_encode(aarch64_enc_cmpw(op1, op2));
15702
15703 ins_pipe(icmp_reg_reg);
15704 %}
15705
15706 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15707 %{
15708 match(Set cr (CmpU op1 zero));
15709
15710 effect(DEF cr, USE op1);
15711
15712 ins_cost(INSN_COST);
15713 format %{ "cmpw $op1, #0\t# unsigned" %}
15714
15715 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15716
15717 ins_pipe(icmp_reg_imm);
15718 %}
15719
15720 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15721 %{
15722 match(Set cr (CmpU op1 op2));
15723
15724 effect(DEF cr, USE op1);
15725
15726 ins_cost(INSN_COST);
15727 format %{ "cmpw $op1, $op2\t# unsigned" %}
15728
15729 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15730
15731 ins_pipe(icmp_reg_imm);
15732 %}
15733
15734 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15735 %{
15736 match(Set cr (CmpU op1 op2));
15737
15738 effect(DEF cr, USE op1);
15739
15740 ins_cost(INSN_COST * 2);
15741 format %{ "cmpw $op1, $op2\t# unsigned" %}
15742
15743 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15744
15745 ins_pipe(icmp_reg_imm);
15746 %}
15747
15748 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15749 %{
15750 match(Set cr (CmpL op1 op2));
15751
15752 effect(DEF cr, USE op1, USE op2);
15753
15754 ins_cost(INSN_COST);
15755 format %{ "cmp $op1, $op2" %}
15756
15757 ins_encode(aarch64_enc_cmp(op1, op2));
15758
15759 ins_pipe(icmp_reg_reg);
15760 %}
15761
15762 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15763 %{
15764 match(Set cr (CmpL op1 zero));
15765
15766 effect(DEF cr, USE op1);
15767
15768 ins_cost(INSN_COST);
15769 format %{ "tst $op1" %}
15770
15771 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15772
15773 ins_pipe(icmp_reg_imm);
15774 %}
15775
15776 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15777 %{
15778 match(Set cr (CmpL op1 op2));
15779
15780 effect(DEF cr, USE op1);
15781
15782 ins_cost(INSN_COST);
15783 format %{ "cmp $op1, $op2" %}
15784
15785 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15786
15787 ins_pipe(icmp_reg_imm);
15788 %}
15789
15790 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15791 %{
15792 match(Set cr (CmpL op1 op2));
15793
15794 effect(DEF cr, USE op1);
15795
15796 ins_cost(INSN_COST * 2);
15797 format %{ "cmp $op1, $op2" %}
15798
15799 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15800
15801 ins_pipe(icmp_reg_imm);
15802 %}
15803
15804 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15805 %{
15806 match(Set cr (CmpUL op1 op2));
15807
15808 effect(DEF cr, USE op1, USE op2);
15809
15810 ins_cost(INSN_COST);
15811 format %{ "cmp $op1, $op2" %}
15812
15813 ins_encode(aarch64_enc_cmp(op1, op2));
15814
15815 ins_pipe(icmp_reg_reg);
15816 %}
15817
15818 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15819 %{
15820 match(Set cr (CmpUL op1 zero));
15821
15822 effect(DEF cr, USE op1);
15823
15824 ins_cost(INSN_COST);
15825 format %{ "tst $op1" %}
15826
15827 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15828
15829 ins_pipe(icmp_reg_imm);
15830 %}
15831
15832 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15833 %{
15834 match(Set cr (CmpUL op1 op2));
15835
15836 effect(DEF cr, USE op1);
15837
15838 ins_cost(INSN_COST);
15839 format %{ "cmp $op1, $op2" %}
15840
15841 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15842
15843 ins_pipe(icmp_reg_imm);
15844 %}
15845
15846 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15847 %{
15848 match(Set cr (CmpUL op1 op2));
15849
15850 effect(DEF cr, USE op1);
15851
15852 ins_cost(INSN_COST * 2);
15853 format %{ "cmp $op1, $op2" %}
15854
15855 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15856
15857 ins_pipe(icmp_reg_imm);
15858 %}
15859
15860 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15861 %{
15862 match(Set cr (CmpP op1 op2));
15863
15864 effect(DEF cr, USE op1, USE op2);
15865
15866 ins_cost(INSN_COST);
15867 format %{ "cmp $op1, $op2\t // ptr" %}
15868
15869 ins_encode(aarch64_enc_cmpp(op1, op2));
15870
15871 ins_pipe(icmp_reg_reg);
15872 %}
15873
15874 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15875 %{
15876 match(Set cr (CmpN op1 op2));
15877
15878 effect(DEF cr, USE op1, USE op2);
15879
15880 ins_cost(INSN_COST);
15881 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15882
15883 ins_encode(aarch64_enc_cmpn(op1, op2));
15884
15885 ins_pipe(icmp_reg_reg);
15886 %}
15887
15888 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15889 %{
15890 match(Set cr (CmpP op1 zero));
15891
15892 effect(DEF cr, USE op1, USE zero);
15893
15894 ins_cost(INSN_COST);
15895 format %{ "cmp $op1, 0\t // ptr" %}
15896
15897 ins_encode(aarch64_enc_testp(op1));
15898
15899 ins_pipe(icmp_reg_imm);
15900 %}
15901
15902 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15903 %{
15904 match(Set cr (CmpN op1 zero));
15905
15906 effect(DEF cr, USE op1, USE zero);
15907
15908 ins_cost(INSN_COST);
15909 format %{ "cmp $op1, 0\t // compressed ptr" %}
15910
15911 ins_encode(aarch64_enc_testn(op1));
15912
15913 ins_pipe(icmp_reg_imm);
15914 %}
15915
15916 // FP comparisons
15917 //
15918 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15919 // using normal cmpOp. See declaration of rFlagsReg for details.
15920
15921 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15922 %{
15923 match(Set cr (CmpF src1 src2));
15924
15925 ins_cost(3 * INSN_COST);
15926 format %{ "fcmps $src1, $src2" %}
15927
15928 ins_encode %{
15929 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15930 %}
15931
15932 ins_pipe(pipe_class_compare);
15933 %}
15934
15935 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15936 %{
15937 match(Set cr (CmpF src1 src2));
15938
15939 ins_cost(3 * INSN_COST);
15940 format %{ "fcmps $src1, 0.0" %}
15941
15942 ins_encode %{
15943 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15944 %}
15945
15946 ins_pipe(pipe_class_compare);
15947 %}
15948 // FROM HERE
15949
15950 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15951 %{
15952 match(Set cr (CmpD src1 src2));
15953
15954 ins_cost(3 * INSN_COST);
15955 format %{ "fcmpd $src1, $src2" %}
15956
15957 ins_encode %{
15958 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15959 %}
15960
15961 ins_pipe(pipe_class_compare);
15962 %}
15963
15964 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15965 %{
15966 match(Set cr (CmpD src1 src2));
15967
15968 ins_cost(3 * INSN_COST);
15969 format %{ "fcmpd $src1, 0.0" %}
15970
15971 ins_encode %{
15972 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15973 %}
15974
15975 ins_pipe(pipe_class_compare);
15976 %}
15977
15978 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15979 %{
15980 match(Set dst (CmpF3 src1 src2));
15981 effect(KILL cr);
15982
15983 ins_cost(5 * INSN_COST);
15984 format %{ "fcmps $src1, $src2\n\t"
15985 "csinvw($dst, zr, zr, eq\n\t"
15986 "csnegw($dst, $dst, $dst, lt)"
15987 %}
15988
15989 ins_encode %{
15990 Label done;
15991 FloatRegister s1 = as_FloatRegister($src1$$reg);
15992 FloatRegister s2 = as_FloatRegister($src2$$reg);
15993 Register d = as_Register($dst$$reg);
15994 __ fcmps(s1, s2);
15995 // installs 0 if EQ else -1
15996 __ csinvw(d, zr, zr, Assembler::EQ);
15997 // keeps -1 if less or unordered else installs 1
15998 __ csnegw(d, d, d, Assembler::LT);
15999 __ bind(done);
16000 %}
16001
16002 ins_pipe(pipe_class_default);
16003
16004 %}
16005
16006 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16007 %{
16008 match(Set dst (CmpD3 src1 src2));
16009 effect(KILL cr);
16010
16011 ins_cost(5 * INSN_COST);
16012 format %{ "fcmpd $src1, $src2\n\t"
16013 "csinvw($dst, zr, zr, eq\n\t"
16014 "csnegw($dst, $dst, $dst, lt)"
16015 %}
16016
16017 ins_encode %{
16018 Label done;
16019 FloatRegister s1 = as_FloatRegister($src1$$reg);
16020 FloatRegister s2 = as_FloatRegister($src2$$reg);
16021 Register d = as_Register($dst$$reg);
16022 __ fcmpd(s1, s2);
16023 // installs 0 if EQ else -1
16024 __ csinvw(d, zr, zr, Assembler::EQ);
16025 // keeps -1 if less or unordered else installs 1
16026 __ csnegw(d, d, d, Assembler::LT);
16027 __ bind(done);
16028 %}
16029 ins_pipe(pipe_class_default);
16030
16031 %}
16032
16033 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16034 %{
16035 match(Set dst (CmpF3 src1 zero));
16036 effect(KILL cr);
16037
16038 ins_cost(5 * INSN_COST);
16039 format %{ "fcmps $src1, 0.0\n\t"
16040 "csinvw($dst, zr, zr, eq\n\t"
16041 "csnegw($dst, $dst, $dst, lt)"
16042 %}
16043
16044 ins_encode %{
16045 Label done;
16046 FloatRegister s1 = as_FloatRegister($src1$$reg);
16047 Register d = as_Register($dst$$reg);
16048 __ fcmps(s1, 0.0);
16049 // installs 0 if EQ else -1
16050 __ csinvw(d, zr, zr, Assembler::EQ);
16051 // keeps -1 if less or unordered else installs 1
16052 __ csnegw(d, d, d, Assembler::LT);
16053 __ bind(done);
16054 %}
16055
16056 ins_pipe(pipe_class_default);
16057
16058 %}
16059
16060 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16061 %{
16062 match(Set dst (CmpD3 src1 zero));
16063 effect(KILL cr);
16064
16065 ins_cost(5 * INSN_COST);
16066 format %{ "fcmpd $src1, 0.0\n\t"
16067 "csinvw($dst, zr, zr, eq\n\t"
16068 "csnegw($dst, $dst, $dst, lt)"
16069 %}
16070
16071 ins_encode %{
16072 Label done;
16073 FloatRegister s1 = as_FloatRegister($src1$$reg);
16074 Register d = as_Register($dst$$reg);
16075 __ fcmpd(s1, 0.0);
16076 // installs 0 if EQ else -1
16077 __ csinvw(d, zr, zr, Assembler::EQ);
16078 // keeps -1 if less or unordered else installs 1
16079 __ csnegw(d, d, d, Assembler::LT);
16080 __ bind(done);
16081 %}
16082 ins_pipe(pipe_class_default);
16083
16084 %}
16085
16086 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16087 %{
16088 match(Set dst (CmpLTMask p q));
16089 effect(KILL cr);
16090
16091 ins_cost(3 * INSN_COST);
16092
16093 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16094 "csetw $dst, lt\n\t"
16095 "subw $dst, zr, $dst"
16096 %}
16097
16098 ins_encode %{
16099 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16100 __ csetw(as_Register($dst$$reg), Assembler::LT);
16101 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16102 %}
16103
16104 ins_pipe(ialu_reg_reg);
16105 %}
16106
16107 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16108 %{
16109 match(Set dst (CmpLTMask src zero));
16110 effect(KILL cr);
16111
16112 ins_cost(INSN_COST);
16113
16114 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16115
16116 ins_encode %{
16117 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16118 %}
16119
16120 ins_pipe(ialu_reg_shift);
16121 %}
16122
16123 // ============================================================================
16124 // Max and Min
16125
16126 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16127
16128 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16129 %{
16130 effect(DEF cr, USE src);
16131 ins_cost(INSN_COST);
16132 format %{ "cmpw $src, 0" %}
16133
16134 ins_encode %{
16135 __ cmpw($src$$Register, 0);
16136 %}
16137 ins_pipe(icmp_reg_imm);
16138 %}
16139
16140 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16141 %{
16142 match(Set dst (MinI src1 src2));
16143 ins_cost(INSN_COST * 3);
16144
16145 expand %{
16146 rFlagsReg cr;
16147 compI_reg_reg(cr, src1, src2);
16148 cmovI_reg_reg_lt(dst, src1, src2, cr);
16149 %}
16150 %}
16151
16152 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16153 %{
16154 match(Set dst (MaxI src1 src2));
16155 ins_cost(INSN_COST * 3);
16156
16157 expand %{
16158 rFlagsReg cr;
16159 compI_reg_reg(cr, src1, src2);
16160 cmovI_reg_reg_gt(dst, src1, src2, cr);
16161 %}
16162 %}
16163
16164
16165 // ============================================================================
16166 // Branch Instructions
16167
16168 // Direct Branch.
16169 instruct branch(label lbl)
16170 %{
16171 match(Goto);
16172
16173 effect(USE lbl);
16174
16175 ins_cost(BRANCH_COST);
16176 format %{ "b $lbl" %}
16177
16178 ins_encode(aarch64_enc_b(lbl));
16179
16180 ins_pipe(pipe_branch);
16181 %}
16182
16183 // Conditional Near Branch
16184 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16185 %{
16186 // Same match rule as `branchConFar'.
16187 match(If cmp cr);
16188
16189 effect(USE lbl);
16190
16191 ins_cost(BRANCH_COST);
16192 // If set to 1 this indicates that the current instruction is a
16193 // short variant of a long branch. This avoids using this
16194 // instruction in first-pass matching. It will then only be used in
16195 // the `Shorten_branches' pass.
16196 // ins_short_branch(1);
16197 format %{ "b$cmp $lbl" %}
16198
16199 ins_encode(aarch64_enc_br_con(cmp, lbl));
16200
16201 ins_pipe(pipe_branch_cond);
16202 %}
16203
16204 // Conditional Near Branch Unsigned
16205 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16206 %{
16207 // Same match rule as `branchConFar'.
16208 match(If cmp cr);
16209
16210 effect(USE lbl);
16211
16212 ins_cost(BRANCH_COST);
16213 // If set to 1 this indicates that the current instruction is a
16214 // short variant of a long branch. This avoids using this
16215 // instruction in first-pass matching. It will then only be used in
16216 // the `Shorten_branches' pass.
16217 // ins_short_branch(1);
16218 format %{ "b$cmp $lbl\t# unsigned" %}
16219
16220 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16221
16222 ins_pipe(pipe_branch_cond);
16223 %}
16224
16225 // Make use of CBZ and CBNZ. These instructions, as well as being
16226 // shorter than (cmp; branch), have the additional benefit of not
16227 // killing the flags.
16228
16229 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16230 match(If cmp (CmpI op1 op2));
16231 effect(USE labl);
16232
16233 ins_cost(BRANCH_COST);
16234 format %{ "cbw$cmp $op1, $labl" %}
16235 ins_encode %{
16236 Label* L = $labl$$label;
16237 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16238 if (cond == Assembler::EQ)
16239 __ cbzw($op1$$Register, *L);
16240 else
16241 __ cbnzw($op1$$Register, *L);
16242 %}
16243 ins_pipe(pipe_cmp_branch);
16244 %}
16245
16246 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16247 match(If cmp (CmpL op1 op2));
16248 effect(USE labl);
16249
16250 ins_cost(BRANCH_COST);
16251 format %{ "cb$cmp $op1, $labl" %}
16252 ins_encode %{
16253 Label* L = $labl$$label;
16254 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16255 if (cond == Assembler::EQ)
16256 __ cbz($op1$$Register, *L);
16257 else
16258 __ cbnz($op1$$Register, *L);
16259 %}
16260 ins_pipe(pipe_cmp_branch);
16261 %}
16262
16263 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16264 match(If cmp (CmpP op1 op2));
16265 effect(USE labl);
16266
16267 ins_cost(BRANCH_COST);
16268 format %{ "cb$cmp $op1, $labl" %}
16269 ins_encode %{
16270 Label* L = $labl$$label;
16271 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16272 if (cond == Assembler::EQ)
16273 __ cbz($op1$$Register, *L);
16274 else
16275 __ cbnz($op1$$Register, *L);
16276 %}
16277 ins_pipe(pipe_cmp_branch);
16278 %}
16279
16280 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16281 match(If cmp (CmpN op1 op2));
16282 effect(USE labl);
16283
16284 ins_cost(BRANCH_COST);
16285 format %{ "cbw$cmp $op1, $labl" %}
16286 ins_encode %{
16287 Label* L = $labl$$label;
16288 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16289 if (cond == Assembler::EQ)
16290 __ cbzw($op1$$Register, *L);
16291 else
16292 __ cbnzw($op1$$Register, *L);
16293 %}
16294 ins_pipe(pipe_cmp_branch);
16295 %}
16296
16297 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16298 match(If cmp (CmpP (DecodeN oop) zero));
16299 effect(USE labl);
16300
16301 ins_cost(BRANCH_COST);
16302 format %{ "cb$cmp $oop, $labl" %}
16303 ins_encode %{
16304 Label* L = $labl$$label;
16305 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16306 if (cond == Assembler::EQ)
16307 __ cbzw($oop$$Register, *L);
16308 else
16309 __ cbnzw($oop$$Register, *L);
16310 %}
16311 ins_pipe(pipe_cmp_branch);
16312 %}
16313
16314 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16315 match(If cmp (CmpU op1 op2));
16316 effect(USE labl);
16317
16318 ins_cost(BRANCH_COST);
16319 format %{ "cbw$cmp $op1, $labl" %}
16320 ins_encode %{
16321 Label* L = $labl$$label;
16322 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16323 if (cond == Assembler::EQ || cond == Assembler::LS)
16324 __ cbzw($op1$$Register, *L);
16325 else
16326 __ cbnzw($op1$$Register, *L);
16327 %}
16328 ins_pipe(pipe_cmp_branch);
16329 %}
16330
16331 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16332 match(If cmp (CmpUL op1 op2));
16333 effect(USE labl);
16334
16335 ins_cost(BRANCH_COST);
16336 format %{ "cb$cmp $op1, $labl" %}
16337 ins_encode %{
16338 Label* L = $labl$$label;
16339 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16340 if (cond == Assembler::EQ || cond == Assembler::LS)
16341 __ cbz($op1$$Register, *L);
16342 else
16343 __ cbnz($op1$$Register, *L);
16344 %}
16345 ins_pipe(pipe_cmp_branch);
16346 %}
16347
16348 // Test bit and Branch
16349
16350 // Patterns for short (< 32KiB) variants
16351 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16352 match(If cmp (CmpL op1 op2));
16353 effect(USE labl);
16354
16355 ins_cost(BRANCH_COST);
16356 format %{ "cb$cmp $op1, $labl # long" %}
16357 ins_encode %{
16358 Label* L = $labl$$label;
16359 Assembler::Condition cond =
16360 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16361 __ tbr(cond, $op1$$Register, 63, *L);
16362 %}
16363 ins_pipe(pipe_cmp_branch);
16364 ins_short_branch(1);
16365 %}
16366
16367 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16368 match(If cmp (CmpI op1 op2));
16369 effect(USE labl);
16370
16371 ins_cost(BRANCH_COST);
16372 format %{ "cb$cmp $op1, $labl # int" %}
16373 ins_encode %{
16374 Label* L = $labl$$label;
16375 Assembler::Condition cond =
16376 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16377 __ tbr(cond, $op1$$Register, 31, *L);
16378 %}
16379 ins_pipe(pipe_cmp_branch);
16380 ins_short_branch(1);
16381 %}
16382
16383 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16384 match(If cmp (CmpL (AndL op1 op2) op3));
16385 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16386 effect(USE labl);
16387
16388 ins_cost(BRANCH_COST);
16389 format %{ "tb$cmp $op1, $op2, $labl" %}
16390 ins_encode %{
16391 Label* L = $labl$$label;
16392 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16393 int bit = exact_log2_long($op2$$constant);
16394 __ tbr(cond, $op1$$Register, bit, *L);
16395 %}
16396 ins_pipe(pipe_cmp_branch);
16397 ins_short_branch(1);
16398 %}
16399
16400 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16401 match(If cmp (CmpI (AndI op1 op2) op3));
16402 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16403 effect(USE labl);
16404
16405 ins_cost(BRANCH_COST);
16406 format %{ "tb$cmp $op1, $op2, $labl" %}
16407 ins_encode %{
16408 Label* L = $labl$$label;
16409 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16410 int bit = exact_log2((juint)$op2$$constant);
16411 __ tbr(cond, $op1$$Register, bit, *L);
16412 %}
16413 ins_pipe(pipe_cmp_branch);
16414 ins_short_branch(1);
16415 %}
16416
16417 // And far variants
16418 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16419 match(If cmp (CmpL op1 op2));
16420 effect(USE labl);
16421
16422 ins_cost(BRANCH_COST);
16423 format %{ "cb$cmp $op1, $labl # long" %}
16424 ins_encode %{
16425 Label* L = $labl$$label;
16426 Assembler::Condition cond =
16427 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16428 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16429 %}
16430 ins_pipe(pipe_cmp_branch);
16431 %}
16432
16433 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16434 match(If cmp (CmpI op1 op2));
16435 effect(USE labl);
16436
16437 ins_cost(BRANCH_COST);
16438 format %{ "cb$cmp $op1, $labl # int" %}
16439 ins_encode %{
16440 Label* L = $labl$$label;
16441 Assembler::Condition cond =
16442 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16443 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16444 %}
16445 ins_pipe(pipe_cmp_branch);
16446 %}
16447
16448 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16449 match(If cmp (CmpL (AndL op1 op2) op3));
16450 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16451 effect(USE labl);
16452
16453 ins_cost(BRANCH_COST);
16454 format %{ "tb$cmp $op1, $op2, $labl" %}
16455 ins_encode %{
16456 Label* L = $labl$$label;
16457 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16458 int bit = exact_log2_long($op2$$constant);
16459 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16460 %}
16461 ins_pipe(pipe_cmp_branch);
16462 %}
16463
16464 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16465 match(If cmp (CmpI (AndI op1 op2) op3));
16466 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16467 effect(USE labl);
16468
16469 ins_cost(BRANCH_COST);
16470 format %{ "tb$cmp $op1, $op2, $labl" %}
16471 ins_encode %{
16472 Label* L = $labl$$label;
16473 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16474 int bit = exact_log2((juint)$op2$$constant);
16475 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16476 %}
16477 ins_pipe(pipe_cmp_branch);
16478 %}
16479
16480 // Test bits
16481
16482 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16483 match(Set cr (CmpL (AndL op1 op2) op3));
16484 predicate(Assembler::operand_valid_for_logical_immediate
16485 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16486
16487 ins_cost(INSN_COST);
16488 format %{ "tst $op1, $op2 # long" %}
16489 ins_encode %{
16490 __ tst($op1$$Register, $op2$$constant);
16491 %}
16492 ins_pipe(ialu_reg_reg);
16493 %}
16494
16495 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16496 match(Set cr (CmpI (AndI op1 op2) op3));
16497 predicate(Assembler::operand_valid_for_logical_immediate
16498 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16499
16500 ins_cost(INSN_COST);
16501 format %{ "tst $op1, $op2 # int" %}
16502 ins_encode %{
16503 __ tstw($op1$$Register, $op2$$constant);
16504 %}
16505 ins_pipe(ialu_reg_reg);
16506 %}
16507
16508 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16509 match(Set cr (CmpL (AndL op1 op2) op3));
16510
16511 ins_cost(INSN_COST);
16512 format %{ "tst $op1, $op2 # long" %}
16513 ins_encode %{
16514 __ tst($op1$$Register, $op2$$Register);
16515 %}
16516 ins_pipe(ialu_reg_reg);
16517 %}
16518
16519 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16520 match(Set cr (CmpI (AndI op1 op2) op3));
16521
16522 ins_cost(INSN_COST);
16523 format %{ "tstw $op1, $op2 # int" %}
16524 ins_encode %{
16525 __ tstw($op1$$Register, $op2$$Register);
16526 %}
16527 ins_pipe(ialu_reg_reg);
16528 %}
16529
16530
16531 // Conditional Far Branch
16532 // Conditional Far Branch Unsigned
16533 // TODO: fixme
16534
16535 // counted loop end branch near
16536 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16537 %{
16538 match(CountedLoopEnd cmp cr);
16539
16540 effect(USE lbl);
16541
16542 ins_cost(BRANCH_COST);
16543 // short variant.
16544 // ins_short_branch(1);
16545 format %{ "b$cmp $lbl \t// counted loop end" %}
16546
16547 ins_encode(aarch64_enc_br_con(cmp, lbl));
16548
16549 ins_pipe(pipe_branch);
16550 %}
16551
16552 // counted loop end branch near Unsigned
16553 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16554 %{
16555 match(CountedLoopEnd cmp cr);
16556
16557 effect(USE lbl);
16558
16559 ins_cost(BRANCH_COST);
16560 // short variant.
16561 // ins_short_branch(1);
16562 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16563
16564 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16565
16566 ins_pipe(pipe_branch);
16567 %}
16568
16569 // counted loop end branch far
16570 // counted loop end branch far unsigned
16571 // TODO: fixme
16572
16573 // ============================================================================
16574 // inlined locking and unlocking
16575
16576 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16577 %{
16578 match(Set cr (FastLock object box));
16579 effect(TEMP tmp, TEMP tmp2);
16580
16581 // TODO
16582 // identify correct cost
16583 ins_cost(5 * INSN_COST);
16584 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16585
16586 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16587
16588 ins_pipe(pipe_serial);
16589 %}
16590
16591 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16592 %{
16593 match(Set cr (FastUnlock object box));
16594 effect(TEMP tmp, TEMP tmp2);
16595
16596 ins_cost(5 * INSN_COST);
16597 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16598
16599 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16600
16601 ins_pipe(pipe_serial);
16602 %}
16603
16604
16605 // ============================================================================
16606 // Safepoint Instructions
16607
16608 // TODO
16609 // provide a near and far version of this code
16610
16611 instruct safePoint(rFlagsReg cr, iRegP poll)
16612 %{
16613 match(SafePoint poll);
16614 effect(KILL cr);
16615
16616 format %{
16617 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16618 %}
16619 ins_encode %{
16620 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16621 %}
16622 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16623 %}
16624
16625
16626 // ============================================================================
16627 // Procedure Call/Return Instructions
16628
16629 // Call Java Static Instruction
16630
16631 instruct CallStaticJavaDirect(method meth)
16632 %{
16633 match(CallStaticJava);
16634
16635 effect(USE meth);
16636
16637 ins_cost(CALL_COST);
16638
16639 format %{ "call,static $meth \t// ==> " %}
16640
16641 ins_encode(aarch64_enc_java_static_call(meth),
16642 aarch64_enc_call_epilog);
16643
16644 ins_pipe(pipe_class_call);
16645 %}
16646
16647 // TO HERE
16648
16649 // Call Java Dynamic Instruction
16650 instruct CallDynamicJavaDirect(method meth)
16651 %{
16652 match(CallDynamicJava);
16653
16654 effect(USE meth);
16655
16656 ins_cost(CALL_COST);
16657
16658 format %{ "CALL,dynamic $meth \t// ==> " %}
16659
16660 ins_encode(aarch64_enc_java_dynamic_call(meth),
16661 aarch64_enc_call_epilog);
16662
16663 ins_pipe(pipe_class_call);
16664 %}
16665
16666 // Call Runtime Instruction
16667
16668 instruct CallRuntimeDirect(method meth)
16669 %{
16670 match(CallRuntime);
16671
16672 effect(USE meth);
16673
16674 ins_cost(CALL_COST);
16675
16676 format %{ "CALL, runtime $meth" %}
16677
16678 ins_encode( aarch64_enc_java_to_runtime(meth) );
16679
16680 ins_pipe(pipe_class_call);
16681 %}
16682
16683 // Call Runtime Instruction
16684
16685 instruct CallLeafDirect(method meth)
16686 %{
16687 match(CallLeaf);
16688
16689 effect(USE meth);
16690
16691 ins_cost(CALL_COST);
16692
16693 format %{ "CALL, runtime leaf $meth" %}
16694
16695 ins_encode( aarch64_enc_java_to_runtime(meth) );
16696
16697 ins_pipe(pipe_class_call);
16698 %}
16699
16700 // Call Runtime Instruction
16701
16702 instruct CallLeafNoFPDirect(method meth)
16703 %{
16704 match(CallLeafNoFP);
16705
16706 effect(USE meth);
16707
16708 ins_cost(CALL_COST);
16709
16710 format %{ "CALL, runtime leaf nofp $meth" %}
16711
16712 ins_encode( aarch64_enc_java_to_runtime(meth) );
16713
16714 ins_pipe(pipe_class_call);
16715 %}
16716
16717 // Tail Call; Jump from runtime stub to Java code.
16718 // Also known as an 'interprocedural jump'.
16719 // Target of jump will eventually return to caller.
16720 // TailJump below removes the return address.
16721 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16722 %{
16723 match(TailCall jump_target method_ptr);
16724
16725 ins_cost(CALL_COST);
16726
16727 format %{ "br $jump_target\t# $method_ptr holds method" %}
16728
16729 ins_encode(aarch64_enc_tail_call(jump_target));
16730
16731 ins_pipe(pipe_class_call);
16732 %}
16733
16734 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16735 %{
16736 match(TailJump jump_target ex_oop);
16737
16738 ins_cost(CALL_COST);
16739
16740 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16741
16742 ins_encode(aarch64_enc_tail_jmp(jump_target));
16743
16744 ins_pipe(pipe_class_call);
16745 %}
16746
16747 // Create exception oop: created by stack-crawling runtime code.
16748 // Created exception is now available to this handler, and is setup
16749 // just prior to jumping to this handler. No code emitted.
16750 // TODO check
16751 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16752 instruct CreateException(iRegP_R0 ex_oop)
16753 %{
16754 match(Set ex_oop (CreateEx));
16755
16756 format %{ " -- \t// exception oop; no code emitted" %}
16757
16758 size(0);
16759
16760 ins_encode( /*empty*/ );
16761
16762 ins_pipe(pipe_class_empty);
16763 %}
16764
16765 // Rethrow exception: The exception oop will come in the first
16766 // argument position. Then JUMP (not call) to the rethrow stub code.
16767 instruct RethrowException() %{
16768 match(Rethrow);
16769 ins_cost(CALL_COST);
16770
16771 format %{ "b rethrow_stub" %}
16772
16773 ins_encode( aarch64_enc_rethrow() );
16774
16775 ins_pipe(pipe_class_call);
16776 %}
16777
16778
16779 // Return Instruction
16780 // epilog node loads ret address into lr as part of frame pop
16781 instruct Ret()
16782 %{
16783 match(Return);
16784
16785 format %{ "ret\t// return register" %}
16786
16787 ins_encode( aarch64_enc_ret() );
16788
16789 ins_pipe(pipe_branch);
16790 %}
16791
16792 // Die now.
16793 instruct ShouldNotReachHere() %{
16794 match(Halt);
16795
16796 ins_cost(CALL_COST);
16797 format %{ "ShouldNotReachHere" %}
16798
16799 ins_encode %{
16800 if (is_reachable()) {
16801 __ stop(_halt_reason);
16802 }
16803 %}
16804
16805 ins_pipe(pipe_class_default);
16806 %}
16807
16808 // ============================================================================
16809 // Partial Subtype Check
16810 //
16811 // superklass array for an instance of the superklass. Set a hidden
16812 // internal cache on a hit (cache is checked with exposed code in
16813 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16814 // encoding ALSO sets flags.
16815
16816 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16817 %{
16818 match(Set result (PartialSubtypeCheck sub super));
16819 effect(KILL cr, KILL temp);
16820
16821 ins_cost(1100); // slightly larger than the next version
16822 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16823
16824 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16825
16826 opcode(0x1); // Force zero of result reg on hit
16827
16828 ins_pipe(pipe_class_memory);
16829 %}
16830
16831 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16832 %{
16833 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16834 effect(KILL temp, KILL result);
16835
16836 ins_cost(1100); // slightly larger than the next version
16837 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16838
16839 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16840
16841 opcode(0x0); // Don't zero result reg on hit
16842
16843 ins_pipe(pipe_class_memory);
16844 %}
16845
16846 // Intrisics for String.compareTo()
16847
16848 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16849 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16850 %{
16851 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16852 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16853 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16854
16855 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16856 ins_encode %{
16857 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16858 __ string_compare($str1$$Register, $str2$$Register,
16859 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16860 $tmp1$$Register, $tmp2$$Register,
16861 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16862 %}
16863 ins_pipe(pipe_class_memory);
16864 %}
16865
16866 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16867 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16868 %{
16869 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16870 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16871 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16872
16873 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16874 ins_encode %{
16875 __ string_compare($str1$$Register, $str2$$Register,
16876 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16877 $tmp1$$Register, $tmp2$$Register,
16878 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16879 %}
16880 ins_pipe(pipe_class_memory);
16881 %}
16882
16883 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16884 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16885 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16886 %{
16887 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16888 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16889 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16890 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16891
16892 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16893 ins_encode %{
16894 __ string_compare($str1$$Register, $str2$$Register,
16895 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16896 $tmp1$$Register, $tmp2$$Register,
16897 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16898 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16899 %}
16900 ins_pipe(pipe_class_memory);
16901 %}
16902
16903 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16904 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16905 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16906 %{
16907 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16908 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16909 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16910 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16911
16912 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16913 ins_encode %{
16914 __ string_compare($str1$$Register, $str2$$Register,
16915 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16916 $tmp1$$Register, $tmp2$$Register,
16917 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16918 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16919 %}
16920 ins_pipe(pipe_class_memory);
16921 %}
16922
16923 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16924 // these string_compare variants as NEON register type for convenience so that the prototype of
16925 // string_compare can be shared with all variants.
16926
16927 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16928 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16929 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16930 pRegGov_P1 pgtmp2, rFlagsReg cr)
16931 %{
16932 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16933 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16934 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16935 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16936
16937 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16938 ins_encode %{
16939 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16940 __ string_compare($str1$$Register, $str2$$Register,
16941 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16942 $tmp1$$Register, $tmp2$$Register,
16943 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16944 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16945 StrIntrinsicNode::LL);
16946 %}
16947 ins_pipe(pipe_class_memory);
16948 %}
16949
16950 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16951 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16952 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16953 pRegGov_P1 pgtmp2, rFlagsReg cr)
16954 %{
16955 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16956 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16957 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16958 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16959
16960 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16961 ins_encode %{
16962 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16963 __ string_compare($str1$$Register, $str2$$Register,
16964 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16965 $tmp1$$Register, $tmp2$$Register,
16966 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16967 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16968 StrIntrinsicNode::LU);
16969 %}
16970 ins_pipe(pipe_class_memory);
16971 %}
16972
16973 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16974 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16975 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16976 pRegGov_P1 pgtmp2, rFlagsReg cr)
16977 %{
16978 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16979 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16980 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16981 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16982
16983 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16984 ins_encode %{
16985 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16986 __ string_compare($str1$$Register, $str2$$Register,
16987 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16988 $tmp1$$Register, $tmp2$$Register,
16989 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16990 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16991 StrIntrinsicNode::UL);
16992 %}
16993 ins_pipe(pipe_class_memory);
16994 %}
16995
16996 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16997 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16998 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16999 pRegGov_P1 pgtmp2, rFlagsReg cr)
17000 %{
17001 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17002 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17003 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17004 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17005
17006 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17007 ins_encode %{
17008 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17009 __ string_compare($str1$$Register, $str2$$Register,
17010 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17011 $tmp1$$Register, $tmp2$$Register,
17012 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17013 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17014 StrIntrinsicNode::UU);
17015 %}
17016 ins_pipe(pipe_class_memory);
17017 %}
17018
17019 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17020 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17021 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17022 %{
17023 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17024 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17025 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17026 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17027 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
17028
17029 ins_encode %{
17030 __ string_indexof($str1$$Register, $str2$$Register,
17031 $cnt1$$Register, $cnt2$$Register,
17032 $tmp1$$Register, $tmp2$$Register,
17033 $tmp3$$Register, $tmp4$$Register,
17034 $tmp5$$Register, $tmp6$$Register,
17035 -1, $result$$Register, StrIntrinsicNode::UU);
17036 %}
17037 ins_pipe(pipe_class_memory);
17038 %}
17039
17040 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17041 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17042 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17043 %{
17044 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17045 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17046 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17047 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17048 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17049
17050 ins_encode %{
17051 __ string_indexof($str1$$Register, $str2$$Register,
17052 $cnt1$$Register, $cnt2$$Register,
17053 $tmp1$$Register, $tmp2$$Register,
17054 $tmp3$$Register, $tmp4$$Register,
17055 $tmp5$$Register, $tmp6$$Register,
17056 -1, $result$$Register, StrIntrinsicNode::LL);
17057 %}
17058 ins_pipe(pipe_class_memory);
17059 %}
17060
17061 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17062 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17063 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17064 %{
17065 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17066 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17067 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17068 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17069 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17070
17071 ins_encode %{
17072 __ string_indexof($str1$$Register, $str2$$Register,
17073 $cnt1$$Register, $cnt2$$Register,
17074 $tmp1$$Register, $tmp2$$Register,
17075 $tmp3$$Register, $tmp4$$Register,
17076 $tmp5$$Register, $tmp6$$Register,
17077 -1, $result$$Register, StrIntrinsicNode::UL);
17078 %}
17079 ins_pipe(pipe_class_memory);
17080 %}
17081
17082 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17083 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17084 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17085 %{
17086 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17087 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17088 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17089 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17090 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17091
17092 ins_encode %{
17093 int icnt2 = (int)$int_cnt2$$constant;
17094 __ string_indexof($str1$$Register, $str2$$Register,
17095 $cnt1$$Register, zr,
17096 $tmp1$$Register, $tmp2$$Register,
17097 $tmp3$$Register, $tmp4$$Register, zr, zr,
17098 icnt2, $result$$Register, StrIntrinsicNode::UU);
17099 %}
17100 ins_pipe(pipe_class_memory);
17101 %}
17102
17103 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17104 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17105 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17106 %{
17107 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17108 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17109 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17110 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17111 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17112
17113 ins_encode %{
17114 int icnt2 = (int)$int_cnt2$$constant;
17115 __ string_indexof($str1$$Register, $str2$$Register,
17116 $cnt1$$Register, zr,
17117 $tmp1$$Register, $tmp2$$Register,
17118 $tmp3$$Register, $tmp4$$Register, zr, zr,
17119 icnt2, $result$$Register, StrIntrinsicNode::LL);
17120 %}
17121 ins_pipe(pipe_class_memory);
17122 %}
17123
17124 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17125 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17126 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17127 %{
17128 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17129 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17130 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17131 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17132 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17133
17134 ins_encode %{
17135 int icnt2 = (int)$int_cnt2$$constant;
17136 __ string_indexof($str1$$Register, $str2$$Register,
17137 $cnt1$$Register, zr,
17138 $tmp1$$Register, $tmp2$$Register,
17139 $tmp3$$Register, $tmp4$$Register, zr, zr,
17140 icnt2, $result$$Register, StrIntrinsicNode::UL);
17141 %}
17142 ins_pipe(pipe_class_memory);
17143 %}
17144
17145 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17146 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17147 iRegINoSp tmp3, rFlagsReg cr)
17148 %{
17149 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17150 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17151 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17152 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17153
17154 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17155
17156 ins_encode %{
17157 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17158 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17159 $tmp3$$Register);
17160 %}
17161 ins_pipe(pipe_class_memory);
17162 %}
17163
17164 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17165 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17166 iRegINoSp tmp3, rFlagsReg cr)
17167 %{
17168 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17169 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17170 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17171 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17172
17173 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17174
17175 ins_encode %{
17176 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17177 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17178 $tmp3$$Register);
17179 %}
17180 ins_pipe(pipe_class_memory);
17181 %}
17182
17183 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17184 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17185 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17186 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17187 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17188 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17189 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17190 ins_encode %{
17191 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17192 $result$$Register, $ztmp1$$FloatRegister,
17193 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17194 $ptmp$$PRegister, true /* isL */);
17195 %}
17196 ins_pipe(pipe_class_memory);
17197 %}
17198
17199 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17200 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17201 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17202 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17203 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17204 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17205 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17206 ins_encode %{
17207 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17208 $result$$Register, $ztmp1$$FloatRegister,
17209 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17210 $ptmp$$PRegister, false /* isL */);
17211 %}
17212 ins_pipe(pipe_class_memory);
17213 %}
17214
17215 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17216 iRegI_R0 result, rFlagsReg cr)
17217 %{
17218 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17219 match(Set result (StrEquals (Binary str1 str2) cnt));
17220 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17221
17222 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17223 ins_encode %{
17224 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17225 __ string_equals($str1$$Register, $str2$$Register,
17226 $result$$Register, $cnt$$Register, 1);
17227 %}
17228 ins_pipe(pipe_class_memory);
17229 %}
17230
17231 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17232 iRegI_R0 result, rFlagsReg cr)
17233 %{
17234 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17235 match(Set result (StrEquals (Binary str1 str2) cnt));
17236 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17237
17238 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17239 ins_encode %{
17240 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17241 __ string_equals($str1$$Register, $str2$$Register,
17242 $result$$Register, $cnt$$Register, 2);
17243 %}
17244 ins_pipe(pipe_class_memory);
17245 %}
17246
17247 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17248 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17249 iRegP_R10 tmp, rFlagsReg cr)
17250 %{
17251 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17252 match(Set result (AryEq ary1 ary2));
17253 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17254
17255 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17256 ins_encode %{
17257 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17258 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17259 $result$$Register, $tmp$$Register, 1);
17260 if (tpc == NULL) {
17261 ciEnv::current()->record_failure("CodeCache is full");
17262 return;
17263 }
17264 %}
17265 ins_pipe(pipe_class_memory);
17266 %}
17267
17268 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17269 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17270 iRegP_R10 tmp, rFlagsReg cr)
17271 %{
17272 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17273 match(Set result (AryEq ary1 ary2));
17274 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17275
17276 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17277 ins_encode %{
17278 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17279 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17280 $result$$Register, $tmp$$Register, 2);
17281 if (tpc == NULL) {
17282 ciEnv::current()->record_failure("CodeCache is full");
17283 return;
17284 }
17285 %}
17286 ins_pipe(pipe_class_memory);
17287 %}
17288
17289 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17290 %{
17291 match(Set result (CountPositives ary1 len));
17292 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17293 format %{ "count positives byte[] $ary1,$len -> $result" %}
17294 ins_encode %{
17295 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17296 if (tpc == NULL) {
17297 ciEnv::current()->record_failure("CodeCache is full");
17298 return;
17299 }
17300 %}
17301 ins_pipe( pipe_slow );
17302 %}
17303
17304 // fast char[] to byte[] compression
17305 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17306 vRegD_V0 tmp1, vRegD_V1 tmp2,
17307 vRegD_V2 tmp3, vRegD_V3 tmp4,
17308 iRegI_R0 result, rFlagsReg cr)
17309 %{
17310 match(Set result (StrCompressedCopy src (Binary dst len)));
17311 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17312 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17313
17314 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
17315 ins_encode %{
17316 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17317 $result$$Register,
17318 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17319 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17320 %}
17321 ins_pipe(pipe_slow);
17322 %}
17323
17324 // fast byte[] to char[] inflation
17325 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17326 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17327 %{
17328 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17329 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17330
17331 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
17332 ins_encode %{
17333 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17334 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17335 $tmp3$$FloatRegister, $tmp4$$Register);
17336 if (tpc == NULL) {
17337 ciEnv::current()->record_failure("CodeCache is full");
17338 return;
17339 }
17340 %}
17341 ins_pipe(pipe_class_memory);
17342 %}
17343
17344 // encode char[] to byte[] in ISO_8859_1
17345 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17346 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17347 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17348 iRegI_R0 result, rFlagsReg cr)
17349 %{
17350 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17351 match(Set result (EncodeISOArray src (Binary dst len)));
17352 effect(USE_KILL src, USE_KILL dst, USE len,
17353 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17354
17355 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17356 ins_encode %{
17357 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17358 $result$$Register, false,
17359 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17360 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17361 %}
17362 ins_pipe(pipe_class_memory);
17363 %}
17364
17365 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17366 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17367 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17368 iRegI_R0 result, rFlagsReg cr)
17369 %{
17370 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17371 match(Set result (EncodeISOArray src (Binary dst len)));
17372 effect(USE_KILL src, USE_KILL dst, USE len,
17373 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17374
17375 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17376 ins_encode %{
17377 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17378 $result$$Register, true,
17379 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17380 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17381 %}
17382 ins_pipe(pipe_class_memory);
17383 %}
17384
17385 // ============================================================================
17386 // This name is KNOWN by the ADLC and cannot be changed.
17387 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17388 // for this guy.
17389 instruct tlsLoadP(thread_RegP dst)
17390 %{
17391 match(Set dst (ThreadLocal));
17392
17393 ins_cost(0);
17394
17395 format %{ " -- \t// $dst=Thread::current(), empty" %}
17396
17397 size(0);
17398
17399 ins_encode( /*empty*/ );
17400
17401 ins_pipe(pipe_class_empty);
17402 %}
17403
17404 //----------PEEPHOLE RULES-----------------------------------------------------
17405 // These must follow all instruction definitions as they use the names
17406 // defined in the instructions definitions.
17407 //
17408 // peepmatch ( root_instr_name [preceding_instruction]* );
17409 //
17410 // peepconstraint %{
17411 // (instruction_number.operand_name relational_op instruction_number.operand_name
17412 // [, ...] );
17413 // // instruction numbers are zero-based using left to right order in peepmatch
17414 //
17415 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17416 // // provide an instruction_number.operand_name for each operand that appears
17417 // // in the replacement instruction's match rule
17418 //
17419 // ---------VM FLAGS---------------------------------------------------------
17420 //
17421 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17422 //
17423 // Each peephole rule is given an identifying number starting with zero and
17424 // increasing by one in the order seen by the parser. An individual peephole
17425 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17426 // on the command-line.
17427 //
17428 // ---------CURRENT LIMITATIONS----------------------------------------------
17429 //
17430 // Only match adjacent instructions in same basic block
17431 // Only equality constraints
17432 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17433 // Only one replacement instruction
17434 //
17435 // ---------EXAMPLE----------------------------------------------------------
17436 //
17437 // // pertinent parts of existing instructions in architecture description
17438 // instruct movI(iRegINoSp dst, iRegI src)
17439 // %{
17440 // match(Set dst (CopyI src));
17441 // %}
17442 //
17443 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17444 // %{
17445 // match(Set dst (AddI dst src));
17446 // effect(KILL cr);
17447 // %}
17448 //
17449 // // Change (inc mov) to lea
17450 // peephole %{
17451 // // increment preceded by register-register move
17452 // peepmatch ( incI_iReg movI );
17453 // // require that the destination register of the increment
17454 // // match the destination register of the move
17455 // peepconstraint ( 0.dst == 1.dst );
17456 // // construct a replacement instruction that sets
17457 // // the destination to ( move's source register + one )
17458 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17459 // %}
17460 //
17461
17462 // Implementation no longer uses movX instructions since
17463 // machine-independent system no longer uses CopyX nodes.
17464 //
17465 // peephole
17466 // %{
17467 // peepmatch (incI_iReg movI);
17468 // peepconstraint (0.dst == 1.dst);
17469 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17470 // %}
17471
17472 // peephole
17473 // %{
17474 // peepmatch (decI_iReg movI);
17475 // peepconstraint (0.dst == 1.dst);
17476 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17477 // %}
17478
17479 // peephole
17480 // %{
17481 // peepmatch (addI_iReg_imm movI);
17482 // peepconstraint (0.dst == 1.dst);
17483 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17484 // %}
17485
17486 // peephole
17487 // %{
17488 // peepmatch (incL_iReg movL);
17489 // peepconstraint (0.dst == 1.dst);
17490 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17491 // %}
17492
17493 // peephole
17494 // %{
17495 // peepmatch (decL_iReg movL);
17496 // peepconstraint (0.dst == 1.dst);
17497 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17498 // %}
17499
17500 // peephole
17501 // %{
17502 // peepmatch (addL_iReg_imm movL);
17503 // peepconstraint (0.dst == 1.dst);
17504 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17505 // %}
17506
17507 // peephole
17508 // %{
17509 // peepmatch (addP_iReg_imm movP);
17510 // peepconstraint (0.dst == 1.dst);
17511 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17512 // %}
17513
17514 // // Change load of spilled value to only a spill
17515 // instruct storeI(memory mem, iRegI src)
17516 // %{
17517 // match(Set mem (StoreI mem src));
17518 // %}
17519 //
17520 // instruct loadI(iRegINoSp dst, memory mem)
17521 // %{
17522 // match(Set dst (LoadI mem));
17523 // %}
17524 //
17525
17526 //----------SMARTSPILL RULES---------------------------------------------------
17527 // These must follow all instruction definitions as they use the names
17528 // defined in the instructions definitions.
17529
17530 // Local Variables:
17531 // mode: c++
17532 // End:
--- EOF ---