1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for heapbase register
668 reg_class heapbase_reg(
669 R27, R27_H
670 );
671
672 // Class for thread register
673 reg_class thread_reg(
674 R28, R28_H
675 );
676
677 // Class for frame pointer register
678 reg_class fp_reg(
679 R29, R29_H
680 );
681
682 // Class for link register
683 reg_class lr_reg(
684 R30, R30_H
685 );
686
687 // Class for long sp register
688 reg_class sp_reg(
689 R31, R31_H
690 );
691
692 // Class for all pointer registers
693 reg_class ptr_reg %{
694 return _PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers
698 reg_class no_special_ptr_reg %{
699 return _NO_SPECIAL_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133
1134 class CallStubImpl {
1135
1136 //--------------------------------------------------------------
1137 //---< Used for optimization in Compile::shorten_branches >---
1138 //--------------------------------------------------------------
1139
1140 public:
1141 // Size of call trampoline stub.
1142 static uint size_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145
1146 // number of relocations needed by a call trampoline stub
1147 static uint reloc_call_trampoline() {
1148 return 0; // no call trampolines on this platform
1149 }
1150 };
1151
1152 class HandlerImpl {
1153
1154 public:
1155
1156 static int emit_exception_handler(CodeBuffer &cbuf);
1157 static int emit_deopt_handler(CodeBuffer& cbuf);
1158
1159 static uint size_exception_handler() {
1160 return MacroAssembler::far_codestub_branch_size();
1161 }
1162
1163 static uint size_deopt_handler() {
1164 // count one adr and one far branch instruction
1165 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1166 }
1167 };
1168
1169 class Node::PD {
1170 public:
1171 enum NodeFlags {
1172 _last_flag = Node::_last_flag
1173 };
1174 };
1175
1176 bool is_CAS(int opcode, bool maybe_volatile);
1177
1178 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1179
1180 bool unnecessary_acquire(const Node *barrier);
1181 bool needs_acquiring_load(const Node *load);
1182
1183 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1184
1185 bool unnecessary_release(const Node *barrier);
1186 bool unnecessary_volatile(const Node *barrier);
1187 bool needs_releasing_store(const Node *store);
1188
1189 // predicate controlling translation of CompareAndSwapX
1190 bool needs_acquiring_load_exclusive(const Node *load);
1191
1192 // predicate controlling addressing modes
1193 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1194 %}
1195
1196 source %{
1197
1198 // Derived RegMask with conditionally allocatable registers
1199
1200 void PhaseOutput::pd_perform_mach_node_analysis() {
1201 }
1202
1203 int MachNode::pd_alignment_required() const {
1204 return 1;
1205 }
1206
1207 int MachNode::compute_padding(int current_offset) const {
1208 return 0;
1209 }
1210
1211 RegMask _ANY_REG32_mask;
1212 RegMask _ANY_REG_mask;
1213 RegMask _PTR_REG_mask;
1214 RegMask _NO_SPECIAL_REG32_mask;
1215 RegMask _NO_SPECIAL_REG_mask;
1216 RegMask _NO_SPECIAL_PTR_REG_mask;
1217
1218 void reg_mask_init() {
1219 // We derive below RegMask(s) from the ones which are auto-generated from
1220 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1221 // registers conditionally reserved.
1222
1223 _ANY_REG32_mask = _ALL_REG32_mask;
1224 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1225
1226 _ANY_REG_mask = _ALL_REG_mask;
1227
1228 _PTR_REG_mask = _ALL_REG_mask;
1229
1230 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1231 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1232
1233 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1234 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1235
1236 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1237 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1238
1239 // r27 is not allocatable when compressed oops is on and heapbase is not
1240 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1241 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1242 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1244 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1245 }
1246
1247 // r29 is not allocatable when PreserveFramePointer is on
1248 if (PreserveFramePointer) {
1249 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1251 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1252 }
1253 }
1254
1255 // Optimizaton of volatile gets and puts
1256 // -------------------------------------
1257 //
1258 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1259 // use to implement volatile reads and writes. For a volatile read
1260 // we simply need
1261 //
1262 // ldar<x>
1263 //
1264 // and for a volatile write we need
1265 //
1266 // stlr<x>
1267 //
1268 // Alternatively, we can implement them by pairing a normal
1269 // load/store with a memory barrier. For a volatile read we need
1270 //
1271 // ldr<x>
1272 // dmb ishld
1273 //
1274 // for a volatile write
1275 //
1276 // dmb ish
1277 // str<x>
1278 // dmb ish
1279 //
1280 // We can also use ldaxr and stlxr to implement compare and swap CAS
1281 // sequences. These are normally translated to an instruction
1282 // sequence like the following
1283 //
1284 // dmb ish
1285 // retry:
1286 // ldxr<x> rval raddr
1287 // cmp rval rold
1288 // b.ne done
1289 // stlxr<x> rval, rnew, rold
1290 // cbnz rval retry
1291 // done:
1292 // cset r0, eq
1293 // dmb ishld
1294 //
1295 // Note that the exclusive store is already using an stlxr
1296 // instruction. That is required to ensure visibility to other
1297 // threads of the exclusive write (assuming it succeeds) before that
1298 // of any subsequent writes.
1299 //
1300 // The following instruction sequence is an improvement on the above
1301 //
1302 // retry:
1303 // ldaxr<x> rval raddr
1304 // cmp rval rold
1305 // b.ne done
1306 // stlxr<x> rval, rnew, rold
1307 // cbnz rval retry
1308 // done:
1309 // cset r0, eq
1310 //
1311 // We don't need the leading dmb ish since the stlxr guarantees
1312 // visibility of prior writes in the case that the swap is
1313 // successful. Crucially we don't have to worry about the case where
1314 // the swap is not successful since no valid program should be
1315 // relying on visibility of prior changes by the attempting thread
1316 // in the case where the CAS fails.
1317 //
1318 // Similarly, we don't need the trailing dmb ishld if we substitute
1319 // an ldaxr instruction since that will provide all the guarantees we
1320 // require regarding observation of changes made by other threads
1321 // before any change to the CAS address observed by the load.
1322 //
1323 // In order to generate the desired instruction sequence we need to
1324 // be able to identify specific 'signature' ideal graph node
1325 // sequences which i) occur as a translation of a volatile reads or
1326 // writes or CAS operations and ii) do not occur through any other
1327 // translation or graph transformation. We can then provide
1328 // alternative aldc matching rules which translate these node
1329 // sequences to the desired machine code sequences. Selection of the
1330 // alternative rules can be implemented by predicates which identify
1331 // the relevant node sequences.
1332 //
1333 // The ideal graph generator translates a volatile read to the node
1334 // sequence
1335 //
1336 // LoadX[mo_acquire]
1337 // MemBarAcquire
1338 //
1339 // As a special case when using the compressed oops optimization we
1340 // may also see this variant
1341 //
1342 // LoadN[mo_acquire]
1343 // DecodeN
1344 // MemBarAcquire
1345 //
1346 // A volatile write is translated to the node sequence
1347 //
1348 // MemBarRelease
1349 // StoreX[mo_release] {CardMark}-optional
1350 // MemBarVolatile
1351 //
1352 // n.b. the above node patterns are generated with a strict
1353 // 'signature' configuration of input and output dependencies (see
1354 // the predicates below for exact details). The card mark may be as
1355 // simple as a few extra nodes or, in a few GC configurations, may
1356 // include more complex control flow between the leading and
1357 // trailing memory barriers. However, whatever the card mark
1358 // configuration these signatures are unique to translated volatile
1359 // reads/stores -- they will not appear as a result of any other
1360 // bytecode translation or inlining nor as a consequence of
1361 // optimizing transforms.
1362 //
1363 // We also want to catch inlined unsafe volatile gets and puts and
1364 // be able to implement them using either ldar<x>/stlr<x> or some
1365 // combination of ldr<x>/stlr<x> and dmb instructions.
1366 //
1367 // Inlined unsafe volatiles puts manifest as a minor variant of the
1368 // normal volatile put node sequence containing an extra cpuorder
1369 // membar
1370 //
1371 // MemBarRelease
1372 // MemBarCPUOrder
1373 // StoreX[mo_release] {CardMark}-optional
1374 // MemBarCPUOrder
1375 // MemBarVolatile
1376 //
1377 // n.b. as an aside, a cpuorder membar is not itself subject to
1378 // matching and translation by adlc rules. However, the rule
1379 // predicates need to detect its presence in order to correctly
1380 // select the desired adlc rules.
1381 //
1382 // Inlined unsafe volatile gets manifest as a slightly different
1383 // node sequence to a normal volatile get because of the
1384 // introduction of some CPUOrder memory barriers to bracket the
1385 // Load. However, but the same basic skeleton of a LoadX feeding a
1386 // MemBarAcquire, possibly through an optional DecodeN, is still
1387 // present
1388 //
1389 // MemBarCPUOrder
1390 // || \\
1391 // MemBarCPUOrder LoadX[mo_acquire]
1392 // || |
1393 // || {DecodeN} optional
1394 // || /
1395 // MemBarAcquire
1396 //
1397 // In this case the acquire membar does not directly depend on the
1398 // load. However, we can be sure that the load is generated from an
1399 // inlined unsafe volatile get if we see it dependent on this unique
1400 // sequence of membar nodes. Similarly, given an acquire membar we
1401 // can know that it was added because of an inlined unsafe volatile
1402 // get if it is fed and feeds a cpuorder membar and if its feed
1403 // membar also feeds an acquiring load.
1404 //
1405 // Finally an inlined (Unsafe) CAS operation is translated to the
1406 // following ideal graph
1407 //
1408 // MemBarRelease
1409 // MemBarCPUOrder
1410 // CompareAndSwapX {CardMark}-optional
1411 // MemBarCPUOrder
1412 // MemBarAcquire
1413 //
1414 // So, where we can identify these volatile read and write
1415 // signatures we can choose to plant either of the above two code
1416 // sequences. For a volatile read we can simply plant a normal
1417 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1418 // also choose to inhibit translation of the MemBarAcquire and
1419 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1420 //
1421 // When we recognise a volatile store signature we can choose to
1422 // plant at a dmb ish as a translation for the MemBarRelease, a
1423 // normal str<x> and then a dmb ish for the MemBarVolatile.
1424 // Alternatively, we can inhibit translation of the MemBarRelease
1425 // and MemBarVolatile and instead plant a simple stlr<x>
1426 // instruction.
1427 //
1428 // when we recognise a CAS signature we can choose to plant a dmb
1429 // ish as a translation for the MemBarRelease, the conventional
1430 // macro-instruction sequence for the CompareAndSwap node (which
1431 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1432 // Alternatively, we can elide generation of the dmb instructions
1433 // and plant the alternative CompareAndSwap macro-instruction
1434 // sequence (which uses ldaxr<x>).
1435 //
1436 // Of course, the above only applies when we see these signature
1437 // configurations. We still want to plant dmb instructions in any
1438 // other cases where we may see a MemBarAcquire, MemBarRelease or
1439 // MemBarVolatile. For example, at the end of a constructor which
1440 // writes final/volatile fields we will see a MemBarRelease
1441 // instruction and this needs a 'dmb ish' lest we risk the
1442 // constructed object being visible without making the
1443 // final/volatile field writes visible.
1444 //
1445 // n.b. the translation rules below which rely on detection of the
1446 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1447 // If we see anything other than the signature configurations we
1448 // always just translate the loads and stores to ldr<x> and str<x>
1449 // and translate acquire, release and volatile membars to the
1450 // relevant dmb instructions.
1451 //
1452
1453 // is_CAS(int opcode, bool maybe_volatile)
1454 //
1455 // return true if opcode is one of the possible CompareAndSwapX
1456 // values otherwise false.
1457
1458 bool is_CAS(int opcode, bool maybe_volatile)
1459 {
1460 switch(opcode) {
1461 // We handle these
1462 case Op_CompareAndSwapI:
1463 case Op_CompareAndSwapL:
1464 case Op_CompareAndSwapP:
1465 case Op_CompareAndSwapN:
1466 case Op_ShenandoahCompareAndSwapP:
1467 case Op_ShenandoahCompareAndSwapN:
1468 case Op_CompareAndSwapB:
1469 case Op_CompareAndSwapS:
1470 case Op_GetAndSetI:
1471 case Op_GetAndSetL:
1472 case Op_GetAndSetP:
1473 case Op_GetAndSetN:
1474 case Op_GetAndAddI:
1475 case Op_GetAndAddL:
1476 return true;
1477 case Op_CompareAndExchangeI:
1478 case Op_CompareAndExchangeN:
1479 case Op_CompareAndExchangeB:
1480 case Op_CompareAndExchangeS:
1481 case Op_CompareAndExchangeL:
1482 case Op_CompareAndExchangeP:
1483 case Op_WeakCompareAndSwapB:
1484 case Op_WeakCompareAndSwapS:
1485 case Op_WeakCompareAndSwapI:
1486 case Op_WeakCompareAndSwapL:
1487 case Op_WeakCompareAndSwapP:
1488 case Op_WeakCompareAndSwapN:
1489 case Op_ShenandoahWeakCompareAndSwapP:
1490 case Op_ShenandoahWeakCompareAndSwapN:
1491 case Op_ShenandoahCompareAndExchangeP:
1492 case Op_ShenandoahCompareAndExchangeN:
1493 return maybe_volatile;
1494 default:
1495 return false;
1496 }
1497 }
1498
1499 // helper to determine the maximum number of Phi nodes we may need to
1500 // traverse when searching from a card mark membar for the merge mem
1501 // feeding a trailing membar or vice versa
1502
1503 // predicates controlling emit of ldr<x>/ldar<x>
1504
1505 bool unnecessary_acquire(const Node *barrier)
1506 {
1507 assert(barrier->is_MemBar(), "expecting a membar");
1508
1509 MemBarNode* mb = barrier->as_MemBar();
1510
1511 if (mb->trailing_load()) {
1512 return true;
1513 }
1514
1515 if (mb->trailing_load_store()) {
1516 Node* load_store = mb->in(MemBarNode::Precedent);
1517 assert(load_store->is_LoadStore(), "unexpected graph shape");
1518 return is_CAS(load_store->Opcode(), true);
1519 }
1520
1521 return false;
1522 }
1523
1524 bool needs_acquiring_load(const Node *n)
1525 {
1526 assert(n->is_Load(), "expecting a load");
1527 LoadNode *ld = n->as_Load();
1528 return ld->is_acquire();
1529 }
1530
1531 bool unnecessary_release(const Node *n)
1532 {
1533 assert((n->is_MemBar() &&
1534 n->Opcode() == Op_MemBarRelease),
1535 "expecting a release membar");
1536
1537 MemBarNode *barrier = n->as_MemBar();
1538 if (!barrier->leading()) {
1539 return false;
1540 } else {
1541 Node* trailing = barrier->trailing_membar();
1542 MemBarNode* trailing_mb = trailing->as_MemBar();
1543 assert(trailing_mb->trailing(), "Not a trailing membar?");
1544 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1545
1546 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1547 if (mem->is_Store()) {
1548 assert(mem->as_Store()->is_release(), "");
1549 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1550 return true;
1551 } else {
1552 assert(mem->is_LoadStore(), "");
1553 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1554 return is_CAS(mem->Opcode(), true);
1555 }
1556 }
1557 return false;
1558 }
1559
1560 bool unnecessary_volatile(const Node *n)
1561 {
1562 // assert n->is_MemBar();
1563 MemBarNode *mbvol = n->as_MemBar();
1564
1565 bool release = mbvol->trailing_store();
1566 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1567 #ifdef ASSERT
1568 if (release) {
1569 Node* leading = mbvol->leading_membar();
1570 assert(leading->Opcode() == Op_MemBarRelease, "");
1571 assert(leading->as_MemBar()->leading_store(), "");
1572 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1573 }
1574 #endif
1575
1576 return release;
1577 }
1578
1579 // predicates controlling emit of str<x>/stlr<x>
1580
1581 bool needs_releasing_store(const Node *n)
1582 {
1583 // assert n->is_Store();
1584 StoreNode *st = n->as_Store();
1585 return st->trailing_membar() != NULL;
1586 }
1587
1588 // predicate controlling translation of CAS
1589 //
1590 // returns true if CAS needs to use an acquiring load otherwise false
1591
1592 bool needs_acquiring_load_exclusive(const Node *n)
1593 {
1594 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1595 LoadStoreNode* ldst = n->as_LoadStore();
1596 if (is_CAS(n->Opcode(), false)) {
1597 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1598 } else {
1599 return ldst->trailing_membar() != NULL;
1600 }
1601
1602 // so we can just return true here
1603 return true;
1604 }
1605
1606 #define __ _masm.
1607
1608 // advance declarations for helper functions to convert register
1609 // indices to register objects
1610
1611 // the ad file has to provide implementations of certain methods
1612 // expected by the generic code
1613 //
1614 // REQUIRED FUNCTIONALITY
1615
1616 //=============================================================================
1617
1618 // !!!!! Special hack to get all types of calls to specify the byte offset
1619 // from the start of the call to the point where the return address
1620 // will point.
1621
1622 int MachCallStaticJavaNode::ret_addr_offset()
1623 {
1624 // call should be a simple bl
1625 int off = 4;
1626 return off;
1627 }
1628
1629 int MachCallDynamicJavaNode::ret_addr_offset()
1630 {
1631 return 16; // movz, movk, movk, bl
1632 }
1633
1634 int MachCallRuntimeNode::ret_addr_offset() {
1635 // for generated stubs the call will be
1636 // bl(addr)
1637 // or with far branches
1638 // bl(trampoline_stub)
1639 // for real runtime callouts it will be six instructions
1640 // see aarch64_enc_java_to_runtime
1641 // adr(rscratch2, retaddr)
1642 // lea(rscratch1, RuntimeAddress(addr)
1643 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1644 // blr(rscratch1)
1645 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1646 if (cb) {
1647 return 1 * NativeInstruction::instruction_size;
1648 } else {
1649 return 6 * NativeInstruction::instruction_size;
1650 }
1651 }
1652
1653 //=============================================================================
1654
1655 #ifndef PRODUCT
1656 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1657 st->print("BREAKPOINT");
1658 }
1659 #endif
1660
1661 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1662 C2_MacroAssembler _masm(&cbuf);
1663 __ brk(0);
1664 }
1665
1666 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1667 return MachNode::size(ra_);
1668 }
1669
1670 //=============================================================================
1671
1672 #ifndef PRODUCT
1673 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1674 st->print("nop \t# %d bytes pad for loops and calls", _count);
1675 }
1676 #endif
1677
1678 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1679 C2_MacroAssembler _masm(&cbuf);
1680 for (int i = 0; i < _count; i++) {
1681 __ nop();
1682 }
1683 }
1684
1685 uint MachNopNode::size(PhaseRegAlloc*) const {
1686 return _count * NativeInstruction::instruction_size;
1687 }
1688
1689 //=============================================================================
1690 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1691
1692 int ConstantTable::calculate_table_base_offset() const {
1693 return 0; // absolute addressing, no offset
1694 }
1695
1696 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1697 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1698 ShouldNotReachHere();
1699 }
1700
1701 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1702 // Empty encoding
1703 }
1704
1705 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1706 return 0;
1707 }
1708
1709 #ifndef PRODUCT
1710 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1711 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1712 }
1713 #endif
1714
1715 #ifndef PRODUCT
1716 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1717 Compile* C = ra_->C;
1718
1719 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1720
1721 if (C->output()->need_stack_bang(framesize))
1722 st->print("# stack bang size=%d\n\t", framesize);
1723
1724 if (VM_Version::use_rop_protection()) {
1725 st->print("ldr zr, [lr]\n\t");
1726 st->print("pacia lr, rfp\n\t");
1727 }
1728 if (framesize < ((1 << 9) + 2 * wordSize)) {
1729 st->print("sub sp, sp, #%d\n\t", framesize);
1730 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1731 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1732 } else {
1733 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1734 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1735 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1736 st->print("sub sp, sp, rscratch1");
1737 }
1738 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1739 st->print("\n\t");
1740 st->print("ldr rscratch1, [guard]\n\t");
1741 st->print("dmb ishld\n\t");
1742 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1743 st->print("cmp rscratch1, rscratch2\n\t");
1744 st->print("b.eq skip");
1745 st->print("\n\t");
1746 st->print("blr #nmethod_entry_barrier_stub\n\t");
1747 st->print("b skip\n\t");
1748 st->print("guard: int\n\t");
1749 st->print("\n\t");
1750 st->print("skip:\n\t");
1751 }
1752 }
1753 #endif
1754
1755 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1756 Compile* C = ra_->C;
1757 C2_MacroAssembler _masm(&cbuf);
1758
1759 // n.b. frame size includes space for return pc and rfp
1760 const int framesize = C->output()->frame_size_in_bytes();
1761
1762 // insert a nop at the start of the prolog so we can patch in a
1763 // branch if we need to invalidate the method later
1764 __ nop();
1765
1766 if (C->clinit_barrier_on_entry()) {
1767 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1768
1769 Label L_skip_barrier;
1770
1771 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1772 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1773 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1774 __ bind(L_skip_barrier);
1775 }
1776
1777 if (C->max_vector_size() > 0) {
1778 __ reinitialize_ptrue();
1779 }
1780
1781 int bangsize = C->output()->bang_size_in_bytes();
1782 if (C->output()->need_stack_bang(bangsize))
1783 __ generate_stack_overflow_check(bangsize);
1784
1785 __ build_frame(framesize);
1786
1787 if (C->stub_function() == NULL) {
1788 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1789 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1790 // Dummy labels for just measuring the code size
1791 Label dummy_slow_path;
1792 Label dummy_continuation;
1793 Label dummy_guard;
1794 Label* slow_path = &dummy_slow_path;
1795 Label* continuation = &dummy_continuation;
1796 Label* guard = &dummy_guard;
1797 if (!Compile::current()->output()->in_scratch_emit_size()) {
1798 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1799 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1800 Compile::current()->output()->add_stub(stub);
1801 slow_path = &stub->entry();
1802 continuation = &stub->continuation();
1803 guard = &stub->guard();
1804 }
1805 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1806 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1807 }
1808 }
1809
1810 if (VerifyStackAtCalls) {
1811 Unimplemented();
1812 }
1813
1814 C->output()->set_frame_complete(cbuf.insts_size());
1815
1816 if (C->has_mach_constant_base_node()) {
1817 // NOTE: We set the table base offset here because users might be
1818 // emitted before MachConstantBaseNode.
1819 ConstantTable& constant_table = C->output()->constant_table();
1820 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1821 }
1822 }
1823
1824 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1825 {
1826 return MachNode::size(ra_); // too many variables; just compute it
1827 // the hard way
1828 }
1829
1830 int MachPrologNode::reloc() const
1831 {
1832 return 0;
1833 }
1834
1835 //=============================================================================
1836
1837 #ifndef PRODUCT
1838 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1839 Compile* C = ra_->C;
1840 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1841
1842 st->print("# pop frame %d\n\t",framesize);
1843
1844 if (framesize == 0) {
1845 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1846 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1847 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1848 st->print("add sp, sp, #%d\n\t", framesize);
1849 } else {
1850 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1851 st->print("add sp, sp, rscratch1\n\t");
1852 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1853 }
1854 if (VM_Version::use_rop_protection()) {
1855 st->print("autia lr, rfp\n\t");
1856 st->print("ldr zr, [lr]\n\t");
1857 }
1858
1859 if (do_polling() && C->is_method_compilation()) {
1860 st->print("# test polling word\n\t");
1861 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1862 st->print("cmp sp, rscratch1\n\t");
1863 st->print("bhi #slow_path");
1864 }
1865 }
1866 #endif
1867
1868 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1869 Compile* C = ra_->C;
1870 C2_MacroAssembler _masm(&cbuf);
1871 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1872
1873 __ remove_frame(framesize);
1874
1875 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1876 __ reserved_stack_check();
1877 }
1878
1879 if (do_polling() && C->is_method_compilation()) {
1880 Label dummy_label;
1881 Label* code_stub = &dummy_label;
1882 if (!C->output()->in_scratch_emit_size()) {
1883 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1884 C->output()->add_stub(stub);
1885 code_stub = &stub->entry();
1886 }
1887 __ relocate(relocInfo::poll_return_type);
1888 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1889 }
1890 }
1891
1892 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1893 // Variable size. Determine dynamically.
1894 return MachNode::size(ra_);
1895 }
1896
1897 int MachEpilogNode::reloc() const {
1898 // Return number of relocatable values contained in this instruction.
1899 return 1; // 1 for polling page.
1900 }
1901
1902 const Pipeline * MachEpilogNode::pipeline() const {
1903 return MachNode::pipeline_class();
1904 }
1905
1906 //=============================================================================
1907
1908 // Figure out which register class each belongs in: rc_int, rc_float or
1909 // rc_stack.
1910 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1911
1912 static enum RC rc_class(OptoReg::Name reg) {
1913
1914 if (reg == OptoReg::Bad) {
1915 return rc_bad;
1916 }
1917
1918 // we have 32 int registers * 2 halves
1919 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1920
1921 if (reg < slots_of_int_registers) {
1922 return rc_int;
1923 }
1924
1925 // we have 32 float register * 8 halves
1926 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1927 if (reg < slots_of_int_registers + slots_of_float_registers) {
1928 return rc_float;
1929 }
1930
1931 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1932 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1933 return rc_predicate;
1934 }
1935
1936 // Between predicate regs & stack is the flags.
1937 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1938
1939 return rc_stack;
1940 }
1941
1942 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1943 Compile* C = ra_->C;
1944
1945 // Get registers to move.
1946 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1947 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1948 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1949 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1950
1951 enum RC src_hi_rc = rc_class(src_hi);
1952 enum RC src_lo_rc = rc_class(src_lo);
1953 enum RC dst_hi_rc = rc_class(dst_hi);
1954 enum RC dst_lo_rc = rc_class(dst_lo);
1955
1956 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1957
1958 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1959 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1960 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1961 "expected aligned-adjacent pairs");
1962 }
1963
1964 if (src_lo == dst_lo && src_hi == dst_hi) {
1965 return 0; // Self copy, no move.
1966 }
1967
1968 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1969 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1970 int src_offset = ra_->reg2offset(src_lo);
1971 int dst_offset = ra_->reg2offset(dst_lo);
1972
1973 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1974 uint ireg = ideal_reg();
1975 if (ireg == Op_VecA && cbuf) {
1976 C2_MacroAssembler _masm(cbuf);
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (cbuf) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 C2_MacroAssembler _masm(cbuf);
1998 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1999 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2000 // stack->stack
2001 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2002 if (ireg == Op_VecD) {
2003 __ unspill(rscratch1, true, src_offset);
2004 __ spill(rscratch1, true, dst_offset);
2005 } else {
2006 __ spill_copy128(src_offset, dst_offset);
2007 }
2008 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2009 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2010 ireg == Op_VecD ? __ T8B : __ T16B,
2011 as_FloatRegister(Matcher::_regEncode[src_lo]));
2012 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2013 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2014 ireg == Op_VecD ? __ D : __ Q,
2015 ra_->reg2offset(dst_lo));
2016 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2017 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2018 ireg == Op_VecD ? __ D : __ Q,
2019 ra_->reg2offset(src_lo));
2020 } else {
2021 ShouldNotReachHere();
2022 }
2023 }
2024 } else if (cbuf) {
2025 C2_MacroAssembler _masm(cbuf);
2026 switch (src_lo_rc) {
2027 case rc_int:
2028 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2029 if (is64) {
2030 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 } else {
2033 C2_MacroAssembler _masm(cbuf);
2034 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2035 as_Register(Matcher::_regEncode[src_lo]));
2036 }
2037 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2038 if (is64) {
2039 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 } else {
2042 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2043 as_Register(Matcher::_regEncode[src_lo]));
2044 }
2045 } else { // gpr --> stack spill
2046 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2047 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2048 }
2049 break;
2050 case rc_float:
2051 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2052 if (is64) {
2053 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 } else {
2056 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2057 as_FloatRegister(Matcher::_regEncode[src_lo]));
2058 }
2059 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2060 if (is64) {
2061 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 } else {
2064 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2065 as_FloatRegister(Matcher::_regEncode[src_lo]));
2066 }
2067 } else { // fpr --> stack spill
2068 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2069 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2070 is64 ? __ D : __ S, dst_offset);
2071 }
2072 break;
2073 case rc_stack:
2074 if (dst_lo_rc == rc_int) { // stack --> gpr load
2075 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2076 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2077 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2078 is64 ? __ D : __ S, src_offset);
2079 } else if (dst_lo_rc == rc_predicate) {
2080 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2081 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2082 } else { // stack --> stack copy
2083 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2084 if (ideal_reg() == Op_RegVectMask) {
2085 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 __ unspill(rscratch1, is64, src_offset);
2089 __ spill(rscratch1, is64, dst_offset);
2090 }
2091 }
2092 break;
2093 case rc_predicate:
2094 if (dst_lo_rc == rc_predicate) {
2095 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2096 } else if (dst_lo_rc == rc_stack) {
2097 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2098 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2099 } else {
2100 assert(false, "bad src and dst rc_class combination.");
2101 ShouldNotReachHere();
2102 }
2103 break;
2104 default:
2105 assert(false, "bad rc_class for spill");
2106 ShouldNotReachHere();
2107 }
2108 }
2109
2110 if (st) {
2111 st->print("spill ");
2112 if (src_lo_rc == rc_stack) {
2113 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2114 } else {
2115 st->print("%s -> ", Matcher::regName[src_lo]);
2116 }
2117 if (dst_lo_rc == rc_stack) {
2118 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2119 } else {
2120 st->print("%s", Matcher::regName[dst_lo]);
2121 }
2122 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2123 int vsize = 0;
2124 switch (ideal_reg()) {
2125 case Op_VecD:
2126 vsize = 64;
2127 break;
2128 case Op_VecX:
2129 vsize = 128;
2130 break;
2131 case Op_VecA:
2132 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2133 break;
2134 default:
2135 assert(false, "bad register type for spill");
2136 ShouldNotReachHere();
2137 }
2138 st->print("\t# vector spill size = %d", vsize);
2139 } else if (ideal_reg() == Op_RegVectMask) {
2140 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2141 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2142 st->print("\t# predicate spill size = %d", vsize);
2143 } else {
2144 st->print("\t# spill size = %d", is64 ? 64 : 32);
2145 }
2146 }
2147
2148 return 0;
2149
2150 }
2151
2152 #ifndef PRODUCT
2153 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2154 if (!ra_)
2155 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2156 else
2157 implementation(NULL, ra_, false, st);
2158 }
2159 #endif
2160
2161 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2162 implementation(&cbuf, ra_, false, NULL);
2163 }
2164
2165 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2166 return MachNode::size(ra_);
2167 }
2168
2169 //=============================================================================
2170
2171 #ifndef PRODUCT
2172 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2173 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2174 int reg = ra_->get_reg_first(this);
2175 st->print("add %s, rsp, #%d]\t# box lock",
2176 Matcher::regName[reg], offset);
2177 }
2178 #endif
2179
2180 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2181 C2_MacroAssembler _masm(&cbuf);
2182
2183 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2184 int reg = ra_->get_encode(this);
2185
2186 // This add will handle any 24-bit signed offset. 24 bits allows an
2187 // 8 megabyte stack frame.
2188 __ add(as_Register(reg), sp, offset);
2189 }
2190
2191 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2192 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2193 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2194
2195 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2196 return NativeInstruction::instruction_size;
2197 } else {
2198 return 2 * NativeInstruction::instruction_size;
2199 }
2200 }
2201
2202 //=============================================================================
2203
2204 #ifndef PRODUCT
2205 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2206 {
2207 st->print_cr("# MachUEPNode");
2208 if (UseCompressedClassPointers) {
2209 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2210 if (CompressedKlassPointers::shift() != 0) {
2211 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2212 }
2213 } else {
2214 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2215 }
2216 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2217 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2218 }
2219 #endif
2220
2221 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2222 {
2223 // This is the unverified entry point.
2224 C2_MacroAssembler _masm(&cbuf);
2225
2226 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2227 Label skip;
2228 // TODO
2229 // can we avoid this skip and still use a reloc?
2230 __ br(Assembler::EQ, skip);
2231 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2232 __ bind(skip);
2233 }
2234
2235 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2236 {
2237 return MachNode::size(ra_);
2238 }
2239
2240 // REQUIRED EMIT CODE
2241
2242 //=============================================================================
2243
2244 // Emit exception handler code.
2245 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2246 {
2247 // mov rscratch1 #exception_blob_entry_point
2248 // br rscratch1
2249 // Note that the code buffer's insts_mark is always relative to insts.
2250 // That's why we must use the macroassembler to generate a handler.
2251 C2_MacroAssembler _masm(&cbuf);
2252 address base = __ start_a_stub(size_exception_handler());
2253 if (base == NULL) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257 int offset = __ offset();
2258 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2259 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2260 __ end_a_stub();
2261 return offset;
2262 }
2263
2264 // Emit deopt handler code.
2265 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2266 {
2267 // Note that the code buffer's insts_mark is always relative to insts.
2268 // That's why we must use the macroassembler to generate a handler.
2269 C2_MacroAssembler _masm(&cbuf);
2270 address base = __ start_a_stub(size_deopt_handler());
2271 if (base == NULL) {
2272 ciEnv::current()->record_failure("CodeCache is full");
2273 return 0; // CodeBuffer::expand failed
2274 }
2275 int offset = __ offset();
2276
2277 __ adr(lr, __ pc());
2278 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2279
2280 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2281 __ end_a_stub();
2282 return offset;
2283 }
2284
2285 // REQUIRED MATCHER CODE
2286
2287 //=============================================================================
2288
2289 const bool Matcher::match_rule_supported(int opcode) {
2290 if (!has_match_rule(opcode))
2291 return false;
2292
2293 bool ret_value = true;
2294 switch (opcode) {
2295 case Op_OnSpinWait:
2296 return VM_Version::supports_on_spin_wait();
2297 case Op_CacheWB:
2298 case Op_CacheWBPreSync:
2299 case Op_CacheWBPostSync:
2300 if (!VM_Version::supports_data_cache_line_flush()) {
2301 ret_value = false;
2302 }
2303 break;
2304 }
2305
2306 return ret_value; // Per default match rules are supported.
2307 }
2308
2309 const RegMask* Matcher::predicate_reg_mask(void) {
2310 return &_PR_REG_mask;
2311 }
2312
2313 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2314 return new TypeVectMask(elemTy, length);
2315 }
2316
2317 // Vector calling convention not yet implemented.
2318 const bool Matcher::supports_vector_calling_convention(void) {
2319 return false;
2320 }
2321
2322 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2323 Unimplemented();
2324 return OptoRegPair(0, 0);
2325 }
2326
2327 // Is this branch offset short enough that a short branch can be used?
2328 //
2329 // NOTE: If the platform does not provide any short branch variants, then
2330 // this method should return false for offset 0.
2331 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2332 // The passed offset is relative to address of the branch.
2333
2334 return (-32768 <= offset && offset < 32768);
2335 }
2336
2337 // Vector width in bytes.
2338 const int Matcher::vector_width_in_bytes(BasicType bt) {
2339 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2340 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2341 // Minimum 2 values in vector
2342 if (size < 2*type2aelembytes(bt)) size = 0;
2343 // But never < 4
2344 if (size < 4) size = 0;
2345 return size;
2346 }
2347
2348 // Limits on vector size (number of elements) loaded into vector.
2349 const int Matcher::max_vector_size(const BasicType bt) {
2350 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2351 }
2352
2353 const int Matcher::min_vector_size(const BasicType bt) {
2354 int max_size = max_vector_size(bt);
2355 // Limit the min vector size to 8 bytes.
2356 int size = 8 / type2aelembytes(bt);
2357 if (bt == T_BYTE) {
2358 // To support vector api shuffle/rearrange.
2359 size = 4;
2360 } else if (bt == T_BOOLEAN) {
2361 // To support vector api load/store mask.
2362 size = 2;
2363 }
2364 if (size < 2) size = 2;
2365 return MIN2(size, max_size);
2366 }
2367
2368 // Actual max scalable vector register length.
2369 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2370 return Matcher::max_vector_size(bt);
2371 }
2372
2373 // Vector ideal reg.
2374 const uint Matcher::vector_ideal_reg(int len) {
2375 if (UseSVE > 0 && 16 < len && len <= 256) {
2376 return Op_VecA;
2377 }
2378 switch(len) {
2379 // For 16-bit/32-bit mask vector, reuse VecD.
2380 case 2:
2381 case 4:
2382 case 8: return Op_VecD;
2383 case 16: return Op_VecX;
2384 }
2385 ShouldNotReachHere();
2386 return 0;
2387 }
2388
2389 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2390 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2391 switch (ideal_reg) {
2392 case Op_VecA: return new vecAOper();
2393 case Op_VecD: return new vecDOper();
2394 case Op_VecX: return new vecXOper();
2395 }
2396 ShouldNotReachHere();
2397 return NULL;
2398 }
2399
2400 bool Matcher::is_reg2reg_move(MachNode* m) {
2401 return false;
2402 }
2403
2404 bool Matcher::is_generic_vector(MachOper* opnd) {
2405 return opnd->opcode() == VREG;
2406 }
2407
2408 // Return whether or not this register is ever used as an argument.
2409 // This function is used on startup to build the trampoline stubs in
2410 // generateOptoStub. Registers not mentioned will be killed by the VM
2411 // call in the trampoline, and arguments in those registers not be
2412 // available to the callee.
2413 bool Matcher::can_be_java_arg(int reg)
2414 {
2415 return
2416 reg == R0_num || reg == R0_H_num ||
2417 reg == R1_num || reg == R1_H_num ||
2418 reg == R2_num || reg == R2_H_num ||
2419 reg == R3_num || reg == R3_H_num ||
2420 reg == R4_num || reg == R4_H_num ||
2421 reg == R5_num || reg == R5_H_num ||
2422 reg == R6_num || reg == R6_H_num ||
2423 reg == R7_num || reg == R7_H_num ||
2424 reg == V0_num || reg == V0_H_num ||
2425 reg == V1_num || reg == V1_H_num ||
2426 reg == V2_num || reg == V2_H_num ||
2427 reg == V3_num || reg == V3_H_num ||
2428 reg == V4_num || reg == V4_H_num ||
2429 reg == V5_num || reg == V5_H_num ||
2430 reg == V6_num || reg == V6_H_num ||
2431 reg == V7_num || reg == V7_H_num;
2432 }
2433
2434 bool Matcher::is_spillable_arg(int reg)
2435 {
2436 return can_be_java_arg(reg);
2437 }
2438
2439 uint Matcher::int_pressure_limit()
2440 {
2441 // JDK-8183543: When taking the number of available registers as int
2442 // register pressure threshold, the jtreg test:
2443 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2444 // failed due to C2 compilation failure with
2445 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2446 //
2447 // A derived pointer is live at CallNode and then is flagged by RA
2448 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2449 // derived pointers and lastly fail to spill after reaching maximum
2450 // number of iterations. Lowering the default pressure threshold to
2451 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2452 // a high register pressure area of the code so that split_DEF can
2453 // generate DefinitionSpillCopy for the derived pointer.
2454 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2455 if (!PreserveFramePointer) {
2456 // When PreserveFramePointer is off, frame pointer is allocatable,
2457 // but different from other SOC registers, it is excluded from
2458 // fatproj's mask because its save type is No-Save. Decrease 1 to
2459 // ensure high pressure at fatproj when PreserveFramePointer is off.
2460 // See check_pressure_at_fatproj().
2461 default_int_pressure_threshold--;
2462 }
2463 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2464 }
2465
2466 uint Matcher::float_pressure_limit()
2467 {
2468 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2469 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2470 }
2471
2472 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2473 return false;
2474 }
2475
2476 RegMask Matcher::divI_proj_mask() {
2477 ShouldNotReachHere();
2478 return RegMask();
2479 }
2480
2481 // Register for MODI projection of divmodI.
2482 RegMask Matcher::modI_proj_mask() {
2483 ShouldNotReachHere();
2484 return RegMask();
2485 }
2486
2487 // Register for DIVL projection of divmodL.
2488 RegMask Matcher::divL_proj_mask() {
2489 ShouldNotReachHere();
2490 return RegMask();
2491 }
2492
2493 // Register for MODL projection of divmodL.
2494 RegMask Matcher::modL_proj_mask() {
2495 ShouldNotReachHere();
2496 return RegMask();
2497 }
2498
2499 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2500 return FP_REG_mask();
2501 }
2502
2503 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2504 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2505 Node* u = addp->fast_out(i);
2506 if (u->is_LoadStore()) {
2507 // On AArch64, LoadStoreNodes (i.e. compare and swap
2508 // instructions) only take register indirect as an operand, so
2509 // any attempt to use an AddPNode as an input to a LoadStoreNode
2510 // must fail.
2511 return false;
2512 }
2513 if (u->is_Mem()) {
2514 int opsize = u->as_Mem()->memory_size();
2515 assert(opsize > 0, "unexpected memory operand size");
2516 if (u->as_Mem()->memory_size() != (1<<shift)) {
2517 return false;
2518 }
2519 }
2520 }
2521 return true;
2522 }
2523
2524 // Binary src (Replicate con)
2525 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2526 if (n == NULL || m == NULL) {
2527 return false;
2528 }
2529
2530 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2531 return false;
2532 }
2533
2534 Node* imm_node = m->in(1);
2535 if (!imm_node->is_Con()) {
2536 return false;
2537 }
2538
2539 const Type* t = imm_node->bottom_type();
2540 if (!(t->isa_int() || t->isa_long())) {
2541 return false;
2542 }
2543
2544 switch (n->Opcode()) {
2545 case Op_AndV:
2546 case Op_OrV:
2547 case Op_XorV: {
2548 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2549 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2550 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2551 }
2552 case Op_AddVB:
2553 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2554 case Op_AddVS:
2555 case Op_AddVI:
2556 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2557 case Op_AddVL:
2558 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2559 default:
2560 return false;
2561 }
2562 }
2563
2564 // (XorV src (Replicate m1))
2565 // (XorVMask src (MaskAll m1))
2566 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2567 if (n != NULL && m != NULL) {
2568 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2569 VectorNode::is_all_ones_vector(m);
2570 }
2571 return false;
2572 }
2573
2574 // Should the matcher clone input 'm' of node 'n'?
2575 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2576 if (is_vshift_con_pattern(n, m) ||
2577 is_vector_bitwise_not_pattern(n, m) ||
2578 is_valid_sve_arith_imm_pattern(n, m)) {
2579 mstack.push(m, Visit);
2580 return true;
2581 }
2582 return false;
2583 }
2584
2585 // Should the Matcher clone shifts on addressing modes, expecting them
2586 // to be subsumed into complex addressing expressions or compute them
2587 // into registers?
2588 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2589 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2590 return true;
2591 }
2592
2593 Node *off = m->in(AddPNode::Offset);
2594 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2595 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2596 // Are there other uses besides address expressions?
2597 !is_visited(off)) {
2598 address_visited.set(off->_idx); // Flag as address_visited
2599 mstack.push(off->in(2), Visit);
2600 Node *conv = off->in(1);
2601 if (conv->Opcode() == Op_ConvI2L &&
2602 // Are there other uses besides address expressions?
2603 !is_visited(conv)) {
2604 address_visited.set(conv->_idx); // Flag as address_visited
2605 mstack.push(conv->in(1), Pre_Visit);
2606 } else {
2607 mstack.push(conv, Pre_Visit);
2608 }
2609 address_visited.test_set(m->_idx); // Flag as address_visited
2610 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2611 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2612 return true;
2613 } else if (off->Opcode() == Op_ConvI2L &&
2614 // Are there other uses besides address expressions?
2615 !is_visited(off)) {
2616 address_visited.test_set(m->_idx); // Flag as address_visited
2617 address_visited.set(off->_idx); // Flag as address_visited
2618 mstack.push(off->in(1), Pre_Visit);
2619 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2620 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2621 return true;
2622 }
2623 return false;
2624 }
2625
2626 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2627 C2_MacroAssembler _masm(&cbuf); \
2628 { \
2629 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2630 guarantee(DISP == 0, "mode not permitted for volatile"); \
2631 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2632 __ INSN(REG, as_Register(BASE)); \
2633 }
2634
2635
2636 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2637 {
2638 Address::extend scale;
2639
2640 // Hooboy, this is fugly. We need a way to communicate to the
2641 // encoder that the index needs to be sign extended, so we have to
2642 // enumerate all the cases.
2643 switch (opcode) {
2644 case INDINDEXSCALEDI2L:
2645 case INDINDEXSCALEDI2LN:
2646 case INDINDEXI2L:
2647 case INDINDEXI2LN:
2648 scale = Address::sxtw(size);
2649 break;
2650 default:
2651 scale = Address::lsl(size);
2652 }
2653
2654 if (index == -1) {
2655 return Address(base, disp);
2656 } else {
2657 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2658 return Address(base, as_Register(index), scale);
2659 }
2660 }
2661
2662
2663 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2664 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2665 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2666 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2667 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2668
2669 // Used for all non-volatile memory accesses. The use of
2670 // $mem->opcode() to discover whether this pattern uses sign-extended
2671 // offsets is something of a kludge.
2672 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2673 Register reg, int opcode,
2674 Register base, int index, int scale, int disp,
2675 int size_in_memory)
2676 {
2677 Address addr = mem2address(opcode, base, index, scale, disp);
2678 if (addr.getMode() == Address::base_plus_offset) {
2679 /* If we get an out-of-range offset it is a bug in the compiler,
2680 so we assert here. */
2681 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2682 "c2 compiler bug");
2683 /* Fix up any out-of-range offsets. */
2684 assert_different_registers(rscratch1, base);
2685 assert_different_registers(rscratch1, reg);
2686 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2687 }
2688 (masm.*insn)(reg, addr);
2689 }
2690
2691 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2692 FloatRegister reg, int opcode,
2693 Register base, int index, int size, int disp,
2694 int size_in_memory)
2695 {
2696 Address::extend scale;
2697
2698 switch (opcode) {
2699 case INDINDEXSCALEDI2L:
2700 case INDINDEXSCALEDI2LN:
2701 scale = Address::sxtw(size);
2702 break;
2703 default:
2704 scale = Address::lsl(size);
2705 }
2706
2707 if (index == -1) {
2708 /* If we get an out-of-range offset it is a bug in the compiler,
2709 so we assert here. */
2710 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2711 /* Fix up any out-of-range offsets. */
2712 assert_different_registers(rscratch1, base);
2713 Address addr = Address(base, disp);
2714 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2715 (masm.*insn)(reg, addr);
2716 } else {
2717 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2718 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2719 }
2720 }
2721
2722 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2723 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2724 int opcode, Register base, int index, int size, int disp)
2725 {
2726 if (index == -1) {
2727 (masm.*insn)(reg, T, Address(base, disp));
2728 } else {
2729 assert(disp == 0, "unsupported address mode");
2730 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2731 }
2732 }
2733
2734 %}
2735
2736
2737
2738 //----------ENCODING BLOCK-----------------------------------------------------
2739 // This block specifies the encoding classes used by the compiler to
2740 // output byte streams. Encoding classes are parameterized macros
2741 // used by Machine Instruction Nodes in order to generate the bit
2742 // encoding of the instruction. Operands specify their base encoding
2743 // interface with the interface keyword. There are currently
2744 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2745 // COND_INTER. REG_INTER causes an operand to generate a function
2746 // which returns its register number when queried. CONST_INTER causes
2747 // an operand to generate a function which returns the value of the
2748 // constant when queried. MEMORY_INTER causes an operand to generate
2749 // four functions which return the Base Register, the Index Register,
2750 // the Scale Value, and the Offset Value of the operand when queried.
2751 // COND_INTER causes an operand to generate six functions which return
2752 // the encoding code (ie - encoding bits for the instruction)
2753 // associated with each basic boolean condition for a conditional
2754 // instruction.
2755 //
2756 // Instructions specify two basic values for encoding. Again, a
2757 // function is available to check if the constant displacement is an
2758 // oop. They use the ins_encode keyword to specify their encoding
2759 // classes (which must be a sequence of enc_class names, and their
2760 // parameters, specified in the encoding block), and they use the
2761 // opcode keyword to specify, in order, their primary, secondary, and
2762 // tertiary opcode. Only the opcode sections which a particular
2763 // instruction needs for encoding need to be specified.
2764 encode %{
2765 // Build emit functions for each basic byte or larger field in the
2766 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2767 // from C++ code in the enc_class source block. Emit functions will
2768 // live in the main source block for now. In future, we can
2769 // generalize this by adding a syntax that specifies the sizes of
2770 // fields in an order, so that the adlc can build the emit functions
2771 // automagically
2772
2773 // catch all for unimplemented encodings
2774 enc_class enc_unimplemented %{
2775 C2_MacroAssembler _masm(&cbuf);
2776 __ unimplemented("C2 catch all");
2777 %}
2778
2779 // BEGIN Non-volatile memory access
2780
2781 // This encoding class is generated automatically from ad_encode.m4.
2782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2783 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2784 Register dst_reg = as_Register($dst$$reg);
2785 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2786 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2787 %}
2788
2789 // This encoding class is generated automatically from ad_encode.m4.
2790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2791 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2792 Register dst_reg = as_Register($dst$$reg);
2793 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2794 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2795 %}
2796
2797 // This encoding class is generated automatically from ad_encode.m4.
2798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2799 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2800 Register dst_reg = as_Register($dst$$reg);
2801 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2802 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2803 %}
2804
2805 // This encoding class is generated automatically from ad_encode.m4.
2806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2807 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2808 Register dst_reg = as_Register($dst$$reg);
2809 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2810 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2811 %}
2812
2813 // This encoding class is generated automatically from ad_encode.m4.
2814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2815 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2816 Register dst_reg = as_Register($dst$$reg);
2817 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2818 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2819 %}
2820
2821 // This encoding class is generated automatically from ad_encode.m4.
2822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2823 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2824 Register dst_reg = as_Register($dst$$reg);
2825 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2826 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2827 %}
2828
2829 // This encoding class is generated automatically from ad_encode.m4.
2830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2831 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2832 Register dst_reg = as_Register($dst$$reg);
2833 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2834 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2835 %}
2836
2837 // This encoding class is generated automatically from ad_encode.m4.
2838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2839 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2840 Register dst_reg = as_Register($dst$$reg);
2841 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2842 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2843 %}
2844
2845 // This encoding class is generated automatically from ad_encode.m4.
2846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2847 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2848 Register dst_reg = as_Register($dst$$reg);
2849 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2850 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2851 %}
2852
2853 // This encoding class is generated automatically from ad_encode.m4.
2854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2855 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2856 Register dst_reg = as_Register($dst$$reg);
2857 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2858 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2859 %}
2860
2861 // This encoding class is generated automatically from ad_encode.m4.
2862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2863 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2864 Register dst_reg = as_Register($dst$$reg);
2865 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2866 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2867 %}
2868
2869 // This encoding class is generated automatically from ad_encode.m4.
2870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2871 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2872 Register dst_reg = as_Register($dst$$reg);
2873 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2874 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2875 %}
2876
2877 // This encoding class is generated automatically from ad_encode.m4.
2878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2879 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2880 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2881 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2883 %}
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2888 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2889 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2896 Register src_reg = as_Register($src$$reg);
2897 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_strb0(memory1 mem) %{
2904 C2_MacroAssembler _masm(&cbuf);
2905 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2912 Register src_reg = as_Register($src$$reg);
2913 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_strh0(memory2 mem) %{
2920 C2_MacroAssembler _masm(&cbuf);
2921 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2928 Register src_reg = as_Register($src$$reg);
2929 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_strw0(memory4 mem) %{
2936 C2_MacroAssembler _masm(&cbuf);
2937 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2944 Register src_reg = as_Register($src$$reg);
2945 // we sometimes get asked to store the stack pointer into the
2946 // current thread -- we cannot do that directly on AArch64
2947 if (src_reg == r31_sp) {
2948 C2_MacroAssembler _masm(&cbuf);
2949 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2950 __ mov(rscratch2, sp);
2951 src_reg = rscratch2;
2952 }
2953 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_str0(memory8 mem) %{
2960 C2_MacroAssembler _masm(&cbuf);
2961 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2968 FloatRegister src_reg = as_FloatRegister($src$$reg);
2969 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2976 FloatRegister src_reg = as_FloatRegister($src$$reg);
2977 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2984 C2_MacroAssembler _masm(&cbuf);
2985 __ membar(Assembler::StoreStore);
2986 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2988 %}
2989
2990 // END Non-volatile memory access
2991
2992 // Vector loads and stores
2993 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
2994 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2995 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
2996 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2997 %}
2998
2999 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3000 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3001 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3002 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3003 %}
3004
3005 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3006 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3007 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3008 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3009 %}
3010
3011 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3012 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3013 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3014 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3015 %}
3016
3017 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3018 FloatRegister src_reg = as_FloatRegister($src$$reg);
3019 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3020 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3021 %}
3022
3023 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3024 FloatRegister src_reg = as_FloatRegister($src$$reg);
3025 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3026 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3027 %}
3028
3029 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3030 FloatRegister src_reg = as_FloatRegister($src$$reg);
3031 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3032 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3033 %}
3034
3035 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3036 FloatRegister src_reg = as_FloatRegister($src$$reg);
3037 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3038 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3039 %}
3040
3041 // volatile loads and stores
3042
3043 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3044 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3045 rscratch1, stlrb);
3046 %}
3047
3048 enc_class aarch64_enc_stlrb0(memory mem) %{
3049 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3050 rscratch1, stlrb);
3051 %}
3052
3053 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3054 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3055 rscratch1, stlrh);
3056 %}
3057
3058 enc_class aarch64_enc_stlrh0(memory mem) %{
3059 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3060 rscratch1, stlrh);
3061 %}
3062
3063 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3064 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3065 rscratch1, stlrw);
3066 %}
3067
3068 enc_class aarch64_enc_stlrw0(memory mem) %{
3069 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3070 rscratch1, stlrw);
3071 %}
3072
3073 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3074 Register dst_reg = as_Register($dst$$reg);
3075 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3076 rscratch1, ldarb);
3077 __ sxtbw(dst_reg, dst_reg);
3078 %}
3079
3080 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3081 Register dst_reg = as_Register($dst$$reg);
3082 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3083 rscratch1, ldarb);
3084 __ sxtb(dst_reg, dst_reg);
3085 %}
3086
3087 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3088 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3089 rscratch1, ldarb);
3090 %}
3091
3092 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3093 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3094 rscratch1, ldarb);
3095 %}
3096
3097 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3098 Register dst_reg = as_Register($dst$$reg);
3099 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3100 rscratch1, ldarh);
3101 __ sxthw(dst_reg, dst_reg);
3102 %}
3103
3104 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3105 Register dst_reg = as_Register($dst$$reg);
3106 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3107 rscratch1, ldarh);
3108 __ sxth(dst_reg, dst_reg);
3109 %}
3110
3111 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3112 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, ldarh);
3114 %}
3115
3116 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3117 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, ldarh);
3119 %}
3120
3121 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3122 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, ldarw);
3124 %}
3125
3126 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3127 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3128 rscratch1, ldarw);
3129 %}
3130
3131 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3132 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, ldar);
3134 %}
3135
3136 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3137 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, ldarw);
3139 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3140 %}
3141
3142 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3143 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, ldar);
3145 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3146 %}
3147
3148 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3149 Register src_reg = as_Register($src$$reg);
3150 // we sometimes get asked to store the stack pointer into the
3151 // current thread -- we cannot do that directly on AArch64
3152 if (src_reg == r31_sp) {
3153 C2_MacroAssembler _masm(&cbuf);
3154 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3155 __ mov(rscratch2, sp);
3156 src_reg = rscratch2;
3157 }
3158 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlr);
3160 %}
3161
3162 enc_class aarch64_enc_stlr0(memory mem) %{
3163 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlr);
3165 %}
3166
3167 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3168 {
3169 C2_MacroAssembler _masm(&cbuf);
3170 FloatRegister src_reg = as_FloatRegister($src$$reg);
3171 __ fmovs(rscratch2, src_reg);
3172 }
3173 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, stlrw);
3175 %}
3176
3177 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3178 {
3179 C2_MacroAssembler _masm(&cbuf);
3180 FloatRegister src_reg = as_FloatRegister($src$$reg);
3181 __ fmovd(rscratch2, src_reg);
3182 }
3183 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, stlr);
3185 %}
3186
3187 // synchronized read/update encodings
3188
3189 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3190 C2_MacroAssembler _masm(&cbuf);
3191 Register dst_reg = as_Register($dst$$reg);
3192 Register base = as_Register($mem$$base);
3193 int index = $mem$$index;
3194 int scale = $mem$$scale;
3195 int disp = $mem$$disp;
3196 if (index == -1) {
3197 if (disp != 0) {
3198 __ lea(rscratch1, Address(base, disp));
3199 __ ldaxr(dst_reg, rscratch1);
3200 } else {
3201 // TODO
3202 // should we ever get anything other than this case?
3203 __ ldaxr(dst_reg, base);
3204 }
3205 } else {
3206 Register index_reg = as_Register(index);
3207 if (disp == 0) {
3208 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3209 __ ldaxr(dst_reg, rscratch1);
3210 } else {
3211 __ lea(rscratch1, Address(base, disp));
3212 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3213 __ ldaxr(dst_reg, rscratch1);
3214 }
3215 }
3216 %}
3217
3218 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3219 C2_MacroAssembler _masm(&cbuf);
3220 Register src_reg = as_Register($src$$reg);
3221 Register base = as_Register($mem$$base);
3222 int index = $mem$$index;
3223 int scale = $mem$$scale;
3224 int disp = $mem$$disp;
3225 if (index == -1) {
3226 if (disp != 0) {
3227 __ lea(rscratch2, Address(base, disp));
3228 __ stlxr(rscratch1, src_reg, rscratch2);
3229 } else {
3230 // TODO
3231 // should we ever get anything other than this case?
3232 __ stlxr(rscratch1, src_reg, base);
3233 }
3234 } else {
3235 Register index_reg = as_Register(index);
3236 if (disp == 0) {
3237 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3238 __ stlxr(rscratch1, src_reg, rscratch2);
3239 } else {
3240 __ lea(rscratch2, Address(base, disp));
3241 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3242 __ stlxr(rscratch1, src_reg, rscratch2);
3243 }
3244 }
3245 __ cmpw(rscratch1, zr);
3246 %}
3247
3248 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3249 C2_MacroAssembler _masm(&cbuf);
3250 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3251 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3252 Assembler::xword, /*acquire*/ false, /*release*/ true,
3253 /*weak*/ false, noreg);
3254 %}
3255
3256 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3257 C2_MacroAssembler _masm(&cbuf);
3258 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3259 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3260 Assembler::word, /*acquire*/ false, /*release*/ true,
3261 /*weak*/ false, noreg);
3262 %}
3263
3264 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3265 C2_MacroAssembler _masm(&cbuf);
3266 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3267 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3268 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3269 /*weak*/ false, noreg);
3270 %}
3271
3272 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3273 C2_MacroAssembler _masm(&cbuf);
3274 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3275 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3276 Assembler::byte, /*acquire*/ false, /*release*/ true,
3277 /*weak*/ false, noreg);
3278 %}
3279
3280
3281 // The only difference between aarch64_enc_cmpxchg and
3282 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3283 // CompareAndSwap sequence to serve as a barrier on acquiring a
3284 // lock.
3285 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3286 C2_MacroAssembler _masm(&cbuf);
3287 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3288 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3289 Assembler::xword, /*acquire*/ true, /*release*/ true,
3290 /*weak*/ false, noreg);
3291 %}
3292
3293 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3294 C2_MacroAssembler _masm(&cbuf);
3295 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3296 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3297 Assembler::word, /*acquire*/ true, /*release*/ true,
3298 /*weak*/ false, noreg);
3299 %}
3300
3301 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3302 C2_MacroAssembler _masm(&cbuf);
3303 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3304 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3305 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3306 /*weak*/ false, noreg);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3310 C2_MacroAssembler _masm(&cbuf);
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::byte, /*acquire*/ true, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 // auxiliary used for CompareAndSwapX to set result register
3318 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3319 C2_MacroAssembler _masm(&cbuf);
3320 Register res_reg = as_Register($res$$reg);
3321 __ cset(res_reg, Assembler::EQ);
3322 %}
3323
3324 // prefetch encodings
3325
3326 enc_class aarch64_enc_prefetchw(memory mem) %{
3327 C2_MacroAssembler _masm(&cbuf);
3328 Register base = as_Register($mem$$base);
3329 int index = $mem$$index;
3330 int scale = $mem$$scale;
3331 int disp = $mem$$disp;
3332 if (index == -1) {
3333 __ prfm(Address(base, disp), PSTL1KEEP);
3334 } else {
3335 Register index_reg = as_Register(index);
3336 if (disp == 0) {
3337 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3338 } else {
3339 __ lea(rscratch1, Address(base, disp));
3340 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3341 }
3342 }
3343 %}
3344
3345 /// mov envcodings
3346
3347 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3348 C2_MacroAssembler _masm(&cbuf);
3349 uint32_t con = (uint32_t)$src$$constant;
3350 Register dst_reg = as_Register($dst$$reg);
3351 if (con == 0) {
3352 __ movw(dst_reg, zr);
3353 } else {
3354 __ movw(dst_reg, con);
3355 }
3356 %}
3357
3358 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3359 C2_MacroAssembler _masm(&cbuf);
3360 Register dst_reg = as_Register($dst$$reg);
3361 uint64_t con = (uint64_t)$src$$constant;
3362 if (con == 0) {
3363 __ mov(dst_reg, zr);
3364 } else {
3365 __ mov(dst_reg, con);
3366 }
3367 %}
3368
3369 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3370 C2_MacroAssembler _masm(&cbuf);
3371 Register dst_reg = as_Register($dst$$reg);
3372 address con = (address)$src$$constant;
3373 if (con == NULL || con == (address)1) {
3374 ShouldNotReachHere();
3375 } else {
3376 relocInfo::relocType rtype = $src->constant_reloc();
3377 if (rtype == relocInfo::oop_type) {
3378 __ movoop(dst_reg, (jobject)con);
3379 } else if (rtype == relocInfo::metadata_type) {
3380 __ mov_metadata(dst_reg, (Metadata*)con);
3381 } else {
3382 assert(rtype == relocInfo::none, "unexpected reloc type");
3383 if (! __ is_valid_AArch64_address(con) ||
3384 con < (address)(uintptr_t)os::vm_page_size()) {
3385 __ mov(dst_reg, con);
3386 } else {
3387 uint64_t offset;
3388 __ adrp(dst_reg, con, offset);
3389 __ add(dst_reg, dst_reg, offset);
3390 }
3391 }
3392 }
3393 %}
3394
3395 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3396 C2_MacroAssembler _masm(&cbuf);
3397 Register dst_reg = as_Register($dst$$reg);
3398 __ mov(dst_reg, zr);
3399 %}
3400
3401 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3402 C2_MacroAssembler _masm(&cbuf);
3403 Register dst_reg = as_Register($dst$$reg);
3404 __ mov(dst_reg, (uint64_t)1);
3405 %}
3406
3407 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3408 C2_MacroAssembler _masm(&cbuf);
3409 __ load_byte_map_base($dst$$Register);
3410 %}
3411
3412 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3413 C2_MacroAssembler _masm(&cbuf);
3414 Register dst_reg = as_Register($dst$$reg);
3415 address con = (address)$src$$constant;
3416 if (con == NULL) {
3417 ShouldNotReachHere();
3418 } else {
3419 relocInfo::relocType rtype = $src->constant_reloc();
3420 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3421 __ set_narrow_oop(dst_reg, (jobject)con);
3422 }
3423 %}
3424
3425 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3426 C2_MacroAssembler _masm(&cbuf);
3427 Register dst_reg = as_Register($dst$$reg);
3428 __ mov(dst_reg, zr);
3429 %}
3430
3431 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3432 C2_MacroAssembler _masm(&cbuf);
3433 Register dst_reg = as_Register($dst$$reg);
3434 address con = (address)$src$$constant;
3435 if (con == NULL) {
3436 ShouldNotReachHere();
3437 } else {
3438 relocInfo::relocType rtype = $src->constant_reloc();
3439 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3440 __ set_narrow_klass(dst_reg, (Klass *)con);
3441 }
3442 %}
3443
3444 // arithmetic encodings
3445
3446 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3447 C2_MacroAssembler _masm(&cbuf);
3448 Register dst_reg = as_Register($dst$$reg);
3449 Register src_reg = as_Register($src1$$reg);
3450 int32_t con = (int32_t)$src2$$constant;
3451 // add has primary == 0, subtract has primary == 1
3452 if ($primary) { con = -con; }
3453 if (con < 0) {
3454 __ subw(dst_reg, src_reg, -con);
3455 } else {
3456 __ addw(dst_reg, src_reg, con);
3457 }
3458 %}
3459
3460 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3461 C2_MacroAssembler _masm(&cbuf);
3462 Register dst_reg = as_Register($dst$$reg);
3463 Register src_reg = as_Register($src1$$reg);
3464 int32_t con = (int32_t)$src2$$constant;
3465 // add has primary == 0, subtract has primary == 1
3466 if ($primary) { con = -con; }
3467 if (con < 0) {
3468 __ sub(dst_reg, src_reg, -con);
3469 } else {
3470 __ add(dst_reg, src_reg, con);
3471 }
3472 %}
3473
3474 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3475 C2_MacroAssembler _masm(&cbuf);
3476 Register dst_reg = as_Register($dst$$reg);
3477 Register src1_reg = as_Register($src1$$reg);
3478 Register src2_reg = as_Register($src2$$reg);
3479 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3480 %}
3481
3482 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3483 C2_MacroAssembler _masm(&cbuf);
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src1_reg = as_Register($src1$$reg);
3486 Register src2_reg = as_Register($src2$$reg);
3487 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3488 %}
3489
3490 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3491 C2_MacroAssembler _masm(&cbuf);
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3496 %}
3497
3498 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3499 C2_MacroAssembler _masm(&cbuf);
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src1_reg = as_Register($src1$$reg);
3502 Register src2_reg = as_Register($src2$$reg);
3503 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3504 %}
3505
3506 // compare instruction encodings
3507
3508 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3509 C2_MacroAssembler _masm(&cbuf);
3510 Register reg1 = as_Register($src1$$reg);
3511 Register reg2 = as_Register($src2$$reg);
3512 __ cmpw(reg1, reg2);
3513 %}
3514
3515 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3516 C2_MacroAssembler _masm(&cbuf);
3517 Register reg = as_Register($src1$$reg);
3518 int32_t val = $src2$$constant;
3519 if (val >= 0) {
3520 __ subsw(zr, reg, val);
3521 } else {
3522 __ addsw(zr, reg, -val);
3523 }
3524 %}
3525
3526 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3527 C2_MacroAssembler _masm(&cbuf);
3528 Register reg1 = as_Register($src1$$reg);
3529 uint32_t val = (uint32_t)$src2$$constant;
3530 __ movw(rscratch1, val);
3531 __ cmpw(reg1, rscratch1);
3532 %}
3533
3534 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3535 C2_MacroAssembler _masm(&cbuf);
3536 Register reg1 = as_Register($src1$$reg);
3537 Register reg2 = as_Register($src2$$reg);
3538 __ cmp(reg1, reg2);
3539 %}
3540
3541 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3542 C2_MacroAssembler _masm(&cbuf);
3543 Register reg = as_Register($src1$$reg);
3544 int64_t val = $src2$$constant;
3545 if (val >= 0) {
3546 __ subs(zr, reg, val);
3547 } else if (val != -val) {
3548 __ adds(zr, reg, -val);
3549 } else {
3550 // aargh, Long.MIN_VALUE is a special case
3551 __ orr(rscratch1, zr, (uint64_t)val);
3552 __ subs(zr, reg, rscratch1);
3553 }
3554 %}
3555
3556 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3557 C2_MacroAssembler _masm(&cbuf);
3558 Register reg1 = as_Register($src1$$reg);
3559 uint64_t val = (uint64_t)$src2$$constant;
3560 __ mov(rscratch1, val);
3561 __ cmp(reg1, rscratch1);
3562 %}
3563
3564 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3565 C2_MacroAssembler _masm(&cbuf);
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmp(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3572 C2_MacroAssembler _masm(&cbuf);
3573 Register reg1 = as_Register($src1$$reg);
3574 Register reg2 = as_Register($src2$$reg);
3575 __ cmpw(reg1, reg2);
3576 %}
3577
3578 enc_class aarch64_enc_testp(iRegP src) %{
3579 C2_MacroAssembler _masm(&cbuf);
3580 Register reg = as_Register($src$$reg);
3581 __ cmp(reg, zr);
3582 %}
3583
3584 enc_class aarch64_enc_testn(iRegN src) %{
3585 C2_MacroAssembler _masm(&cbuf);
3586 Register reg = as_Register($src$$reg);
3587 __ cmpw(reg, zr);
3588 %}
3589
3590 enc_class aarch64_enc_b(label lbl) %{
3591 C2_MacroAssembler _masm(&cbuf);
3592 Label *L = $lbl$$label;
3593 __ b(*L);
3594 %}
3595
3596 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3597 C2_MacroAssembler _masm(&cbuf);
3598 Label *L = $lbl$$label;
3599 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3600 %}
3601
3602 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3603 C2_MacroAssembler _masm(&cbuf);
3604 Label *L = $lbl$$label;
3605 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3606 %}
3607
3608 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3609 %{
3610 Register sub_reg = as_Register($sub$$reg);
3611 Register super_reg = as_Register($super$$reg);
3612 Register temp_reg = as_Register($temp$$reg);
3613 Register result_reg = as_Register($result$$reg);
3614
3615 Label miss;
3616 C2_MacroAssembler _masm(&cbuf);
3617 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3618 NULL, &miss,
3619 /*set_cond_codes:*/ true);
3620 if ($primary) {
3621 __ mov(result_reg, zr);
3622 }
3623 __ bind(miss);
3624 %}
3625
3626 enc_class aarch64_enc_java_static_call(method meth) %{
3627 C2_MacroAssembler _masm(&cbuf);
3628
3629 address addr = (address)$meth$$method;
3630 address call;
3631 if (!_method) {
3632 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3633 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3634 if (call == NULL) {
3635 ciEnv::current()->record_failure("CodeCache is full");
3636 return;
3637 }
3638 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3639 // The NOP here is purely to ensure that eliding a call to
3640 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3641 __ nop();
3642 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3643 } else {
3644 int method_index = resolved_method_index(cbuf);
3645 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3646 : static_call_Relocation::spec(method_index);
3647 call = __ trampoline_call(Address(addr, rspec));
3648 if (call == NULL) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3653 // Calls of the same statically bound method can share
3654 // a stub to the interpreter.
3655 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3656 } else {
3657 // Emit stub for static call
3658 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3659 if (stub == NULL) {
3660 ciEnv::current()->record_failure("CodeCache is full");
3661 return;
3662 }
3663 }
3664 }
3665
3666 __ post_call_nop();
3667
3668 // Only non uncommon_trap calls need to reinitialize ptrue.
3669 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3670 __ reinitialize_ptrue();
3671 }
3672 %}
3673
3674 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3675 C2_MacroAssembler _masm(&cbuf);
3676 int method_index = resolved_method_index(cbuf);
3677 address call = __ ic_call((address)$meth$$method, method_index);
3678 if (call == NULL) {
3679 ciEnv::current()->record_failure("CodeCache is full");
3680 return;
3681 }
3682 __ post_call_nop();
3683 if (Compile::current()->max_vector_size() > 0) {
3684 __ reinitialize_ptrue();
3685 }
3686 %}
3687
3688 enc_class aarch64_enc_call_epilog() %{
3689 C2_MacroAssembler _masm(&cbuf);
3690 if (VerifyStackAtCalls) {
3691 // Check that stack depth is unchanged: find majik cookie on stack
3692 __ call_Unimplemented();
3693 }
3694 %}
3695
3696 enc_class aarch64_enc_java_to_runtime(method meth) %{
3697 C2_MacroAssembler _masm(&cbuf);
3698
3699 // some calls to generated routines (arraycopy code) are scheduled
3700 // by C2 as runtime calls. if so we can call them using a br (they
3701 // will be in a reachable segment) otherwise we have to use a blr
3702 // which loads the absolute address into a register.
3703 address entry = (address)$meth$$method;
3704 CodeBlob *cb = CodeCache::find_blob(entry);
3705 if (cb) {
3706 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3707 if (call == NULL) {
3708 ciEnv::current()->record_failure("CodeCache is full");
3709 return;
3710 }
3711 __ post_call_nop();
3712 } else {
3713 Label retaddr;
3714 __ adr(rscratch2, retaddr);
3715 __ lea(rscratch1, RuntimeAddress(entry));
3716 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3717 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3718 __ blr(rscratch1);
3719 __ bind(retaddr);
3720 __ post_call_nop();
3721 __ add(sp, sp, 2 * wordSize);
3722 }
3723 if (Compile::current()->max_vector_size() > 0) {
3724 __ reinitialize_ptrue();
3725 }
3726 %}
3727
3728 enc_class aarch64_enc_rethrow() %{
3729 C2_MacroAssembler _masm(&cbuf);
3730 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3731 %}
3732
3733 enc_class aarch64_enc_ret() %{
3734 C2_MacroAssembler _masm(&cbuf);
3735 #ifdef ASSERT
3736 if (Compile::current()->max_vector_size() > 0) {
3737 __ verify_ptrue();
3738 }
3739 #endif
3740 __ ret(lr);
3741 %}
3742
3743 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3744 C2_MacroAssembler _masm(&cbuf);
3745 Register target_reg = as_Register($jump_target$$reg);
3746 __ br(target_reg);
3747 %}
3748
3749 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3750 C2_MacroAssembler _masm(&cbuf);
3751 Register target_reg = as_Register($jump_target$$reg);
3752 // exception oop should be in r0
3753 // ret addr has been popped into lr
3754 // callee expects it in r3
3755 __ mov(r3, lr);
3756 __ br(target_reg);
3757 %}
3758
3759 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3760 C2_MacroAssembler _masm(&cbuf);
3761 Register oop = as_Register($object$$reg);
3762 Register box = as_Register($box$$reg);
3763 Register disp_hdr = as_Register($tmp$$reg);
3764 Register tmp = as_Register($tmp2$$reg);
3765 Label cont;
3766 Label object_has_monitor;
3767 Label no_count;
3768
3769 assert_different_registers(oop, box, tmp, disp_hdr);
3770
3771 // Load markWord from object into displaced_header.
3772 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3773
3774 if (DiagnoseSyncOnValueBasedClasses != 0) {
3775 __ load_klass(tmp, oop);
3776 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3777 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3778 __ br(Assembler::NE, cont);
3779 }
3780
3781 // Check for existing monitor
3782 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3783
3784 if (!UseHeavyMonitors) {
3785 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3786 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3787
3788 // Initialize the box. (Must happen before we update the object mark!)
3789 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3790
3791 // Compare object markWord with an unlocked value (tmp) and if
3792 // equal exchange the stack address of our box with object markWord.
3793 // On failure disp_hdr contains the possibly locked markWord.
3794 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3795 /*release*/ true, /*weak*/ false, disp_hdr);
3796 __ br(Assembler::EQ, cont);
3797
3798 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3799
3800 // If the compare-and-exchange succeeded, then we found an unlocked
3801 // object, will have now locked it will continue at label cont
3802
3803 // Check if the owner is self by comparing the value in the
3804 // markWord of object (disp_hdr) with the stack pointer.
3805 __ mov(rscratch1, sp);
3806 __ sub(disp_hdr, disp_hdr, rscratch1);
3807 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3808 // If condition is true we are cont and hence we can store 0 as the
3809 // displaced header in the box, which indicates that it is a recursive lock.
3810 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3811 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3812 } else {
3813 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3814 }
3815 __ b(cont);
3816
3817 // Handle existing monitor.
3818 __ bind(object_has_monitor);
3819
3820 // The object's monitor m is unlocked iff m->owner == NULL,
3821 // otherwise m->owner may contain a thread or a stack address.
3822 //
3823 // Try to CAS m->owner from NULL to current thread.
3824 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3825 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3826 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3827
3828 // Store a non-null value into the box to avoid looking like a re-entrant
3829 // lock. The fast-path monitor unlock code checks for
3830 // markWord::monitor_value so use markWord::unused_mark which has the
3831 // relevant bit set, and also matches ObjectSynchronizer::enter.
3832 __ mov(tmp, (address)markWord::unused_mark().value());
3833 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3834
3835 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3836
3837 __ cmp(rscratch1, rthread);
3838 __ br(Assembler::NE, cont); // Check for recursive locking
3839
3840 // Recursive lock case
3841 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3842 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3843
3844 __ bind(cont);
3845 // flag == EQ indicates success
3846 // flag == NE indicates failure
3847 __ br(Assembler::NE, no_count);
3848
3849 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3850
3851 __ bind(no_count);
3852 %}
3853
3854 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3855 C2_MacroAssembler _masm(&cbuf);
3856 Register oop = as_Register($object$$reg);
3857 Register box = as_Register($box$$reg);
3858 Register disp_hdr = as_Register($tmp$$reg);
3859 Register tmp = as_Register($tmp2$$reg);
3860 Label cont;
3861 Label object_has_monitor;
3862 Label no_count;
3863
3864 assert_different_registers(oop, box, tmp, disp_hdr);
3865
3866 if (!UseHeavyMonitors) {
3867 // Find the lock address and load the displaced header from the stack.
3868 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3869
3870 // If the displaced header is 0, we have a recursive unlock.
3871 __ cmp(disp_hdr, zr);
3872 __ br(Assembler::EQ, cont);
3873 }
3874
3875 // Handle existing monitor.
3876 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3877 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3878
3879 if (!UseHeavyMonitors) {
3880 // Check if it is still a light weight lock, this is is true if we
3881 // see the stack address of the basicLock in the markWord of the
3882 // object.
3883
3884 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3885 /*release*/ true, /*weak*/ false, tmp);
3886 } else {
3887 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3888 }
3889 __ b(cont);
3890
3891 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3892
3893 // Handle existing monitor.
3894 __ bind(object_has_monitor);
3895 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3896 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3897 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3898
3899 Label notRecursive;
3900 __ cbz(disp_hdr, notRecursive);
3901
3902 // Recursive lock
3903 __ sub(disp_hdr, disp_hdr, 1u);
3904 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3905 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3906 __ b(cont);
3907
3908 __ bind(notRecursive);
3909 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3910 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3911 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3912 __ cmp(rscratch1, zr); // Sets flags for result
3913 __ cbnz(rscratch1, cont);
3914 // need a release store here
3915 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3916 __ stlr(zr, tmp); // set unowned
3917
3918 __ bind(cont);
3919 // flag == EQ indicates success
3920 // flag == NE indicates failure
3921 __ br(Assembler::NE, no_count);
3922
3923 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
3924
3925 __ bind(no_count);
3926 %}
3927
3928 %}
3929
3930 //----------FRAME--------------------------------------------------------------
3931 // Definition of frame structure and management information.
3932 //
3933 // S T A C K L A Y O U T Allocators stack-slot number
3934 // | (to get allocators register number
3935 // G Owned by | | v add OptoReg::stack0())
3936 // r CALLER | |
3937 // o | +--------+ pad to even-align allocators stack-slot
3938 // w V | pad0 | numbers; owned by CALLER
3939 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3940 // h ^ | in | 5
3941 // | | args | 4 Holes in incoming args owned by SELF
3942 // | | | | 3
3943 // | | +--------+
3944 // V | | old out| Empty on Intel, window on Sparc
3945 // | old |preserve| Must be even aligned.
3946 // | SP-+--------+----> Matcher::_old_SP, even aligned
3947 // | | in | 3 area for Intel ret address
3948 // Owned by |preserve| Empty on Sparc.
3949 // SELF +--------+
3950 // | | pad2 | 2 pad to align old SP
3951 // | +--------+ 1
3952 // | | locks | 0
3953 // | +--------+----> OptoReg::stack0(), even aligned
3954 // | | pad1 | 11 pad to align new SP
3955 // | +--------+
3956 // | | | 10
3957 // | | spills | 9 spills
3958 // V | | 8 (pad0 slot for callee)
3959 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3960 // ^ | out | 7
3961 // | | args | 6 Holes in outgoing args owned by CALLEE
3962 // Owned by +--------+
3963 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3964 // | new |preserve| Must be even-aligned.
3965 // | SP-+--------+----> Matcher::_new_SP, even aligned
3966 // | | |
3967 //
3968 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3969 // known from SELF's arguments and the Java calling convention.
3970 // Region 6-7 is determined per call site.
3971 // Note 2: If the calling convention leaves holes in the incoming argument
3972 // area, those holes are owned by SELF. Holes in the outgoing area
3973 // are owned by the CALLEE. Holes should not be necessary in the
3974 // incoming area, as the Java calling convention is completely under
3975 // the control of the AD file. Doubles can be sorted and packed to
3976 // avoid holes. Holes in the outgoing arguments may be necessary for
3977 // varargs C calling conventions.
3978 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3979 // even aligned with pad0 as needed.
3980 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3981 // (the latter is true on Intel but is it false on AArch64?)
3982 // region 6-11 is even aligned; it may be padded out more so that
3983 // the region from SP to FP meets the minimum stack alignment.
3984 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3985 // alignment. Region 11, pad1, may be dynamically extended so that
3986 // SP meets the minimum alignment.
3987
3988 frame %{
3989 // These three registers define part of the calling convention
3990 // between compiled code and the interpreter.
3991
3992 // Inline Cache Register or Method for I2C.
3993 inline_cache_reg(R12);
3994
3995 // Number of stack slots consumed by locking an object
3996 sync_stack_slots(2);
3997
3998 // Compiled code's Frame Pointer
3999 frame_pointer(R31);
4000
4001 // Interpreter stores its frame pointer in a register which is
4002 // stored to the stack by I2CAdaptors.
4003 // I2CAdaptors convert from interpreted java to compiled java.
4004 interpreter_frame_pointer(R29);
4005
4006 // Stack alignment requirement
4007 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4008
4009 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4010 // for calls to C. Supports the var-args backing area for register parms.
4011 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4012
4013 // The after-PROLOG location of the return address. Location of
4014 // return address specifies a type (REG or STACK) and a number
4015 // representing the register number (i.e. - use a register name) or
4016 // stack slot.
4017 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4018 // Otherwise, it is above the locks and verification slot and alignment word
4019 // TODO this may well be correct but need to check why that - 2 is there
4020 // ppc port uses 0 but we definitely need to allow for fixed_slots
4021 // which folds in the space used for monitors
4022 return_addr(STACK - 2 +
4023 align_up((Compile::current()->in_preserve_stack_slots() +
4024 Compile::current()->fixed_slots()),
4025 stack_alignment_in_slots()));
4026
4027 // Location of compiled Java return values. Same as C for now.
4028 return_value
4029 %{
4030 // TODO do we allow ideal_reg == Op_RegN???
4031 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4032 "only return normal values");
4033
4034 static const int lo[Op_RegL + 1] = { // enum name
4035 0, // Op_Node
4036 0, // Op_Set
4037 R0_num, // Op_RegN
4038 R0_num, // Op_RegI
4039 R0_num, // Op_RegP
4040 V0_num, // Op_RegF
4041 V0_num, // Op_RegD
4042 R0_num // Op_RegL
4043 };
4044
4045 static const int hi[Op_RegL + 1] = { // enum name
4046 0, // Op_Node
4047 0, // Op_Set
4048 OptoReg::Bad, // Op_RegN
4049 OptoReg::Bad, // Op_RegI
4050 R0_H_num, // Op_RegP
4051 OptoReg::Bad, // Op_RegF
4052 V0_H_num, // Op_RegD
4053 R0_H_num // Op_RegL
4054 };
4055
4056 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4057 %}
4058 %}
4059
4060 //----------ATTRIBUTES---------------------------------------------------------
4061 //----------Operand Attributes-------------------------------------------------
4062 op_attrib op_cost(1); // Required cost attribute
4063
4064 //----------Instruction Attributes---------------------------------------------
4065 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4066 ins_attrib ins_size(32); // Required size attribute (in bits)
4067 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4068 // a non-matching short branch variant
4069 // of some long branch?
4070 ins_attrib ins_alignment(4); // Required alignment attribute (must
4071 // be a power of 2) specifies the
4072 // alignment that some part of the
4073 // instruction (not necessarily the
4074 // start) requires. If > 1, a
4075 // compute_padding() function must be
4076 // provided for the instruction
4077
4078 //----------OPERANDS-----------------------------------------------------------
4079 // Operand definitions must precede instruction definitions for correct parsing
4080 // in the ADLC because operands constitute user defined types which are used in
4081 // instruction definitions.
4082
4083 //----------Simple Operands----------------------------------------------------
4084
4085 // Integer operands 32 bit
4086 // 32 bit immediate
4087 operand immI()
4088 %{
4089 match(ConI);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 // 32 bit zero
4097 operand immI0()
4098 %{
4099 predicate(n->get_int() == 0);
4100 match(ConI);
4101
4102 op_cost(0);
4103 format %{ %}
4104 interface(CONST_INTER);
4105 %}
4106
4107 // 32 bit unit increment
4108 operand immI_1()
4109 %{
4110 predicate(n->get_int() == 1);
4111 match(ConI);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 // 32 bit unit decrement
4119 operand immI_M1()
4120 %{
4121 predicate(n->get_int() == -1);
4122 match(ConI);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 // Shift values for add/sub extension shift
4130 operand immIExt()
4131 %{
4132 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4133 match(ConI);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 operand immI_gt_1()
4141 %{
4142 predicate(n->get_int() > 1);
4143 match(ConI);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immI_le_4()
4151 %{
4152 predicate(n->get_int() <= 4);
4153 match(ConI);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 operand immI_16()
4161 %{
4162 predicate(n->get_int() == 16);
4163 match(ConI);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immI_24()
4171 %{
4172 predicate(n->get_int() == 24);
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immI_32()
4181 %{
4182 predicate(n->get_int() == 32);
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immI_48()
4191 %{
4192 predicate(n->get_int() == 48);
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immI_56()
4201 %{
4202 predicate(n->get_int() == 56);
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immI_63()
4211 %{
4212 predicate(n->get_int() == 63);
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immI_64()
4221 %{
4222 predicate(n->get_int() == 64);
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immI_255()
4231 %{
4232 predicate(n->get_int() == 255);
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immI_65535()
4241 %{
4242 predicate(n->get_int() == 65535);
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immI_positive()
4251 %{
4252 predicate(n->get_int() > 0);
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immL_255()
4261 %{
4262 predicate(n->get_long() == 255L);
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immL_65535()
4271 %{
4272 predicate(n->get_long() == 65535L);
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immL_4294967295()
4281 %{
4282 predicate(n->get_long() == 4294967295L);
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immL_bitmask()
4291 %{
4292 predicate((n->get_long() != 0)
4293 && ((n->get_long() & 0xc000000000000000l) == 0)
4294 && is_power_of_2(n->get_long() + 1));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 operand immI_bitmask()
4303 %{
4304 predicate((n->get_int() != 0)
4305 && ((n->get_int() & 0xc0000000) == 0)
4306 && is_power_of_2(n->get_int() + 1));
4307 match(ConI);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immL_positive_bitmaskI()
4315 %{
4316 predicate((n->get_long() != 0)
4317 && ((julong)n->get_long() < 0x80000000ULL)
4318 && is_power_of_2(n->get_long() + 1));
4319 match(ConL);
4320
4321 op_cost(0);
4322 format %{ %}
4323 interface(CONST_INTER);
4324 %}
4325
4326 // Scale values for scaled offset addressing modes (up to long but not quad)
4327 operand immIScale()
4328 %{
4329 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4330 match(ConI);
4331
4332 op_cost(0);
4333 format %{ %}
4334 interface(CONST_INTER);
4335 %}
4336
4337 // 26 bit signed offset -- for pc-relative branches
4338 operand immI26()
4339 %{
4340 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4341 match(ConI);
4342
4343 op_cost(0);
4344 format %{ %}
4345 interface(CONST_INTER);
4346 %}
4347
4348 // 19 bit signed offset -- for pc-relative loads
4349 operand immI19()
4350 %{
4351 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4352 match(ConI);
4353
4354 op_cost(0);
4355 format %{ %}
4356 interface(CONST_INTER);
4357 %}
4358
4359 // 12 bit unsigned offset -- for base plus immediate loads
4360 operand immIU12()
4361 %{
4362 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4363 match(ConI);
4364
4365 op_cost(0);
4366 format %{ %}
4367 interface(CONST_INTER);
4368 %}
4369
4370 operand immLU12()
4371 %{
4372 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4373 match(ConL);
4374
4375 op_cost(0);
4376 format %{ %}
4377 interface(CONST_INTER);
4378 %}
4379
4380 // Offset for scaled or unscaled immediate loads and stores
4381 operand immIOffset()
4382 %{
4383 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 operand immIOffset1()
4392 %{
4393 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immIOffset2()
4402 %{
4403 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immIOffset4()
4412 %{
4413 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immIOffset8()
4422 %{
4423 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 operand immIOffset16()
4432 %{
4433 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4434 match(ConI);
4435
4436 op_cost(0);
4437 format %{ %}
4438 interface(CONST_INTER);
4439 %}
4440
4441 operand immLoffset()
4442 %{
4443 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4444 match(ConL);
4445
4446 op_cost(0);
4447 format %{ %}
4448 interface(CONST_INTER);
4449 %}
4450
4451 operand immLoffset1()
4452 %{
4453 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4454 match(ConL);
4455
4456 op_cost(0);
4457 format %{ %}
4458 interface(CONST_INTER);
4459 %}
4460
4461 operand immLoffset2()
4462 %{
4463 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4464 match(ConL);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 operand immLoffset4()
4472 %{
4473 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4474 match(ConL);
4475
4476 op_cost(0);
4477 format %{ %}
4478 interface(CONST_INTER);
4479 %}
4480
4481 operand immLoffset8()
4482 %{
4483 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4484 match(ConL);
4485
4486 op_cost(0);
4487 format %{ %}
4488 interface(CONST_INTER);
4489 %}
4490
4491 operand immLoffset16()
4492 %{
4493 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4494 match(ConL);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // 8 bit signed value.
4502 operand immI8()
4503 %{
4504 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4505 match(ConI);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // 8 bit signed value (simm8), or #simm8 LSL 8.
4513 operand immI8_shift8()
4514 %{
4515 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4516 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4517 match(ConI);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // 8 bit signed value (simm8), or #simm8 LSL 8.
4525 operand immL8_shift8()
4526 %{
4527 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4528 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4529 match(ConL);
4530
4531 op_cost(0);
4532 format %{ %}
4533 interface(CONST_INTER);
4534 %}
4535
4536 // 8 bit integer valid for vector add sub immediate
4537 operand immBAddSubV()
4538 %{
4539 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4540 match(ConI);
4541
4542 op_cost(0);
4543 format %{ %}
4544 interface(CONST_INTER);
4545 %}
4546
4547 // 32 bit integer valid for add sub immediate
4548 operand immIAddSub()
4549 %{
4550 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4551 match(ConI);
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // 32 bit integer valid for vector add sub immediate
4558 operand immIAddSubV()
4559 %{
4560 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4561 match(ConI);
4562
4563 op_cost(0);
4564 format %{ %}
4565 interface(CONST_INTER);
4566 %}
4567
4568 // 32 bit unsigned integer valid for logical immediate
4569
4570 operand immBLog()
4571 %{
4572 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4573 match(ConI);
4574
4575 op_cost(0);
4576 format %{ %}
4577 interface(CONST_INTER);
4578 %}
4579
4580 operand immSLog()
4581 %{
4582 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4583 match(ConI);
4584
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 operand immILog()
4591 %{
4592 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4593 match(ConI);
4594
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Integer operands 64 bit
4601 // 64 bit immediate
4602 operand immL()
4603 %{
4604 match(ConL);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // 64 bit zero
4612 operand immL0()
4613 %{
4614 predicate(n->get_long() == 0);
4615 match(ConL);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 // 64 bit unit increment
4623 operand immL_1()
4624 %{
4625 predicate(n->get_long() == 1);
4626 match(ConL);
4627
4628 op_cost(0);
4629 format %{ %}
4630 interface(CONST_INTER);
4631 %}
4632
4633 // 64 bit unit decrement
4634 operand immL_M1()
4635 %{
4636 predicate(n->get_long() == -1);
4637 match(ConL);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // 32 bit offset of pc in thread anchor
4645
4646 operand immL_pc_off()
4647 %{
4648 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4649 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4650 match(ConL);
4651
4652 op_cost(0);
4653 format %{ %}
4654 interface(CONST_INTER);
4655 %}
4656
4657 // 64 bit integer valid for add sub immediate
4658 operand immLAddSub()
4659 %{
4660 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4661 match(ConL);
4662 op_cost(0);
4663 format %{ %}
4664 interface(CONST_INTER);
4665 %}
4666
4667 // 64 bit integer valid for addv subv immediate
4668 operand immLAddSubV()
4669 %{
4670 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4671 match(ConL);
4672
4673 op_cost(0);
4674 format %{ %}
4675 interface(CONST_INTER);
4676 %}
4677
4678 // 64 bit integer valid for logical immediate
4679 operand immLLog()
4680 %{
4681 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4682 match(ConL);
4683 op_cost(0);
4684 format %{ %}
4685 interface(CONST_INTER);
4686 %}
4687
4688 // Long Immediate: low 32-bit mask
4689 operand immL_32bits()
4690 %{
4691 predicate(n->get_long() == 0xFFFFFFFFL);
4692 match(ConL);
4693 op_cost(0);
4694 format %{ %}
4695 interface(CONST_INTER);
4696 %}
4697
4698 // Pointer operands
4699 // Pointer Immediate
4700 operand immP()
4701 %{
4702 match(ConP);
4703
4704 op_cost(0);
4705 format %{ %}
4706 interface(CONST_INTER);
4707 %}
4708
4709 // NULL Pointer Immediate
4710 operand immP0()
4711 %{
4712 predicate(n->get_ptr() == 0);
4713 match(ConP);
4714
4715 op_cost(0);
4716 format %{ %}
4717 interface(CONST_INTER);
4718 %}
4719
4720 // Pointer Immediate One
4721 // this is used in object initialization (initial object header)
4722 operand immP_1()
4723 %{
4724 predicate(n->get_ptr() == 1);
4725 match(ConP);
4726
4727 op_cost(0);
4728 format %{ %}
4729 interface(CONST_INTER);
4730 %}
4731
4732 // Card Table Byte Map Base
4733 operand immByteMapBase()
4734 %{
4735 // Get base of card map
4736 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4737 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4738 match(ConP);
4739
4740 op_cost(0);
4741 format %{ %}
4742 interface(CONST_INTER);
4743 %}
4744
4745 // Pointer Immediate Minus One
4746 // this is used when we want to write the current PC to the thread anchor
4747 operand immP_M1()
4748 %{
4749 predicate(n->get_ptr() == -1);
4750 match(ConP);
4751
4752 op_cost(0);
4753 format %{ %}
4754 interface(CONST_INTER);
4755 %}
4756
4757 // Pointer Immediate Minus Two
4758 // this is used when we want to write the current PC to the thread anchor
4759 operand immP_M2()
4760 %{
4761 predicate(n->get_ptr() == -2);
4762 match(ConP);
4763
4764 op_cost(0);
4765 format %{ %}
4766 interface(CONST_INTER);
4767 %}
4768
4769 // Float and Double operands
4770 // Double Immediate
4771 operand immD()
4772 %{
4773 match(ConD);
4774 op_cost(0);
4775 format %{ %}
4776 interface(CONST_INTER);
4777 %}
4778
4779 // Double Immediate: +0.0d
4780 operand immD0()
4781 %{
4782 predicate(jlong_cast(n->getd()) == 0);
4783 match(ConD);
4784
4785 op_cost(0);
4786 format %{ %}
4787 interface(CONST_INTER);
4788 %}
4789
4790 // constant 'double +0.0'.
4791 operand immDPacked()
4792 %{
4793 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4794 match(ConD);
4795 op_cost(0);
4796 format %{ %}
4797 interface(CONST_INTER);
4798 %}
4799
4800 // Float Immediate
4801 operand immF()
4802 %{
4803 match(ConF);
4804 op_cost(0);
4805 format %{ %}
4806 interface(CONST_INTER);
4807 %}
4808
4809 // Float Immediate: +0.0f.
4810 operand immF0()
4811 %{
4812 predicate(jint_cast(n->getf()) == 0);
4813 match(ConF);
4814
4815 op_cost(0);
4816 format %{ %}
4817 interface(CONST_INTER);
4818 %}
4819
4820 //
4821 operand immFPacked()
4822 %{
4823 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4824 match(ConF);
4825 op_cost(0);
4826 format %{ %}
4827 interface(CONST_INTER);
4828 %}
4829
4830 // Narrow pointer operands
4831 // Narrow Pointer Immediate
4832 operand immN()
4833 %{
4834 match(ConN);
4835
4836 op_cost(0);
4837 format %{ %}
4838 interface(CONST_INTER);
4839 %}
4840
4841 // Narrow NULL Pointer Immediate
4842 operand immN0()
4843 %{
4844 predicate(n->get_narrowcon() == 0);
4845 match(ConN);
4846
4847 op_cost(0);
4848 format %{ %}
4849 interface(CONST_INTER);
4850 %}
4851
4852 operand immNKlass()
4853 %{
4854 match(ConNKlass);
4855
4856 op_cost(0);
4857 format %{ %}
4858 interface(CONST_INTER);
4859 %}
4860
4861 // Integer 32 bit Register Operands
4862 // Integer 32 bitRegister (excludes SP)
4863 operand iRegI()
4864 %{
4865 constraint(ALLOC_IN_RC(any_reg32));
4866 match(RegI);
4867 match(iRegINoSp);
4868 op_cost(0);
4869 format %{ %}
4870 interface(REG_INTER);
4871 %}
4872
4873 // Integer 32 bit Register not Special
4874 operand iRegINoSp()
4875 %{
4876 constraint(ALLOC_IN_RC(no_special_reg32));
4877 match(RegI);
4878 op_cost(0);
4879 format %{ %}
4880 interface(REG_INTER);
4881 %}
4882
4883 // Integer 64 bit Register Operands
4884 // Integer 64 bit Register (includes SP)
4885 operand iRegL()
4886 %{
4887 constraint(ALLOC_IN_RC(any_reg));
4888 match(RegL);
4889 match(iRegLNoSp);
4890 op_cost(0);
4891 format %{ %}
4892 interface(REG_INTER);
4893 %}
4894
4895 // Integer 64 bit Register not Special
4896 operand iRegLNoSp()
4897 %{
4898 constraint(ALLOC_IN_RC(no_special_reg));
4899 match(RegL);
4900 match(iRegL_R0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Pointer Register Operands
4906 // Pointer Register
4907 operand iRegP()
4908 %{
4909 constraint(ALLOC_IN_RC(ptr_reg));
4910 match(RegP);
4911 match(iRegPNoSp);
4912 match(iRegP_R0);
4913 //match(iRegP_R2);
4914 //match(iRegP_R4);
4915 match(iRegP_R5);
4916 match(thread_RegP);
4917 op_cost(0);
4918 format %{ %}
4919 interface(REG_INTER);
4920 %}
4921
4922 // Pointer 64 bit Register not Special
4923 operand iRegPNoSp()
4924 %{
4925 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4926 match(RegP);
4927 // match(iRegP);
4928 // match(iRegP_R0);
4929 // match(iRegP_R2);
4930 // match(iRegP_R4);
4931 // match(iRegP_R5);
4932 // match(thread_RegP);
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938 // Pointer 64 bit Register R0 only
4939 operand iRegP_R0()
4940 %{
4941 constraint(ALLOC_IN_RC(r0_reg));
4942 match(RegP);
4943 // match(iRegP);
4944 match(iRegPNoSp);
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950 // Pointer 64 bit Register R1 only
4951 operand iRegP_R1()
4952 %{
4953 constraint(ALLOC_IN_RC(r1_reg));
4954 match(RegP);
4955 // match(iRegP);
4956 match(iRegPNoSp);
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 // Pointer 64 bit Register R2 only
4963 operand iRegP_R2()
4964 %{
4965 constraint(ALLOC_IN_RC(r2_reg));
4966 match(RegP);
4967 // match(iRegP);
4968 match(iRegPNoSp);
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 // Pointer 64 bit Register R3 only
4975 operand iRegP_R3()
4976 %{
4977 constraint(ALLOC_IN_RC(r3_reg));
4978 match(RegP);
4979 // match(iRegP);
4980 match(iRegPNoSp);
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 // Pointer 64 bit Register R4 only
4987 operand iRegP_R4()
4988 %{
4989 constraint(ALLOC_IN_RC(r4_reg));
4990 match(RegP);
4991 // match(iRegP);
4992 match(iRegPNoSp);
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 // Pointer 64 bit Register R5 only
4999 operand iRegP_R5()
5000 %{
5001 constraint(ALLOC_IN_RC(r5_reg));
5002 match(RegP);
5003 // match(iRegP);
5004 match(iRegPNoSp);
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 // Pointer 64 bit Register R10 only
5011 operand iRegP_R10()
5012 %{
5013 constraint(ALLOC_IN_RC(r10_reg));
5014 match(RegP);
5015 // match(iRegP);
5016 match(iRegPNoSp);
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 // Long 64 bit Register R0 only
5023 operand iRegL_R0()
5024 %{
5025 constraint(ALLOC_IN_RC(r0_reg));
5026 match(RegL);
5027 match(iRegLNoSp);
5028 op_cost(0);
5029 format %{ %}
5030 interface(REG_INTER);
5031 %}
5032
5033 // Long 64 bit Register R2 only
5034 operand iRegL_R2()
5035 %{
5036 constraint(ALLOC_IN_RC(r2_reg));
5037 match(RegL);
5038 match(iRegLNoSp);
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 // Long 64 bit Register R3 only
5045 operand iRegL_R3()
5046 %{
5047 constraint(ALLOC_IN_RC(r3_reg));
5048 match(RegL);
5049 match(iRegLNoSp);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 // Long 64 bit Register R11 only
5056 operand iRegL_R11()
5057 %{
5058 constraint(ALLOC_IN_RC(r11_reg));
5059 match(RegL);
5060 match(iRegLNoSp);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 // Pointer 64 bit Register FP only
5067 operand iRegP_FP()
5068 %{
5069 constraint(ALLOC_IN_RC(fp_reg));
5070 match(RegP);
5071 // match(iRegP);
5072 op_cost(0);
5073 format %{ %}
5074 interface(REG_INTER);
5075 %}
5076
5077 // Register R0 only
5078 operand iRegI_R0()
5079 %{
5080 constraint(ALLOC_IN_RC(int_r0_reg));
5081 match(RegI);
5082 match(iRegINoSp);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 // Register R2 only
5089 operand iRegI_R2()
5090 %{
5091 constraint(ALLOC_IN_RC(int_r2_reg));
5092 match(RegI);
5093 match(iRegINoSp);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 // Register R3 only
5100 operand iRegI_R3()
5101 %{
5102 constraint(ALLOC_IN_RC(int_r3_reg));
5103 match(RegI);
5104 match(iRegINoSp);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110
5111 // Register R4 only
5112 operand iRegI_R4()
5113 %{
5114 constraint(ALLOC_IN_RC(int_r4_reg));
5115 match(RegI);
5116 match(iRegINoSp);
5117 op_cost(0);
5118 format %{ %}
5119 interface(REG_INTER);
5120 %}
5121
5122
5123 // Pointer Register Operands
5124 // Narrow Pointer Register
5125 operand iRegN()
5126 %{
5127 constraint(ALLOC_IN_RC(any_reg32));
5128 match(RegN);
5129 match(iRegNNoSp);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 operand iRegN_R0()
5136 %{
5137 constraint(ALLOC_IN_RC(r0_reg));
5138 match(iRegN);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand iRegN_R2()
5145 %{
5146 constraint(ALLOC_IN_RC(r2_reg));
5147 match(iRegN);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 operand iRegN_R3()
5154 %{
5155 constraint(ALLOC_IN_RC(r3_reg));
5156 match(iRegN);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 // Integer 64 bit Register not Special
5163 operand iRegNNoSp()
5164 %{
5165 constraint(ALLOC_IN_RC(no_special_reg32));
5166 match(RegN);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 // heap base register -- used for encoding immN0
5173
5174 operand iRegIHeapbase()
5175 %{
5176 constraint(ALLOC_IN_RC(heapbase_reg));
5177 match(RegI);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 // Float Register
5184 // Float register operands
5185 operand vRegF()
5186 %{
5187 constraint(ALLOC_IN_RC(float_reg));
5188 match(RegF);
5189
5190 op_cost(0);
5191 format %{ %}
5192 interface(REG_INTER);
5193 %}
5194
5195 // Double Register
5196 // Double register operands
5197 operand vRegD()
5198 %{
5199 constraint(ALLOC_IN_RC(double_reg));
5200 match(RegD);
5201
5202 op_cost(0);
5203 format %{ %}
5204 interface(REG_INTER);
5205 %}
5206
5207 // Generic vector class. This will be used for
5208 // all vector operands, including NEON and SVE.
5209 operand vReg()
5210 %{
5211 constraint(ALLOC_IN_RC(dynamic));
5212 match(VecA);
5213 match(VecD);
5214 match(VecX);
5215
5216 op_cost(0);
5217 format %{ %}
5218 interface(REG_INTER);
5219 %}
5220
5221 operand vecA()
5222 %{
5223 constraint(ALLOC_IN_RC(vectora_reg));
5224 match(VecA);
5225
5226 op_cost(0);
5227 format %{ %}
5228 interface(REG_INTER);
5229 %}
5230
5231 operand vecD()
5232 %{
5233 constraint(ALLOC_IN_RC(vectord_reg));
5234 match(VecD);
5235
5236 op_cost(0);
5237 format %{ %}
5238 interface(REG_INTER);
5239 %}
5240
5241 operand vecX()
5242 %{
5243 constraint(ALLOC_IN_RC(vectorx_reg));
5244 match(VecX);
5245
5246 op_cost(0);
5247 format %{ %}
5248 interface(REG_INTER);
5249 %}
5250
5251 operand vRegD_V0()
5252 %{
5253 constraint(ALLOC_IN_RC(v0_reg));
5254 match(RegD);
5255 op_cost(0);
5256 format %{ %}
5257 interface(REG_INTER);
5258 %}
5259
5260 operand vRegD_V1()
5261 %{
5262 constraint(ALLOC_IN_RC(v1_reg));
5263 match(RegD);
5264 op_cost(0);
5265 format %{ %}
5266 interface(REG_INTER);
5267 %}
5268
5269 operand vRegD_V2()
5270 %{
5271 constraint(ALLOC_IN_RC(v2_reg));
5272 match(RegD);
5273 op_cost(0);
5274 format %{ %}
5275 interface(REG_INTER);
5276 %}
5277
5278 operand vRegD_V3()
5279 %{
5280 constraint(ALLOC_IN_RC(v3_reg));
5281 match(RegD);
5282 op_cost(0);
5283 format %{ %}
5284 interface(REG_INTER);
5285 %}
5286
5287 operand vRegD_V4()
5288 %{
5289 constraint(ALLOC_IN_RC(v4_reg));
5290 match(RegD);
5291 op_cost(0);
5292 format %{ %}
5293 interface(REG_INTER);
5294 %}
5295
5296 operand vRegD_V5()
5297 %{
5298 constraint(ALLOC_IN_RC(v5_reg));
5299 match(RegD);
5300 op_cost(0);
5301 format %{ %}
5302 interface(REG_INTER);
5303 %}
5304
5305 operand vRegD_V6()
5306 %{
5307 constraint(ALLOC_IN_RC(v6_reg));
5308 match(RegD);
5309 op_cost(0);
5310 format %{ %}
5311 interface(REG_INTER);
5312 %}
5313
5314 operand vRegD_V7()
5315 %{
5316 constraint(ALLOC_IN_RC(v7_reg));
5317 match(RegD);
5318 op_cost(0);
5319 format %{ %}
5320 interface(REG_INTER);
5321 %}
5322
5323 operand vRegD_V8()
5324 %{
5325 constraint(ALLOC_IN_RC(v8_reg));
5326 match(RegD);
5327 op_cost(0);
5328 format %{ %}
5329 interface(REG_INTER);
5330 %}
5331
5332 operand vRegD_V9()
5333 %{
5334 constraint(ALLOC_IN_RC(v9_reg));
5335 match(RegD);
5336 op_cost(0);
5337 format %{ %}
5338 interface(REG_INTER);
5339 %}
5340
5341 operand vRegD_V10()
5342 %{
5343 constraint(ALLOC_IN_RC(v10_reg));
5344 match(RegD);
5345 op_cost(0);
5346 format %{ %}
5347 interface(REG_INTER);
5348 %}
5349
5350 operand vRegD_V11()
5351 %{
5352 constraint(ALLOC_IN_RC(v11_reg));
5353 match(RegD);
5354 op_cost(0);
5355 format %{ %}
5356 interface(REG_INTER);
5357 %}
5358
5359 operand vRegD_V12()
5360 %{
5361 constraint(ALLOC_IN_RC(v12_reg));
5362 match(RegD);
5363 op_cost(0);
5364 format %{ %}
5365 interface(REG_INTER);
5366 %}
5367
5368 operand vRegD_V13()
5369 %{
5370 constraint(ALLOC_IN_RC(v13_reg));
5371 match(RegD);
5372 op_cost(0);
5373 format %{ %}
5374 interface(REG_INTER);
5375 %}
5376
5377 operand vRegD_V14()
5378 %{
5379 constraint(ALLOC_IN_RC(v14_reg));
5380 match(RegD);
5381 op_cost(0);
5382 format %{ %}
5383 interface(REG_INTER);
5384 %}
5385
5386 operand vRegD_V15()
5387 %{
5388 constraint(ALLOC_IN_RC(v15_reg));
5389 match(RegD);
5390 op_cost(0);
5391 format %{ %}
5392 interface(REG_INTER);
5393 %}
5394
5395 operand vRegD_V16()
5396 %{
5397 constraint(ALLOC_IN_RC(v16_reg));
5398 match(RegD);
5399 op_cost(0);
5400 format %{ %}
5401 interface(REG_INTER);
5402 %}
5403
5404 operand vRegD_V17()
5405 %{
5406 constraint(ALLOC_IN_RC(v17_reg));
5407 match(RegD);
5408 op_cost(0);
5409 format %{ %}
5410 interface(REG_INTER);
5411 %}
5412
5413 operand vRegD_V18()
5414 %{
5415 constraint(ALLOC_IN_RC(v18_reg));
5416 match(RegD);
5417 op_cost(0);
5418 format %{ %}
5419 interface(REG_INTER);
5420 %}
5421
5422 operand vRegD_V19()
5423 %{
5424 constraint(ALLOC_IN_RC(v19_reg));
5425 match(RegD);
5426 op_cost(0);
5427 format %{ %}
5428 interface(REG_INTER);
5429 %}
5430
5431 operand vRegD_V20()
5432 %{
5433 constraint(ALLOC_IN_RC(v20_reg));
5434 match(RegD);
5435 op_cost(0);
5436 format %{ %}
5437 interface(REG_INTER);
5438 %}
5439
5440 operand vRegD_V21()
5441 %{
5442 constraint(ALLOC_IN_RC(v21_reg));
5443 match(RegD);
5444 op_cost(0);
5445 format %{ %}
5446 interface(REG_INTER);
5447 %}
5448
5449 operand vRegD_V22()
5450 %{
5451 constraint(ALLOC_IN_RC(v22_reg));
5452 match(RegD);
5453 op_cost(0);
5454 format %{ %}
5455 interface(REG_INTER);
5456 %}
5457
5458 operand vRegD_V23()
5459 %{
5460 constraint(ALLOC_IN_RC(v23_reg));
5461 match(RegD);
5462 op_cost(0);
5463 format %{ %}
5464 interface(REG_INTER);
5465 %}
5466
5467 operand vRegD_V24()
5468 %{
5469 constraint(ALLOC_IN_RC(v24_reg));
5470 match(RegD);
5471 op_cost(0);
5472 format %{ %}
5473 interface(REG_INTER);
5474 %}
5475
5476 operand vRegD_V25()
5477 %{
5478 constraint(ALLOC_IN_RC(v25_reg));
5479 match(RegD);
5480 op_cost(0);
5481 format %{ %}
5482 interface(REG_INTER);
5483 %}
5484
5485 operand vRegD_V26()
5486 %{
5487 constraint(ALLOC_IN_RC(v26_reg));
5488 match(RegD);
5489 op_cost(0);
5490 format %{ %}
5491 interface(REG_INTER);
5492 %}
5493
5494 operand vRegD_V27()
5495 %{
5496 constraint(ALLOC_IN_RC(v27_reg));
5497 match(RegD);
5498 op_cost(0);
5499 format %{ %}
5500 interface(REG_INTER);
5501 %}
5502
5503 operand vRegD_V28()
5504 %{
5505 constraint(ALLOC_IN_RC(v28_reg));
5506 match(RegD);
5507 op_cost(0);
5508 format %{ %}
5509 interface(REG_INTER);
5510 %}
5511
5512 operand vRegD_V29()
5513 %{
5514 constraint(ALLOC_IN_RC(v29_reg));
5515 match(RegD);
5516 op_cost(0);
5517 format %{ %}
5518 interface(REG_INTER);
5519 %}
5520
5521 operand vRegD_V30()
5522 %{
5523 constraint(ALLOC_IN_RC(v30_reg));
5524 match(RegD);
5525 op_cost(0);
5526 format %{ %}
5527 interface(REG_INTER);
5528 %}
5529
5530 operand vRegD_V31()
5531 %{
5532 constraint(ALLOC_IN_RC(v31_reg));
5533 match(RegD);
5534 op_cost(0);
5535 format %{ %}
5536 interface(REG_INTER);
5537 %}
5538
5539 operand pReg()
5540 %{
5541 constraint(ALLOC_IN_RC(pr_reg));
5542 match(RegVectMask);
5543 match(pRegGov);
5544 op_cost(0);
5545 format %{ %}
5546 interface(REG_INTER);
5547 %}
5548
5549 operand pRegGov()
5550 %{
5551 constraint(ALLOC_IN_RC(gov_pr));
5552 match(RegVectMask);
5553 match(pReg);
5554 op_cost(0);
5555 format %{ %}
5556 interface(REG_INTER);
5557 %}
5558
5559 operand pRegGov_P0()
5560 %{
5561 constraint(ALLOC_IN_RC(p0_reg));
5562 match(RegVectMask);
5563 op_cost(0);
5564 format %{ %}
5565 interface(REG_INTER);
5566 %}
5567
5568 operand pRegGov_P1()
5569 %{
5570 constraint(ALLOC_IN_RC(p1_reg));
5571 match(RegVectMask);
5572 op_cost(0);
5573 format %{ %}
5574 interface(REG_INTER);
5575 %}
5576
5577 // Flags register, used as output of signed compare instructions
5578
5579 // note that on AArch64 we also use this register as the output for
5580 // for floating point compare instructions (CmpF CmpD). this ensures
5581 // that ordered inequality tests use GT, GE, LT or LE none of which
5582 // pass through cases where the result is unordered i.e. one or both
5583 // inputs to the compare is a NaN. this means that the ideal code can
5584 // replace e.g. a GT with an LE and not end up capturing the NaN case
5585 // (where the comparison should always fail). EQ and NE tests are
5586 // always generated in ideal code so that unordered folds into the NE
5587 // case, matching the behaviour of AArch64 NE.
5588 //
5589 // This differs from x86 where the outputs of FP compares use a
5590 // special FP flags registers and where compares based on this
5591 // register are distinguished into ordered inequalities (cmpOpUCF) and
5592 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5593 // to explicitly handle the unordered case in branches. x86 also has
5594 // to include extra CMoveX rules to accept a cmpOpUCF input.
5595
5596 operand rFlagsReg()
5597 %{
5598 constraint(ALLOC_IN_RC(int_flags));
5599 match(RegFlags);
5600
5601 op_cost(0);
5602 format %{ "RFLAGS" %}
5603 interface(REG_INTER);
5604 %}
5605
5606 // Flags register, used as output of unsigned compare instructions
5607 operand rFlagsRegU()
5608 %{
5609 constraint(ALLOC_IN_RC(int_flags));
5610 match(RegFlags);
5611
5612 op_cost(0);
5613 format %{ "RFLAGSU" %}
5614 interface(REG_INTER);
5615 %}
5616
5617 // Special Registers
5618
5619 // Method Register
5620 operand inline_cache_RegP(iRegP reg)
5621 %{
5622 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5623 match(reg);
5624 match(iRegPNoSp);
5625 op_cost(0);
5626 format %{ %}
5627 interface(REG_INTER);
5628 %}
5629
5630 // Thread Register
5631 operand thread_RegP(iRegP reg)
5632 %{
5633 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5634 match(reg);
5635 op_cost(0);
5636 format %{ %}
5637 interface(REG_INTER);
5638 %}
5639
5640 operand lr_RegP(iRegP reg)
5641 %{
5642 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5643 match(reg);
5644 op_cost(0);
5645 format %{ %}
5646 interface(REG_INTER);
5647 %}
5648
5649 //----------Memory Operands----------------------------------------------------
5650
5651 operand indirect(iRegP reg)
5652 %{
5653 constraint(ALLOC_IN_RC(ptr_reg));
5654 match(reg);
5655 op_cost(0);
5656 format %{ "[$reg]" %}
5657 interface(MEMORY_INTER) %{
5658 base($reg);
5659 index(0xffffffff);
5660 scale(0x0);
5661 disp(0x0);
5662 %}
5663 %}
5664
5665 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5666 %{
5667 constraint(ALLOC_IN_RC(ptr_reg));
5668 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5669 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5670 op_cost(0);
5671 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5672 interface(MEMORY_INTER) %{
5673 base($reg);
5674 index($ireg);
5675 scale($scale);
5676 disp(0x0);
5677 %}
5678 %}
5679
5680 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5681 %{
5682 constraint(ALLOC_IN_RC(ptr_reg));
5683 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5684 match(AddP reg (LShiftL lreg scale));
5685 op_cost(0);
5686 format %{ "$reg, $lreg lsl($scale)" %}
5687 interface(MEMORY_INTER) %{
5688 base($reg);
5689 index($lreg);
5690 scale($scale);
5691 disp(0x0);
5692 %}
5693 %}
5694
5695 operand indIndexI2L(iRegP reg, iRegI ireg)
5696 %{
5697 constraint(ALLOC_IN_RC(ptr_reg));
5698 match(AddP reg (ConvI2L ireg));
5699 op_cost(0);
5700 format %{ "$reg, $ireg, 0, I2L" %}
5701 interface(MEMORY_INTER) %{
5702 base($reg);
5703 index($ireg);
5704 scale(0x0);
5705 disp(0x0);
5706 %}
5707 %}
5708
5709 operand indIndex(iRegP reg, iRegL lreg)
5710 %{
5711 constraint(ALLOC_IN_RC(ptr_reg));
5712 match(AddP reg lreg);
5713 op_cost(0);
5714 format %{ "$reg, $lreg" %}
5715 interface(MEMORY_INTER) %{
5716 base($reg);
5717 index($lreg);
5718 scale(0x0);
5719 disp(0x0);
5720 %}
5721 %}
5722
5723 operand indOffI(iRegP reg, immIOffset off)
5724 %{
5725 constraint(ALLOC_IN_RC(ptr_reg));
5726 match(AddP reg off);
5727 op_cost(0);
5728 format %{ "[$reg, $off]" %}
5729 interface(MEMORY_INTER) %{
5730 base($reg);
5731 index(0xffffffff);
5732 scale(0x0);
5733 disp($off);
5734 %}
5735 %}
5736
5737 operand indOffI1(iRegP reg, immIOffset1 off)
5738 %{
5739 constraint(ALLOC_IN_RC(ptr_reg));
5740 match(AddP reg off);
5741 op_cost(0);
5742 format %{ "[$reg, $off]" %}
5743 interface(MEMORY_INTER) %{
5744 base($reg);
5745 index(0xffffffff);
5746 scale(0x0);
5747 disp($off);
5748 %}
5749 %}
5750
5751 operand indOffI2(iRegP reg, immIOffset2 off)
5752 %{
5753 constraint(ALLOC_IN_RC(ptr_reg));
5754 match(AddP reg off);
5755 op_cost(0);
5756 format %{ "[$reg, $off]" %}
5757 interface(MEMORY_INTER) %{
5758 base($reg);
5759 index(0xffffffff);
5760 scale(0x0);
5761 disp($off);
5762 %}
5763 %}
5764
5765 operand indOffI4(iRegP reg, immIOffset4 off)
5766 %{
5767 constraint(ALLOC_IN_RC(ptr_reg));
5768 match(AddP reg off);
5769 op_cost(0);
5770 format %{ "[$reg, $off]" %}
5771 interface(MEMORY_INTER) %{
5772 base($reg);
5773 index(0xffffffff);
5774 scale(0x0);
5775 disp($off);
5776 %}
5777 %}
5778
5779 operand indOffI8(iRegP reg, immIOffset8 off)
5780 %{
5781 constraint(ALLOC_IN_RC(ptr_reg));
5782 match(AddP reg off);
5783 op_cost(0);
5784 format %{ "[$reg, $off]" %}
5785 interface(MEMORY_INTER) %{
5786 base($reg);
5787 index(0xffffffff);
5788 scale(0x0);
5789 disp($off);
5790 %}
5791 %}
5792
5793 operand indOffI16(iRegP reg, immIOffset16 off)
5794 %{
5795 constraint(ALLOC_IN_RC(ptr_reg));
5796 match(AddP reg off);
5797 op_cost(0);
5798 format %{ "[$reg, $off]" %}
5799 interface(MEMORY_INTER) %{
5800 base($reg);
5801 index(0xffffffff);
5802 scale(0x0);
5803 disp($off);
5804 %}
5805 %}
5806
5807 operand indOffL(iRegP reg, immLoffset off)
5808 %{
5809 constraint(ALLOC_IN_RC(ptr_reg));
5810 match(AddP reg off);
5811 op_cost(0);
5812 format %{ "[$reg, $off]" %}
5813 interface(MEMORY_INTER) %{
5814 base($reg);
5815 index(0xffffffff);
5816 scale(0x0);
5817 disp($off);
5818 %}
5819 %}
5820
5821 operand indOffL1(iRegP reg, immLoffset1 off)
5822 %{
5823 constraint(ALLOC_IN_RC(ptr_reg));
5824 match(AddP reg off);
5825 op_cost(0);
5826 format %{ "[$reg, $off]" %}
5827 interface(MEMORY_INTER) %{
5828 base($reg);
5829 index(0xffffffff);
5830 scale(0x0);
5831 disp($off);
5832 %}
5833 %}
5834
5835 operand indOffL2(iRegP reg, immLoffset2 off)
5836 %{
5837 constraint(ALLOC_IN_RC(ptr_reg));
5838 match(AddP reg off);
5839 op_cost(0);
5840 format %{ "[$reg, $off]" %}
5841 interface(MEMORY_INTER) %{
5842 base($reg);
5843 index(0xffffffff);
5844 scale(0x0);
5845 disp($off);
5846 %}
5847 %}
5848
5849 operand indOffL4(iRegP reg, immLoffset4 off)
5850 %{
5851 constraint(ALLOC_IN_RC(ptr_reg));
5852 match(AddP reg off);
5853 op_cost(0);
5854 format %{ "[$reg, $off]" %}
5855 interface(MEMORY_INTER) %{
5856 base($reg);
5857 index(0xffffffff);
5858 scale(0x0);
5859 disp($off);
5860 %}
5861 %}
5862
5863 operand indOffL8(iRegP reg, immLoffset8 off)
5864 %{
5865 constraint(ALLOC_IN_RC(ptr_reg));
5866 match(AddP reg off);
5867 op_cost(0);
5868 format %{ "[$reg, $off]" %}
5869 interface(MEMORY_INTER) %{
5870 base($reg);
5871 index(0xffffffff);
5872 scale(0x0);
5873 disp($off);
5874 %}
5875 %}
5876
5877 operand indOffL16(iRegP reg, immLoffset16 off)
5878 %{
5879 constraint(ALLOC_IN_RC(ptr_reg));
5880 match(AddP reg off);
5881 op_cost(0);
5882 format %{ "[$reg, $off]" %}
5883 interface(MEMORY_INTER) %{
5884 base($reg);
5885 index(0xffffffff);
5886 scale(0x0);
5887 disp($off);
5888 %}
5889 %}
5890
5891 operand indirectN(iRegN reg)
5892 %{
5893 predicate(CompressedOops::shift() == 0);
5894 constraint(ALLOC_IN_RC(ptr_reg));
5895 match(DecodeN reg);
5896 op_cost(0);
5897 format %{ "[$reg]\t# narrow" %}
5898 interface(MEMORY_INTER) %{
5899 base($reg);
5900 index(0xffffffff);
5901 scale(0x0);
5902 disp(0x0);
5903 %}
5904 %}
5905
5906 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5907 %{
5908 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5909 constraint(ALLOC_IN_RC(ptr_reg));
5910 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5911 op_cost(0);
5912 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5913 interface(MEMORY_INTER) %{
5914 base($reg);
5915 index($ireg);
5916 scale($scale);
5917 disp(0x0);
5918 %}
5919 %}
5920
5921 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5922 %{
5923 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5924 constraint(ALLOC_IN_RC(ptr_reg));
5925 match(AddP (DecodeN reg) (LShiftL lreg scale));
5926 op_cost(0);
5927 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5928 interface(MEMORY_INTER) %{
5929 base($reg);
5930 index($lreg);
5931 scale($scale);
5932 disp(0x0);
5933 %}
5934 %}
5935
5936 operand indIndexI2LN(iRegN reg, iRegI ireg)
5937 %{
5938 predicate(CompressedOops::shift() == 0);
5939 constraint(ALLOC_IN_RC(ptr_reg));
5940 match(AddP (DecodeN reg) (ConvI2L ireg));
5941 op_cost(0);
5942 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5943 interface(MEMORY_INTER) %{
5944 base($reg);
5945 index($ireg);
5946 scale(0x0);
5947 disp(0x0);
5948 %}
5949 %}
5950
5951 operand indIndexN(iRegN reg, iRegL lreg)
5952 %{
5953 predicate(CompressedOops::shift() == 0);
5954 constraint(ALLOC_IN_RC(ptr_reg));
5955 match(AddP (DecodeN reg) lreg);
5956 op_cost(0);
5957 format %{ "$reg, $lreg\t# narrow" %}
5958 interface(MEMORY_INTER) %{
5959 base($reg);
5960 index($lreg);
5961 scale(0x0);
5962 disp(0x0);
5963 %}
5964 %}
5965
5966 operand indOffIN(iRegN reg, immIOffset off)
5967 %{
5968 predicate(CompressedOops::shift() == 0);
5969 constraint(ALLOC_IN_RC(ptr_reg));
5970 match(AddP (DecodeN reg) off);
5971 op_cost(0);
5972 format %{ "[$reg, $off]\t# narrow" %}
5973 interface(MEMORY_INTER) %{
5974 base($reg);
5975 index(0xffffffff);
5976 scale(0x0);
5977 disp($off);
5978 %}
5979 %}
5980
5981 operand indOffLN(iRegN reg, immLoffset off)
5982 %{
5983 predicate(CompressedOops::shift() == 0);
5984 constraint(ALLOC_IN_RC(ptr_reg));
5985 match(AddP (DecodeN reg) off);
5986 op_cost(0);
5987 format %{ "[$reg, $off]\t# narrow" %}
5988 interface(MEMORY_INTER) %{
5989 base($reg);
5990 index(0xffffffff);
5991 scale(0x0);
5992 disp($off);
5993 %}
5994 %}
5995
5996
5997
5998 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5999 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6000 %{
6001 constraint(ALLOC_IN_RC(ptr_reg));
6002 match(AddP reg off);
6003 op_cost(0);
6004 format %{ "[$reg, $off]" %}
6005 interface(MEMORY_INTER) %{
6006 base($reg);
6007 index(0xffffffff);
6008 scale(0x0);
6009 disp($off);
6010 %}
6011 %}
6012
6013 //----------Special Memory Operands--------------------------------------------
6014 // Stack Slot Operand - This operand is used for loading and storing temporary
6015 // values on the stack where a match requires a value to
6016 // flow through memory.
6017 operand stackSlotP(sRegP reg)
6018 %{
6019 constraint(ALLOC_IN_RC(stack_slots));
6020 op_cost(100);
6021 // No match rule because this operand is only generated in matching
6022 // match(RegP);
6023 format %{ "[$reg]" %}
6024 interface(MEMORY_INTER) %{
6025 base(0x1e); // RSP
6026 index(0x0); // No Index
6027 scale(0x0); // No Scale
6028 disp($reg); // Stack Offset
6029 %}
6030 %}
6031
6032 operand stackSlotI(sRegI reg)
6033 %{
6034 constraint(ALLOC_IN_RC(stack_slots));
6035 // No match rule because this operand is only generated in matching
6036 // match(RegI);
6037 format %{ "[$reg]" %}
6038 interface(MEMORY_INTER) %{
6039 base(0x1e); // RSP
6040 index(0x0); // No Index
6041 scale(0x0); // No Scale
6042 disp($reg); // Stack Offset
6043 %}
6044 %}
6045
6046 operand stackSlotF(sRegF reg)
6047 %{
6048 constraint(ALLOC_IN_RC(stack_slots));
6049 // No match rule because this operand is only generated in matching
6050 // match(RegF);
6051 format %{ "[$reg]" %}
6052 interface(MEMORY_INTER) %{
6053 base(0x1e); // RSP
6054 index(0x0); // No Index
6055 scale(0x0); // No Scale
6056 disp($reg); // Stack Offset
6057 %}
6058 %}
6059
6060 operand stackSlotD(sRegD reg)
6061 %{
6062 constraint(ALLOC_IN_RC(stack_slots));
6063 // No match rule because this operand is only generated in matching
6064 // match(RegD);
6065 format %{ "[$reg]" %}
6066 interface(MEMORY_INTER) %{
6067 base(0x1e); // RSP
6068 index(0x0); // No Index
6069 scale(0x0); // No Scale
6070 disp($reg); // Stack Offset
6071 %}
6072 %}
6073
6074 operand stackSlotL(sRegL reg)
6075 %{
6076 constraint(ALLOC_IN_RC(stack_slots));
6077 // No match rule because this operand is only generated in matching
6078 // match(RegL);
6079 format %{ "[$reg]" %}
6080 interface(MEMORY_INTER) %{
6081 base(0x1e); // RSP
6082 index(0x0); // No Index
6083 scale(0x0); // No Scale
6084 disp($reg); // Stack Offset
6085 %}
6086 %}
6087
6088 // Operands for expressing Control Flow
6089 // NOTE: Label is a predefined operand which should not be redefined in
6090 // the AD file. It is generically handled within the ADLC.
6091
6092 //----------Conditional Branch Operands----------------------------------------
6093 // Comparison Op - This is the operation of the comparison, and is limited to
6094 // the following set of codes:
6095 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6096 //
6097 // Other attributes of the comparison, such as unsignedness, are specified
6098 // by the comparison instruction that sets a condition code flags register.
6099 // That result is represented by a flags operand whose subtype is appropriate
6100 // to the unsignedness (etc.) of the comparison.
6101 //
6102 // Later, the instruction which matches both the Comparison Op (a Bool) and
6103 // the flags (produced by the Cmp) specifies the coding of the comparison op
6104 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6105
6106 // used for signed integral comparisons and fp comparisons
6107
6108 operand cmpOp()
6109 %{
6110 match(Bool);
6111
6112 format %{ "" %}
6113 interface(COND_INTER) %{
6114 equal(0x0, "eq");
6115 not_equal(0x1, "ne");
6116 less(0xb, "lt");
6117 greater_equal(0xa, "ge");
6118 less_equal(0xd, "le");
6119 greater(0xc, "gt");
6120 overflow(0x6, "vs");
6121 no_overflow(0x7, "vc");
6122 %}
6123 %}
6124
6125 // used for unsigned integral comparisons
6126
6127 operand cmpOpU()
6128 %{
6129 match(Bool);
6130
6131 format %{ "" %}
6132 interface(COND_INTER) %{
6133 equal(0x0, "eq");
6134 not_equal(0x1, "ne");
6135 less(0x3, "lo");
6136 greater_equal(0x2, "hs");
6137 less_equal(0x9, "ls");
6138 greater(0x8, "hi");
6139 overflow(0x6, "vs");
6140 no_overflow(0x7, "vc");
6141 %}
6142 %}
6143
6144 // used for certain integral comparisons which can be
6145 // converted to cbxx or tbxx instructions
6146
6147 operand cmpOpEqNe()
6148 %{
6149 match(Bool);
6150 op_cost(0);
6151 predicate(n->as_Bool()->_test._test == BoolTest::ne
6152 || n->as_Bool()->_test._test == BoolTest::eq);
6153
6154 format %{ "" %}
6155 interface(COND_INTER) %{
6156 equal(0x0, "eq");
6157 not_equal(0x1, "ne");
6158 less(0xb, "lt");
6159 greater_equal(0xa, "ge");
6160 less_equal(0xd, "le");
6161 greater(0xc, "gt");
6162 overflow(0x6, "vs");
6163 no_overflow(0x7, "vc");
6164 %}
6165 %}
6166
6167 // used for certain integral comparisons which can be
6168 // converted to cbxx or tbxx instructions
6169
6170 operand cmpOpLtGe()
6171 %{
6172 match(Bool);
6173 op_cost(0);
6174
6175 predicate(n->as_Bool()->_test._test == BoolTest::lt
6176 || n->as_Bool()->_test._test == BoolTest::ge);
6177
6178 format %{ "" %}
6179 interface(COND_INTER) %{
6180 equal(0x0, "eq");
6181 not_equal(0x1, "ne");
6182 less(0xb, "lt");
6183 greater_equal(0xa, "ge");
6184 less_equal(0xd, "le");
6185 greater(0xc, "gt");
6186 overflow(0x6, "vs");
6187 no_overflow(0x7, "vc");
6188 %}
6189 %}
6190
6191 // used for certain unsigned integral comparisons which can be
6192 // converted to cbxx or tbxx instructions
6193
6194 operand cmpOpUEqNeLtGe()
6195 %{
6196 match(Bool);
6197 op_cost(0);
6198
6199 predicate(n->as_Bool()->_test._test == BoolTest::eq
6200 || n->as_Bool()->_test._test == BoolTest::ne
6201 || n->as_Bool()->_test._test == BoolTest::lt
6202 || n->as_Bool()->_test._test == BoolTest::ge);
6203
6204 format %{ "" %}
6205 interface(COND_INTER) %{
6206 equal(0x0, "eq");
6207 not_equal(0x1, "ne");
6208 less(0xb, "lt");
6209 greater_equal(0xa, "ge");
6210 less_equal(0xd, "le");
6211 greater(0xc, "gt");
6212 overflow(0x6, "vs");
6213 no_overflow(0x7, "vc");
6214 %}
6215 %}
6216
6217 // Special operand allowing long args to int ops to be truncated for free
6218
6219 operand iRegL2I(iRegL reg) %{
6220
6221 op_cost(0);
6222
6223 match(ConvL2I reg);
6224
6225 format %{ "l2i($reg)" %}
6226
6227 interface(REG_INTER)
6228 %}
6229
6230 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6231 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6232 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6233 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6234
6235 //----------OPERAND CLASSES----------------------------------------------------
6236 // Operand Classes are groups of operands that are used as to simplify
6237 // instruction definitions by not requiring the AD writer to specify
6238 // separate instructions for every form of operand when the
6239 // instruction accepts multiple operand types with the same basic
6240 // encoding and format. The classic case of this is memory operands.
6241
6242 // memory is used to define read/write location for load/store
6243 // instruction defs. we can turn a memory op into an Address
6244
6245 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6246 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6247
6248 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6249 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6250
6251 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6252 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6253
6254 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6255 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6256
6257 // All of the memory operands. For the pipeline description.
6258 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6259 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6260 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6261
6262
6263 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6264 // operations. it allows the src to be either an iRegI or a (ConvL2I
6265 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6266 // can be elided because the 32-bit instruction will just employ the
6267 // lower 32 bits anyway.
6268 //
6269 // n.b. this does not elide all L2I conversions. if the truncated
6270 // value is consumed by more than one operation then the ConvL2I
6271 // cannot be bundled into the consuming nodes so an l2i gets planted
6272 // (actually a movw $dst $src) and the downstream instructions consume
6273 // the result of the l2i as an iRegI input. That's a shame since the
6274 // movw is actually redundant but its not too costly.
6275
6276 opclass iRegIorL2I(iRegI, iRegL2I);
6277
6278 //----------PIPELINE-----------------------------------------------------------
6279 // Rules which define the behavior of the target architectures pipeline.
6280
6281 // For specific pipelines, eg A53, define the stages of that pipeline
6282 //pipe_desc(ISS, EX1, EX2, WR);
6283 #define ISS S0
6284 #define EX1 S1
6285 #define EX2 S2
6286 #define WR S3
6287
6288 // Integer ALU reg operation
6289 pipeline %{
6290
6291 attributes %{
6292 // ARM instructions are of fixed length
6293 fixed_size_instructions; // Fixed size instructions TODO does
6294 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6295 // ARM instructions come in 32-bit word units
6296 instruction_unit_size = 4; // An instruction is 4 bytes long
6297 instruction_fetch_unit_size = 64; // The processor fetches one line
6298 instruction_fetch_units = 1; // of 64 bytes
6299
6300 // List of nop instructions
6301 nops( MachNop );
6302 %}
6303
6304 // We don't use an actual pipeline model so don't care about resources
6305 // or description. we do use pipeline classes to introduce fixed
6306 // latencies
6307
6308 //----------RESOURCES----------------------------------------------------------
6309 // Resources are the functional units available to the machine
6310
6311 resources( INS0, INS1, INS01 = INS0 | INS1,
6312 ALU0, ALU1, ALU = ALU0 | ALU1,
6313 MAC,
6314 DIV,
6315 BRANCH,
6316 LDST,
6317 NEON_FP);
6318
6319 //----------PIPELINE DESCRIPTION-----------------------------------------------
6320 // Pipeline Description specifies the stages in the machine's pipeline
6321
6322 // Define the pipeline as a generic 6 stage pipeline
6323 pipe_desc(S0, S1, S2, S3, S4, S5);
6324
6325 //----------PIPELINE CLASSES---------------------------------------------------
6326 // Pipeline Classes describe the stages in which input and output are
6327 // referenced by the hardware pipeline.
6328
6329 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6330 %{
6331 single_instruction;
6332 src1 : S1(read);
6333 src2 : S2(read);
6334 dst : S5(write);
6335 INS01 : ISS;
6336 NEON_FP : S5;
6337 %}
6338
6339 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6340 %{
6341 single_instruction;
6342 src1 : S1(read);
6343 src2 : S2(read);
6344 dst : S5(write);
6345 INS01 : ISS;
6346 NEON_FP : S5;
6347 %}
6348
6349 pipe_class fp_uop_s(vRegF dst, vRegF src)
6350 %{
6351 single_instruction;
6352 src : S1(read);
6353 dst : S5(write);
6354 INS01 : ISS;
6355 NEON_FP : S5;
6356 %}
6357
6358 pipe_class fp_uop_d(vRegD dst, vRegD src)
6359 %{
6360 single_instruction;
6361 src : S1(read);
6362 dst : S5(write);
6363 INS01 : ISS;
6364 NEON_FP : S5;
6365 %}
6366
6367 pipe_class fp_d2f(vRegF dst, vRegD src)
6368 %{
6369 single_instruction;
6370 src : S1(read);
6371 dst : S5(write);
6372 INS01 : ISS;
6373 NEON_FP : S5;
6374 %}
6375
6376 pipe_class fp_f2d(vRegD dst, vRegF src)
6377 %{
6378 single_instruction;
6379 src : S1(read);
6380 dst : S5(write);
6381 INS01 : ISS;
6382 NEON_FP : S5;
6383 %}
6384
6385 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6386 %{
6387 single_instruction;
6388 src : S1(read);
6389 dst : S5(write);
6390 INS01 : ISS;
6391 NEON_FP : S5;
6392 %}
6393
6394 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6395 %{
6396 single_instruction;
6397 src : S1(read);
6398 dst : S5(write);
6399 INS01 : ISS;
6400 NEON_FP : S5;
6401 %}
6402
6403 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6404 %{
6405 single_instruction;
6406 src : S1(read);
6407 dst : S5(write);
6408 INS01 : ISS;
6409 NEON_FP : S5;
6410 %}
6411
6412 pipe_class fp_l2f(vRegF dst, iRegL src)
6413 %{
6414 single_instruction;
6415 src : S1(read);
6416 dst : S5(write);
6417 INS01 : ISS;
6418 NEON_FP : S5;
6419 %}
6420
6421 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6422 %{
6423 single_instruction;
6424 src : S1(read);
6425 dst : S5(write);
6426 INS01 : ISS;
6427 NEON_FP : S5;
6428 %}
6429
6430 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6431 %{
6432 single_instruction;
6433 src : S1(read);
6434 dst : S5(write);
6435 INS01 : ISS;
6436 NEON_FP : S5;
6437 %}
6438
6439 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6440 %{
6441 single_instruction;
6442 src : S1(read);
6443 dst : S5(write);
6444 INS01 : ISS;
6445 NEON_FP : S5;
6446 %}
6447
6448 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6449 %{
6450 single_instruction;
6451 src : S1(read);
6452 dst : S5(write);
6453 INS01 : ISS;
6454 NEON_FP : S5;
6455 %}
6456
6457 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6458 %{
6459 single_instruction;
6460 src1 : S1(read);
6461 src2 : S2(read);
6462 dst : S5(write);
6463 INS0 : ISS;
6464 NEON_FP : S5;
6465 %}
6466
6467 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6468 %{
6469 single_instruction;
6470 src1 : S1(read);
6471 src2 : S2(read);
6472 dst : S5(write);
6473 INS0 : ISS;
6474 NEON_FP : S5;
6475 %}
6476
6477 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6478 %{
6479 single_instruction;
6480 cr : S1(read);
6481 src1 : S1(read);
6482 src2 : S1(read);
6483 dst : S3(write);
6484 INS01 : ISS;
6485 NEON_FP : S3;
6486 %}
6487
6488 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6489 %{
6490 single_instruction;
6491 cr : S1(read);
6492 src1 : S1(read);
6493 src2 : S1(read);
6494 dst : S3(write);
6495 INS01 : ISS;
6496 NEON_FP : S3;
6497 %}
6498
6499 pipe_class fp_imm_s(vRegF dst)
6500 %{
6501 single_instruction;
6502 dst : S3(write);
6503 INS01 : ISS;
6504 NEON_FP : S3;
6505 %}
6506
6507 pipe_class fp_imm_d(vRegD dst)
6508 %{
6509 single_instruction;
6510 dst : S3(write);
6511 INS01 : ISS;
6512 NEON_FP : S3;
6513 %}
6514
6515 pipe_class fp_load_constant_s(vRegF dst)
6516 %{
6517 single_instruction;
6518 dst : S4(write);
6519 INS01 : ISS;
6520 NEON_FP : S4;
6521 %}
6522
6523 pipe_class fp_load_constant_d(vRegD dst)
6524 %{
6525 single_instruction;
6526 dst : S4(write);
6527 INS01 : ISS;
6528 NEON_FP : S4;
6529 %}
6530
6531 //------- Integer ALU operations --------------------------
6532
6533 // Integer ALU reg-reg operation
6534 // Operands needed in EX1, result generated in EX2
6535 // Eg. ADD x0, x1, x2
6536 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6537 %{
6538 single_instruction;
6539 dst : EX2(write);
6540 src1 : EX1(read);
6541 src2 : EX1(read);
6542 INS01 : ISS; // Dual issue as instruction 0 or 1
6543 ALU : EX2;
6544 %}
6545
6546 // Integer ALU reg-reg operation with constant shift
6547 // Shifted register must be available in LATE_ISS instead of EX1
6548 // Eg. ADD x0, x1, x2, LSL #2
6549 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6550 %{
6551 single_instruction;
6552 dst : EX2(write);
6553 src1 : EX1(read);
6554 src2 : ISS(read);
6555 INS01 : ISS;
6556 ALU : EX2;
6557 %}
6558
6559 // Integer ALU reg operation with constant shift
6560 // Eg. LSL x0, x1, #shift
6561 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6562 %{
6563 single_instruction;
6564 dst : EX2(write);
6565 src1 : ISS(read);
6566 INS01 : ISS;
6567 ALU : EX2;
6568 %}
6569
6570 // Integer ALU reg-reg operation with variable shift
6571 // Both operands must be available in LATE_ISS instead of EX1
6572 // Result is available in EX1 instead of EX2
6573 // Eg. LSLV x0, x1, x2
6574 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6575 %{
6576 single_instruction;
6577 dst : EX1(write);
6578 src1 : ISS(read);
6579 src2 : ISS(read);
6580 INS01 : ISS;
6581 ALU : EX1;
6582 %}
6583
6584 // Integer ALU reg-reg operation with extract
6585 // As for _vshift above, but result generated in EX2
6586 // Eg. EXTR x0, x1, x2, #N
6587 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6588 %{
6589 single_instruction;
6590 dst : EX2(write);
6591 src1 : ISS(read);
6592 src2 : ISS(read);
6593 INS1 : ISS; // Can only dual issue as Instruction 1
6594 ALU : EX1;
6595 %}
6596
6597 // Integer ALU reg operation
6598 // Eg. NEG x0, x1
6599 pipe_class ialu_reg(iRegI dst, iRegI src)
6600 %{
6601 single_instruction;
6602 dst : EX2(write);
6603 src : EX1(read);
6604 INS01 : ISS;
6605 ALU : EX2;
6606 %}
6607
6608 // Integer ALU reg mmediate operation
6609 // Eg. ADD x0, x1, #N
6610 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6611 %{
6612 single_instruction;
6613 dst : EX2(write);
6614 src1 : EX1(read);
6615 INS01 : ISS;
6616 ALU : EX2;
6617 %}
6618
6619 // Integer ALU immediate operation (no source operands)
6620 // Eg. MOV x0, #N
6621 pipe_class ialu_imm(iRegI dst)
6622 %{
6623 single_instruction;
6624 dst : EX1(write);
6625 INS01 : ISS;
6626 ALU : EX1;
6627 %}
6628
6629 //------- Compare operation -------------------------------
6630
6631 // Compare reg-reg
6632 // Eg. CMP x0, x1
6633 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6634 %{
6635 single_instruction;
6636 // fixed_latency(16);
6637 cr : EX2(write);
6638 op1 : EX1(read);
6639 op2 : EX1(read);
6640 INS01 : ISS;
6641 ALU : EX2;
6642 %}
6643
6644 // Compare reg-reg
6645 // Eg. CMP x0, #N
6646 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6647 %{
6648 single_instruction;
6649 // fixed_latency(16);
6650 cr : EX2(write);
6651 op1 : EX1(read);
6652 INS01 : ISS;
6653 ALU : EX2;
6654 %}
6655
6656 //------- Conditional instructions ------------------------
6657
6658 // Conditional no operands
6659 // Eg. CSINC x0, zr, zr, <cond>
6660 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6661 %{
6662 single_instruction;
6663 cr : EX1(read);
6664 dst : EX2(write);
6665 INS01 : ISS;
6666 ALU : EX2;
6667 %}
6668
6669 // Conditional 2 operand
6670 // EG. CSEL X0, X1, X2, <cond>
6671 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6672 %{
6673 single_instruction;
6674 cr : EX1(read);
6675 src1 : EX1(read);
6676 src2 : EX1(read);
6677 dst : EX2(write);
6678 INS01 : ISS;
6679 ALU : EX2;
6680 %}
6681
6682 // Conditional 2 operand
6683 // EG. CSEL X0, X1, X2, <cond>
6684 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6685 %{
6686 single_instruction;
6687 cr : EX1(read);
6688 src : EX1(read);
6689 dst : EX2(write);
6690 INS01 : ISS;
6691 ALU : EX2;
6692 %}
6693
6694 //------- Multiply pipeline operations --------------------
6695
6696 // Multiply reg-reg
6697 // Eg. MUL w0, w1, w2
6698 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6699 %{
6700 single_instruction;
6701 dst : WR(write);
6702 src1 : ISS(read);
6703 src2 : ISS(read);
6704 INS01 : ISS;
6705 MAC : WR;
6706 %}
6707
6708 // Multiply accumulate
6709 // Eg. MADD w0, w1, w2, w3
6710 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6711 %{
6712 single_instruction;
6713 dst : WR(write);
6714 src1 : ISS(read);
6715 src2 : ISS(read);
6716 src3 : ISS(read);
6717 INS01 : ISS;
6718 MAC : WR;
6719 %}
6720
6721 // Eg. MUL w0, w1, w2
6722 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6723 %{
6724 single_instruction;
6725 fixed_latency(3); // Maximum latency for 64 bit mul
6726 dst : WR(write);
6727 src1 : ISS(read);
6728 src2 : ISS(read);
6729 INS01 : ISS;
6730 MAC : WR;
6731 %}
6732
6733 // Multiply accumulate
6734 // Eg. MADD w0, w1, w2, w3
6735 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6736 %{
6737 single_instruction;
6738 fixed_latency(3); // Maximum latency for 64 bit mul
6739 dst : WR(write);
6740 src1 : ISS(read);
6741 src2 : ISS(read);
6742 src3 : ISS(read);
6743 INS01 : ISS;
6744 MAC : WR;
6745 %}
6746
6747 //------- Divide pipeline operations --------------------
6748
6749 // Eg. SDIV w0, w1, w2
6750 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6751 %{
6752 single_instruction;
6753 fixed_latency(8); // Maximum latency for 32 bit divide
6754 dst : WR(write);
6755 src1 : ISS(read);
6756 src2 : ISS(read);
6757 INS0 : ISS; // Can only dual issue as instruction 0
6758 DIV : WR;
6759 %}
6760
6761 // Eg. SDIV x0, x1, x2
6762 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6763 %{
6764 single_instruction;
6765 fixed_latency(16); // Maximum latency for 64 bit divide
6766 dst : WR(write);
6767 src1 : ISS(read);
6768 src2 : ISS(read);
6769 INS0 : ISS; // Can only dual issue as instruction 0
6770 DIV : WR;
6771 %}
6772
6773 //------- Load pipeline operations ------------------------
6774
6775 // Load - prefetch
6776 // Eg. PFRM <mem>
6777 pipe_class iload_prefetch(memory mem)
6778 %{
6779 single_instruction;
6780 mem : ISS(read);
6781 INS01 : ISS;
6782 LDST : WR;
6783 %}
6784
6785 // Load - reg, mem
6786 // Eg. LDR x0, <mem>
6787 pipe_class iload_reg_mem(iRegI dst, memory mem)
6788 %{
6789 single_instruction;
6790 dst : WR(write);
6791 mem : ISS(read);
6792 INS01 : ISS;
6793 LDST : WR;
6794 %}
6795
6796 // Load - reg, reg
6797 // Eg. LDR x0, [sp, x1]
6798 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6799 %{
6800 single_instruction;
6801 dst : WR(write);
6802 src : ISS(read);
6803 INS01 : ISS;
6804 LDST : WR;
6805 %}
6806
6807 //------- Store pipeline operations -----------------------
6808
6809 // Store - zr, mem
6810 // Eg. STR zr, <mem>
6811 pipe_class istore_mem(memory mem)
6812 %{
6813 single_instruction;
6814 mem : ISS(read);
6815 INS01 : ISS;
6816 LDST : WR;
6817 %}
6818
6819 // Store - reg, mem
6820 // Eg. STR x0, <mem>
6821 pipe_class istore_reg_mem(iRegI src, memory mem)
6822 %{
6823 single_instruction;
6824 mem : ISS(read);
6825 src : EX2(read);
6826 INS01 : ISS;
6827 LDST : WR;
6828 %}
6829
6830 // Store - reg, reg
6831 // Eg. STR x0, [sp, x1]
6832 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6833 %{
6834 single_instruction;
6835 dst : ISS(read);
6836 src : EX2(read);
6837 INS01 : ISS;
6838 LDST : WR;
6839 %}
6840
6841 //------- Store pipeline operations -----------------------
6842
6843 // Branch
6844 pipe_class pipe_branch()
6845 %{
6846 single_instruction;
6847 INS01 : ISS;
6848 BRANCH : EX1;
6849 %}
6850
6851 // Conditional branch
6852 pipe_class pipe_branch_cond(rFlagsReg cr)
6853 %{
6854 single_instruction;
6855 cr : EX1(read);
6856 INS01 : ISS;
6857 BRANCH : EX1;
6858 %}
6859
6860 // Compare & Branch
6861 // EG. CBZ/CBNZ
6862 pipe_class pipe_cmp_branch(iRegI op1)
6863 %{
6864 single_instruction;
6865 op1 : EX1(read);
6866 INS01 : ISS;
6867 BRANCH : EX1;
6868 %}
6869
6870 //------- Synchronisation operations ----------------------
6871
6872 // Any operation requiring serialization.
6873 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6874 pipe_class pipe_serial()
6875 %{
6876 single_instruction;
6877 force_serialization;
6878 fixed_latency(16);
6879 INS01 : ISS(2); // Cannot dual issue with any other instruction
6880 LDST : WR;
6881 %}
6882
6883 // Generic big/slow expanded idiom - also serialized
6884 pipe_class pipe_slow()
6885 %{
6886 instruction_count(10);
6887 multiple_bundles;
6888 force_serialization;
6889 fixed_latency(16);
6890 INS01 : ISS(2); // Cannot dual issue with any other instruction
6891 LDST : WR;
6892 %}
6893
6894 // Empty pipeline class
6895 pipe_class pipe_class_empty()
6896 %{
6897 single_instruction;
6898 fixed_latency(0);
6899 %}
6900
6901 // Default pipeline class.
6902 pipe_class pipe_class_default()
6903 %{
6904 single_instruction;
6905 fixed_latency(2);
6906 %}
6907
6908 // Pipeline class for compares.
6909 pipe_class pipe_class_compare()
6910 %{
6911 single_instruction;
6912 fixed_latency(16);
6913 %}
6914
6915 // Pipeline class for memory operations.
6916 pipe_class pipe_class_memory()
6917 %{
6918 single_instruction;
6919 fixed_latency(16);
6920 %}
6921
6922 // Pipeline class for call.
6923 pipe_class pipe_class_call()
6924 %{
6925 single_instruction;
6926 fixed_latency(100);
6927 %}
6928
6929 // Define the class for the Nop node.
6930 define %{
6931 MachNop = pipe_class_empty;
6932 %}
6933
6934 %}
6935 //----------INSTRUCTIONS-------------------------------------------------------
6936 //
6937 // match -- States which machine-independent subtree may be replaced
6938 // by this instruction.
6939 // ins_cost -- The estimated cost of this instruction is used by instruction
6940 // selection to identify a minimum cost tree of machine
6941 // instructions that matches a tree of machine-independent
6942 // instructions.
6943 // format -- A string providing the disassembly for this instruction.
6944 // The value of an instruction's operand may be inserted
6945 // by referring to it with a '$' prefix.
6946 // opcode -- Three instruction opcodes may be provided. These are referred
6947 // to within an encode class as $primary, $secondary, and $tertiary
6948 // rrspectively. The primary opcode is commonly used to
6949 // indicate the type of machine instruction, while secondary
6950 // and tertiary are often used for prefix options or addressing
6951 // modes.
6952 // ins_encode -- A list of encode classes with parameters. The encode class
6953 // name must have been defined in an 'enc_class' specification
6954 // in the encode section of the architecture description.
6955
6956 // ============================================================================
6957 // Memory (Load/Store) Instructions
6958
6959 // Load Instructions
6960
6961 // Load Byte (8 bit signed)
6962 instruct loadB(iRegINoSp dst, memory1 mem)
6963 %{
6964 match(Set dst (LoadB mem));
6965 predicate(!needs_acquiring_load(n));
6966
6967 ins_cost(4 * INSN_COST);
6968 format %{ "ldrsbw $dst, $mem\t# byte" %}
6969
6970 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6971
6972 ins_pipe(iload_reg_mem);
6973 %}
6974
6975 // Load Byte (8 bit signed) into long
6976 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6977 %{
6978 match(Set dst (ConvI2L (LoadB mem)));
6979 predicate(!needs_acquiring_load(n->in(1)));
6980
6981 ins_cost(4 * INSN_COST);
6982 format %{ "ldrsb $dst, $mem\t# byte" %}
6983
6984 ins_encode(aarch64_enc_ldrsb(dst, mem));
6985
6986 ins_pipe(iload_reg_mem);
6987 %}
6988
6989 // Load Byte (8 bit unsigned)
6990 instruct loadUB(iRegINoSp dst, memory1 mem)
6991 %{
6992 match(Set dst (LoadUB mem));
6993 predicate(!needs_acquiring_load(n));
6994
6995 ins_cost(4 * INSN_COST);
6996 format %{ "ldrbw $dst, $mem\t# byte" %}
6997
6998 ins_encode(aarch64_enc_ldrb(dst, mem));
6999
7000 ins_pipe(iload_reg_mem);
7001 %}
7002
7003 // Load Byte (8 bit unsigned) into long
7004 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7005 %{
7006 match(Set dst (ConvI2L (LoadUB mem)));
7007 predicate(!needs_acquiring_load(n->in(1)));
7008
7009 ins_cost(4 * INSN_COST);
7010 format %{ "ldrb $dst, $mem\t# byte" %}
7011
7012 ins_encode(aarch64_enc_ldrb(dst, mem));
7013
7014 ins_pipe(iload_reg_mem);
7015 %}
7016
7017 // Load Short (16 bit signed)
7018 instruct loadS(iRegINoSp dst, memory2 mem)
7019 %{
7020 match(Set dst (LoadS mem));
7021 predicate(!needs_acquiring_load(n));
7022
7023 ins_cost(4 * INSN_COST);
7024 format %{ "ldrshw $dst, $mem\t# short" %}
7025
7026 ins_encode(aarch64_enc_ldrshw(dst, mem));
7027
7028 ins_pipe(iload_reg_mem);
7029 %}
7030
7031 // Load Short (16 bit signed) into long
7032 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7033 %{
7034 match(Set dst (ConvI2L (LoadS mem)));
7035 predicate(!needs_acquiring_load(n->in(1)));
7036
7037 ins_cost(4 * INSN_COST);
7038 format %{ "ldrsh $dst, $mem\t# short" %}
7039
7040 ins_encode(aarch64_enc_ldrsh(dst, mem));
7041
7042 ins_pipe(iload_reg_mem);
7043 %}
7044
7045 // Load Char (16 bit unsigned)
7046 instruct loadUS(iRegINoSp dst, memory2 mem)
7047 %{
7048 match(Set dst (LoadUS mem));
7049 predicate(!needs_acquiring_load(n));
7050
7051 ins_cost(4 * INSN_COST);
7052 format %{ "ldrh $dst, $mem\t# short" %}
7053
7054 ins_encode(aarch64_enc_ldrh(dst, mem));
7055
7056 ins_pipe(iload_reg_mem);
7057 %}
7058
7059 // Load Short/Char (16 bit unsigned) into long
7060 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7061 %{
7062 match(Set dst (ConvI2L (LoadUS mem)));
7063 predicate(!needs_acquiring_load(n->in(1)));
7064
7065 ins_cost(4 * INSN_COST);
7066 format %{ "ldrh $dst, $mem\t# short" %}
7067
7068 ins_encode(aarch64_enc_ldrh(dst, mem));
7069
7070 ins_pipe(iload_reg_mem);
7071 %}
7072
7073 // Load Integer (32 bit signed)
7074 instruct loadI(iRegINoSp dst, memory4 mem)
7075 %{
7076 match(Set dst (LoadI mem));
7077 predicate(!needs_acquiring_load(n));
7078
7079 ins_cost(4 * INSN_COST);
7080 format %{ "ldrw $dst, $mem\t# int" %}
7081
7082 ins_encode(aarch64_enc_ldrw(dst, mem));
7083
7084 ins_pipe(iload_reg_mem);
7085 %}
7086
7087 // Load Integer (32 bit signed) into long
7088 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7089 %{
7090 match(Set dst (ConvI2L (LoadI mem)));
7091 predicate(!needs_acquiring_load(n->in(1)));
7092
7093 ins_cost(4 * INSN_COST);
7094 format %{ "ldrsw $dst, $mem\t# int" %}
7095
7096 ins_encode(aarch64_enc_ldrsw(dst, mem));
7097
7098 ins_pipe(iload_reg_mem);
7099 %}
7100
7101 // Load Integer (32 bit unsigned) into long
7102 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7103 %{
7104 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7105 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7106
7107 ins_cost(4 * INSN_COST);
7108 format %{ "ldrw $dst, $mem\t# int" %}
7109
7110 ins_encode(aarch64_enc_ldrw(dst, mem));
7111
7112 ins_pipe(iload_reg_mem);
7113 %}
7114
7115 // Load Long (64 bit signed)
7116 instruct loadL(iRegLNoSp dst, memory8 mem)
7117 %{
7118 match(Set dst (LoadL mem));
7119 predicate(!needs_acquiring_load(n));
7120
7121 ins_cost(4 * INSN_COST);
7122 format %{ "ldr $dst, $mem\t# int" %}
7123
7124 ins_encode(aarch64_enc_ldr(dst, mem));
7125
7126 ins_pipe(iload_reg_mem);
7127 %}
7128
7129 // Load Range
7130 instruct loadRange(iRegINoSp dst, memory4 mem)
7131 %{
7132 match(Set dst (LoadRange mem));
7133
7134 ins_cost(4 * INSN_COST);
7135 format %{ "ldrw $dst, $mem\t# range" %}
7136
7137 ins_encode(aarch64_enc_ldrw(dst, mem));
7138
7139 ins_pipe(iload_reg_mem);
7140 %}
7141
7142 // Load Pointer
7143 instruct loadP(iRegPNoSp dst, memory8 mem)
7144 %{
7145 match(Set dst (LoadP mem));
7146 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7147
7148 ins_cost(4 * INSN_COST);
7149 format %{ "ldr $dst, $mem\t# ptr" %}
7150
7151 ins_encode(aarch64_enc_ldr(dst, mem));
7152
7153 ins_pipe(iload_reg_mem);
7154 %}
7155
7156 // Load Compressed Pointer
7157 instruct loadN(iRegNNoSp dst, memory4 mem)
7158 %{
7159 match(Set dst (LoadN mem));
7160 predicate(!needs_acquiring_load(n));
7161
7162 ins_cost(4 * INSN_COST);
7163 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7164
7165 ins_encode(aarch64_enc_ldrw(dst, mem));
7166
7167 ins_pipe(iload_reg_mem);
7168 %}
7169
7170 // Load Klass Pointer
7171 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7172 %{
7173 match(Set dst (LoadKlass mem));
7174 predicate(!needs_acquiring_load(n));
7175
7176 ins_cost(4 * INSN_COST);
7177 format %{ "ldr $dst, $mem\t# class" %}
7178
7179 ins_encode(aarch64_enc_ldr(dst, mem));
7180
7181 ins_pipe(iload_reg_mem);
7182 %}
7183
7184 // Load Narrow Klass Pointer
7185 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7186 %{
7187 match(Set dst (LoadNKlass mem));
7188 predicate(!needs_acquiring_load(n));
7189
7190 ins_cost(4 * INSN_COST);
7191 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7192
7193 ins_encode(aarch64_enc_ldrw(dst, mem));
7194
7195 ins_pipe(iload_reg_mem);
7196 %}
7197
7198 // Load Float
7199 instruct loadF(vRegF dst, memory4 mem)
7200 %{
7201 match(Set dst (LoadF mem));
7202 predicate(!needs_acquiring_load(n));
7203
7204 ins_cost(4 * INSN_COST);
7205 format %{ "ldrs $dst, $mem\t# float" %}
7206
7207 ins_encode( aarch64_enc_ldrs(dst, mem) );
7208
7209 ins_pipe(pipe_class_memory);
7210 %}
7211
7212 // Load Double
7213 instruct loadD(vRegD dst, memory8 mem)
7214 %{
7215 match(Set dst (LoadD mem));
7216 predicate(!needs_acquiring_load(n));
7217
7218 ins_cost(4 * INSN_COST);
7219 format %{ "ldrd $dst, $mem\t# double" %}
7220
7221 ins_encode( aarch64_enc_ldrd(dst, mem) );
7222
7223 ins_pipe(pipe_class_memory);
7224 %}
7225
7226
7227 // Load Int Constant
7228 instruct loadConI(iRegINoSp dst, immI src)
7229 %{
7230 match(Set dst src);
7231
7232 ins_cost(INSN_COST);
7233 format %{ "mov $dst, $src\t# int" %}
7234
7235 ins_encode( aarch64_enc_movw_imm(dst, src) );
7236
7237 ins_pipe(ialu_imm);
7238 %}
7239
7240 // Load Long Constant
7241 instruct loadConL(iRegLNoSp dst, immL src)
7242 %{
7243 match(Set dst src);
7244
7245 ins_cost(INSN_COST);
7246 format %{ "mov $dst, $src\t# long" %}
7247
7248 ins_encode( aarch64_enc_mov_imm(dst, src) );
7249
7250 ins_pipe(ialu_imm);
7251 %}
7252
7253 // Load Pointer Constant
7254
7255 instruct loadConP(iRegPNoSp dst, immP con)
7256 %{
7257 match(Set dst con);
7258
7259 ins_cost(INSN_COST * 4);
7260 format %{
7261 "mov $dst, $con\t# ptr\n\t"
7262 %}
7263
7264 ins_encode(aarch64_enc_mov_p(dst, con));
7265
7266 ins_pipe(ialu_imm);
7267 %}
7268
7269 // Load Null Pointer Constant
7270
7271 instruct loadConP0(iRegPNoSp dst, immP0 con)
7272 %{
7273 match(Set dst con);
7274
7275 ins_cost(INSN_COST);
7276 format %{ "mov $dst, $con\t# NULL ptr" %}
7277
7278 ins_encode(aarch64_enc_mov_p0(dst, con));
7279
7280 ins_pipe(ialu_imm);
7281 %}
7282
7283 // Load Pointer Constant One
7284
7285 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7286 %{
7287 match(Set dst con);
7288
7289 ins_cost(INSN_COST);
7290 format %{ "mov $dst, $con\t# NULL ptr" %}
7291
7292 ins_encode(aarch64_enc_mov_p1(dst, con));
7293
7294 ins_pipe(ialu_imm);
7295 %}
7296
7297 // Load Byte Map Base Constant
7298
7299 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7300 %{
7301 match(Set dst con);
7302
7303 ins_cost(INSN_COST);
7304 format %{ "adr $dst, $con\t# Byte Map Base" %}
7305
7306 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7307
7308 ins_pipe(ialu_imm);
7309 %}
7310
7311 // Load Narrow Pointer Constant
7312
7313 instruct loadConN(iRegNNoSp dst, immN con)
7314 %{
7315 match(Set dst con);
7316
7317 ins_cost(INSN_COST * 4);
7318 format %{ "mov $dst, $con\t# compressed ptr" %}
7319
7320 ins_encode(aarch64_enc_mov_n(dst, con));
7321
7322 ins_pipe(ialu_imm);
7323 %}
7324
7325 // Load Narrow Null Pointer Constant
7326
7327 instruct loadConN0(iRegNNoSp dst, immN0 con)
7328 %{
7329 match(Set dst con);
7330
7331 ins_cost(INSN_COST);
7332 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7333
7334 ins_encode(aarch64_enc_mov_n0(dst, con));
7335
7336 ins_pipe(ialu_imm);
7337 %}
7338
7339 // Load Narrow Klass Constant
7340
7341 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7342 %{
7343 match(Set dst con);
7344
7345 ins_cost(INSN_COST);
7346 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7347
7348 ins_encode(aarch64_enc_mov_nk(dst, con));
7349
7350 ins_pipe(ialu_imm);
7351 %}
7352
7353 // Load Packed Float Constant
7354
7355 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7356 match(Set dst con);
7357 ins_cost(INSN_COST * 4);
7358 format %{ "fmovs $dst, $con"%}
7359 ins_encode %{
7360 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7361 %}
7362
7363 ins_pipe(fp_imm_s);
7364 %}
7365
7366 // Load Float Constant
7367
7368 instruct loadConF(vRegF dst, immF con) %{
7369 match(Set dst con);
7370
7371 ins_cost(INSN_COST * 4);
7372
7373 format %{
7374 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7375 %}
7376
7377 ins_encode %{
7378 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7379 %}
7380
7381 ins_pipe(fp_load_constant_s);
7382 %}
7383
7384 // Load Packed Double Constant
7385
7386 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7387 match(Set dst con);
7388 ins_cost(INSN_COST);
7389 format %{ "fmovd $dst, $con"%}
7390 ins_encode %{
7391 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7392 %}
7393
7394 ins_pipe(fp_imm_d);
7395 %}
7396
7397 // Load Double Constant
7398
7399 instruct loadConD(vRegD dst, immD con) %{
7400 match(Set dst con);
7401
7402 ins_cost(INSN_COST * 5);
7403 format %{
7404 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7405 %}
7406
7407 ins_encode %{
7408 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7409 %}
7410
7411 ins_pipe(fp_load_constant_d);
7412 %}
7413
7414 // Store Instructions
7415
7416 // Store CMS card-mark Immediate
7417 instruct storeimmCM0(immI0 zero, memory1 mem)
7418 %{
7419 match(Set mem (StoreCM mem zero));
7420
7421 ins_cost(INSN_COST);
7422 format %{ "storestore (elided)\n\t"
7423 "strb zr, $mem\t# byte" %}
7424
7425 ins_encode(aarch64_enc_strb0(mem));
7426
7427 ins_pipe(istore_mem);
7428 %}
7429
7430 // Store CMS card-mark Immediate with intervening StoreStore
7431 // needed when using CMS with no conditional card marking
7432 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7433 %{
7434 match(Set mem (StoreCM mem zero));
7435
7436 ins_cost(INSN_COST * 2);
7437 format %{ "storestore\n\t"
7438 "dmb ishst"
7439 "\n\tstrb zr, $mem\t# byte" %}
7440
7441 ins_encode(aarch64_enc_strb0_ordered(mem));
7442
7443 ins_pipe(istore_mem);
7444 %}
7445
7446 // Store Byte
7447 instruct storeB(iRegIorL2I src, memory1 mem)
7448 %{
7449 match(Set mem (StoreB mem src));
7450 predicate(!needs_releasing_store(n));
7451
7452 ins_cost(INSN_COST);
7453 format %{ "strb $src, $mem\t# byte" %}
7454
7455 ins_encode(aarch64_enc_strb(src, mem));
7456
7457 ins_pipe(istore_reg_mem);
7458 %}
7459
7460
7461 instruct storeimmB0(immI0 zero, memory1 mem)
7462 %{
7463 match(Set mem (StoreB mem zero));
7464 predicate(!needs_releasing_store(n));
7465
7466 ins_cost(INSN_COST);
7467 format %{ "strb rscractch2, $mem\t# byte" %}
7468
7469 ins_encode(aarch64_enc_strb0(mem));
7470
7471 ins_pipe(istore_mem);
7472 %}
7473
7474 // Store Char/Short
7475 instruct storeC(iRegIorL2I src, memory2 mem)
7476 %{
7477 match(Set mem (StoreC mem src));
7478 predicate(!needs_releasing_store(n));
7479
7480 ins_cost(INSN_COST);
7481 format %{ "strh $src, $mem\t# short" %}
7482
7483 ins_encode(aarch64_enc_strh(src, mem));
7484
7485 ins_pipe(istore_reg_mem);
7486 %}
7487
7488 instruct storeimmC0(immI0 zero, memory2 mem)
7489 %{
7490 match(Set mem (StoreC mem zero));
7491 predicate(!needs_releasing_store(n));
7492
7493 ins_cost(INSN_COST);
7494 format %{ "strh zr, $mem\t# short" %}
7495
7496 ins_encode(aarch64_enc_strh0(mem));
7497
7498 ins_pipe(istore_mem);
7499 %}
7500
7501 // Store Integer
7502
7503 instruct storeI(iRegIorL2I src, memory4 mem)
7504 %{
7505 match(Set mem(StoreI mem src));
7506 predicate(!needs_releasing_store(n));
7507
7508 ins_cost(INSN_COST);
7509 format %{ "strw $src, $mem\t# int" %}
7510
7511 ins_encode(aarch64_enc_strw(src, mem));
7512
7513 ins_pipe(istore_reg_mem);
7514 %}
7515
7516 instruct storeimmI0(immI0 zero, memory4 mem)
7517 %{
7518 match(Set mem(StoreI mem zero));
7519 predicate(!needs_releasing_store(n));
7520
7521 ins_cost(INSN_COST);
7522 format %{ "strw zr, $mem\t# int" %}
7523
7524 ins_encode(aarch64_enc_strw0(mem));
7525
7526 ins_pipe(istore_mem);
7527 %}
7528
7529 // Store Long (64 bit signed)
7530 instruct storeL(iRegL src, memory8 mem)
7531 %{
7532 match(Set mem (StoreL mem src));
7533 predicate(!needs_releasing_store(n));
7534
7535 ins_cost(INSN_COST);
7536 format %{ "str $src, $mem\t# int" %}
7537
7538 ins_encode(aarch64_enc_str(src, mem));
7539
7540 ins_pipe(istore_reg_mem);
7541 %}
7542
7543 // Store Long (64 bit signed)
7544 instruct storeimmL0(immL0 zero, memory8 mem)
7545 %{
7546 match(Set mem (StoreL mem zero));
7547 predicate(!needs_releasing_store(n));
7548
7549 ins_cost(INSN_COST);
7550 format %{ "str zr, $mem\t# int" %}
7551
7552 ins_encode(aarch64_enc_str0(mem));
7553
7554 ins_pipe(istore_mem);
7555 %}
7556
7557 // Store Pointer
7558 instruct storeP(iRegP src, memory8 mem)
7559 %{
7560 match(Set mem (StoreP mem src));
7561 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7562
7563 ins_cost(INSN_COST);
7564 format %{ "str $src, $mem\t# ptr" %}
7565
7566 ins_encode(aarch64_enc_str(src, mem));
7567
7568 ins_pipe(istore_reg_mem);
7569 %}
7570
7571 // Store Pointer
7572 instruct storeimmP0(immP0 zero, memory8 mem)
7573 %{
7574 match(Set mem (StoreP mem zero));
7575 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7576
7577 ins_cost(INSN_COST);
7578 format %{ "str zr, $mem\t# ptr" %}
7579
7580 ins_encode(aarch64_enc_str0(mem));
7581
7582 ins_pipe(istore_mem);
7583 %}
7584
7585 // Store Compressed Pointer
7586 instruct storeN(iRegN src, memory4 mem)
7587 %{
7588 match(Set mem (StoreN mem src));
7589 predicate(!needs_releasing_store(n));
7590
7591 ins_cost(INSN_COST);
7592 format %{ "strw $src, $mem\t# compressed ptr" %}
7593
7594 ins_encode(aarch64_enc_strw(src, mem));
7595
7596 ins_pipe(istore_reg_mem);
7597 %}
7598
7599 instruct storeImmN0(immN0 zero, memory4 mem)
7600 %{
7601 match(Set mem (StoreN mem zero));
7602 predicate(!needs_releasing_store(n));
7603
7604 ins_cost(INSN_COST);
7605 format %{ "strw zr, $mem\t# compressed ptr" %}
7606
7607 ins_encode(aarch64_enc_strw0(mem));
7608
7609 ins_pipe(istore_mem);
7610 %}
7611
7612 // Store Float
7613 instruct storeF(vRegF src, memory4 mem)
7614 %{
7615 match(Set mem (StoreF mem src));
7616 predicate(!needs_releasing_store(n));
7617
7618 ins_cost(INSN_COST);
7619 format %{ "strs $src, $mem\t# float" %}
7620
7621 ins_encode( aarch64_enc_strs(src, mem) );
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 // TODO
7627 // implement storeImmF0 and storeFImmPacked
7628
7629 // Store Double
7630 instruct storeD(vRegD src, memory8 mem)
7631 %{
7632 match(Set mem (StoreD mem src));
7633 predicate(!needs_releasing_store(n));
7634
7635 ins_cost(INSN_COST);
7636 format %{ "strd $src, $mem\t# double" %}
7637
7638 ins_encode( aarch64_enc_strd(src, mem) );
7639
7640 ins_pipe(pipe_class_memory);
7641 %}
7642
7643 // Store Compressed Klass Pointer
7644 instruct storeNKlass(iRegN src, memory4 mem)
7645 %{
7646 predicate(!needs_releasing_store(n));
7647 match(Set mem (StoreNKlass mem src));
7648
7649 ins_cost(INSN_COST);
7650 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7651
7652 ins_encode(aarch64_enc_strw(src, mem));
7653
7654 ins_pipe(istore_reg_mem);
7655 %}
7656
7657 // TODO
7658 // implement storeImmD0 and storeDImmPacked
7659
7660 // prefetch instructions
7661 // Must be safe to execute with invalid address (cannot fault).
7662
7663 instruct prefetchalloc( memory8 mem ) %{
7664 match(PrefetchAllocation mem);
7665
7666 ins_cost(INSN_COST);
7667 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7668
7669 ins_encode( aarch64_enc_prefetchw(mem) );
7670
7671 ins_pipe(iload_prefetch);
7672 %}
7673
7674 // ---------------- volatile loads and stores ----------------
7675
7676 // Load Byte (8 bit signed)
7677 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7678 %{
7679 match(Set dst (LoadB mem));
7680
7681 ins_cost(VOLATILE_REF_COST);
7682 format %{ "ldarsb $dst, $mem\t# byte" %}
7683
7684 ins_encode(aarch64_enc_ldarsb(dst, mem));
7685
7686 ins_pipe(pipe_serial);
7687 %}
7688
7689 // Load Byte (8 bit signed) into long
7690 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7691 %{
7692 match(Set dst (ConvI2L (LoadB mem)));
7693
7694 ins_cost(VOLATILE_REF_COST);
7695 format %{ "ldarsb $dst, $mem\t# byte" %}
7696
7697 ins_encode(aarch64_enc_ldarsb(dst, mem));
7698
7699 ins_pipe(pipe_serial);
7700 %}
7701
7702 // Load Byte (8 bit unsigned)
7703 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7704 %{
7705 match(Set dst (LoadUB mem));
7706
7707 ins_cost(VOLATILE_REF_COST);
7708 format %{ "ldarb $dst, $mem\t# byte" %}
7709
7710 ins_encode(aarch64_enc_ldarb(dst, mem));
7711
7712 ins_pipe(pipe_serial);
7713 %}
7714
7715 // Load Byte (8 bit unsigned) into long
7716 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7717 %{
7718 match(Set dst (ConvI2L (LoadUB mem)));
7719
7720 ins_cost(VOLATILE_REF_COST);
7721 format %{ "ldarb $dst, $mem\t# byte" %}
7722
7723 ins_encode(aarch64_enc_ldarb(dst, mem));
7724
7725 ins_pipe(pipe_serial);
7726 %}
7727
7728 // Load Short (16 bit signed)
7729 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7730 %{
7731 match(Set dst (LoadS mem));
7732
7733 ins_cost(VOLATILE_REF_COST);
7734 format %{ "ldarshw $dst, $mem\t# short" %}
7735
7736 ins_encode(aarch64_enc_ldarshw(dst, mem));
7737
7738 ins_pipe(pipe_serial);
7739 %}
7740
7741 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7742 %{
7743 match(Set dst (LoadUS mem));
7744
7745 ins_cost(VOLATILE_REF_COST);
7746 format %{ "ldarhw $dst, $mem\t# short" %}
7747
7748 ins_encode(aarch64_enc_ldarhw(dst, mem));
7749
7750 ins_pipe(pipe_serial);
7751 %}
7752
7753 // Load Short/Char (16 bit unsigned) into long
7754 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7755 %{
7756 match(Set dst (ConvI2L (LoadUS mem)));
7757
7758 ins_cost(VOLATILE_REF_COST);
7759 format %{ "ldarh $dst, $mem\t# short" %}
7760
7761 ins_encode(aarch64_enc_ldarh(dst, mem));
7762
7763 ins_pipe(pipe_serial);
7764 %}
7765
7766 // Load Short/Char (16 bit signed) into long
7767 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7768 %{
7769 match(Set dst (ConvI2L (LoadS mem)));
7770
7771 ins_cost(VOLATILE_REF_COST);
7772 format %{ "ldarh $dst, $mem\t# short" %}
7773
7774 ins_encode(aarch64_enc_ldarsh(dst, mem));
7775
7776 ins_pipe(pipe_serial);
7777 %}
7778
7779 // Load Integer (32 bit signed)
7780 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7781 %{
7782 match(Set dst (LoadI mem));
7783
7784 ins_cost(VOLATILE_REF_COST);
7785 format %{ "ldarw $dst, $mem\t# int" %}
7786
7787 ins_encode(aarch64_enc_ldarw(dst, mem));
7788
7789 ins_pipe(pipe_serial);
7790 %}
7791
7792 // Load Integer (32 bit unsigned) into long
7793 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7794 %{
7795 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7796
7797 ins_cost(VOLATILE_REF_COST);
7798 format %{ "ldarw $dst, $mem\t# int" %}
7799
7800 ins_encode(aarch64_enc_ldarw(dst, mem));
7801
7802 ins_pipe(pipe_serial);
7803 %}
7804
7805 // Load Long (64 bit signed)
7806 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7807 %{
7808 match(Set dst (LoadL mem));
7809
7810 ins_cost(VOLATILE_REF_COST);
7811 format %{ "ldar $dst, $mem\t# int" %}
7812
7813 ins_encode(aarch64_enc_ldar(dst, mem));
7814
7815 ins_pipe(pipe_serial);
7816 %}
7817
7818 // Load Pointer
7819 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7820 %{
7821 match(Set dst (LoadP mem));
7822 predicate(n->as_Load()->barrier_data() == 0);
7823
7824 ins_cost(VOLATILE_REF_COST);
7825 format %{ "ldar $dst, $mem\t# ptr" %}
7826
7827 ins_encode(aarch64_enc_ldar(dst, mem));
7828
7829 ins_pipe(pipe_serial);
7830 %}
7831
7832 // Load Compressed Pointer
7833 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7834 %{
7835 match(Set dst (LoadN mem));
7836
7837 ins_cost(VOLATILE_REF_COST);
7838 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7839
7840 ins_encode(aarch64_enc_ldarw(dst, mem));
7841
7842 ins_pipe(pipe_serial);
7843 %}
7844
7845 // Load Float
7846 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7847 %{
7848 match(Set dst (LoadF mem));
7849
7850 ins_cost(VOLATILE_REF_COST);
7851 format %{ "ldars $dst, $mem\t# float" %}
7852
7853 ins_encode( aarch64_enc_fldars(dst, mem) );
7854
7855 ins_pipe(pipe_serial);
7856 %}
7857
7858 // Load Double
7859 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7860 %{
7861 match(Set dst (LoadD mem));
7862
7863 ins_cost(VOLATILE_REF_COST);
7864 format %{ "ldard $dst, $mem\t# double" %}
7865
7866 ins_encode( aarch64_enc_fldard(dst, mem) );
7867
7868 ins_pipe(pipe_serial);
7869 %}
7870
7871 // Store Byte
7872 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7873 %{
7874 match(Set mem (StoreB mem src));
7875
7876 ins_cost(VOLATILE_REF_COST);
7877 format %{ "stlrb $src, $mem\t# byte" %}
7878
7879 ins_encode(aarch64_enc_stlrb(src, mem));
7880
7881 ins_pipe(pipe_class_memory);
7882 %}
7883
7884 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7885 %{
7886 match(Set mem (StoreB mem zero));
7887
7888 ins_cost(VOLATILE_REF_COST);
7889 format %{ "stlrb zr, $mem\t# byte" %}
7890
7891 ins_encode(aarch64_enc_stlrb0(mem));
7892
7893 ins_pipe(pipe_class_memory);
7894 %}
7895
7896 // Store Char/Short
7897 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7898 %{
7899 match(Set mem (StoreC mem src));
7900
7901 ins_cost(VOLATILE_REF_COST);
7902 format %{ "stlrh $src, $mem\t# short" %}
7903
7904 ins_encode(aarch64_enc_stlrh(src, mem));
7905
7906 ins_pipe(pipe_class_memory);
7907 %}
7908
7909 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7910 %{
7911 match(Set mem (StoreC mem zero));
7912
7913 ins_cost(VOLATILE_REF_COST);
7914 format %{ "stlrh zr, $mem\t# short" %}
7915
7916 ins_encode(aarch64_enc_stlrh0(mem));
7917
7918 ins_pipe(pipe_class_memory);
7919 %}
7920
7921 // Store Integer
7922
7923 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7924 %{
7925 match(Set mem(StoreI mem src));
7926
7927 ins_cost(VOLATILE_REF_COST);
7928 format %{ "stlrw $src, $mem\t# int" %}
7929
7930 ins_encode(aarch64_enc_stlrw(src, mem));
7931
7932 ins_pipe(pipe_class_memory);
7933 %}
7934
7935 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7936 %{
7937 match(Set mem(StoreI mem zero));
7938
7939 ins_cost(VOLATILE_REF_COST);
7940 format %{ "stlrw zr, $mem\t# int" %}
7941
7942 ins_encode(aarch64_enc_stlrw0(mem));
7943
7944 ins_pipe(pipe_class_memory);
7945 %}
7946
7947 // Store Long (64 bit signed)
7948 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7949 %{
7950 match(Set mem (StoreL mem src));
7951
7952 ins_cost(VOLATILE_REF_COST);
7953 format %{ "stlr $src, $mem\t# int" %}
7954
7955 ins_encode(aarch64_enc_stlr(src, mem));
7956
7957 ins_pipe(pipe_class_memory);
7958 %}
7959
7960 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7961 %{
7962 match(Set mem (StoreL mem zero));
7963
7964 ins_cost(VOLATILE_REF_COST);
7965 format %{ "stlr zr, $mem\t# int" %}
7966
7967 ins_encode(aarch64_enc_stlr0(mem));
7968
7969 ins_pipe(pipe_class_memory);
7970 %}
7971
7972 // Store Pointer
7973 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7974 %{
7975 match(Set mem (StoreP mem src));
7976 predicate(n->as_Store()->barrier_data() == 0);
7977
7978 ins_cost(VOLATILE_REF_COST);
7979 format %{ "stlr $src, $mem\t# ptr" %}
7980
7981 ins_encode(aarch64_enc_stlr(src, mem));
7982
7983 ins_pipe(pipe_class_memory);
7984 %}
7985
7986 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7987 %{
7988 match(Set mem (StoreP mem zero));
7989 predicate(n->as_Store()->barrier_data() == 0);
7990
7991 ins_cost(VOLATILE_REF_COST);
7992 format %{ "stlr zr, $mem\t# ptr" %}
7993
7994 ins_encode(aarch64_enc_stlr0(mem));
7995
7996 ins_pipe(pipe_class_memory);
7997 %}
7998
7999 // Store Compressed Pointer
8000 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8001 %{
8002 match(Set mem (StoreN mem src));
8003
8004 ins_cost(VOLATILE_REF_COST);
8005 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8006
8007 ins_encode(aarch64_enc_stlrw(src, mem));
8008
8009 ins_pipe(pipe_class_memory);
8010 %}
8011
8012 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8013 %{
8014 match(Set mem (StoreN mem zero));
8015
8016 ins_cost(VOLATILE_REF_COST);
8017 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8018
8019 ins_encode(aarch64_enc_stlrw0(mem));
8020
8021 ins_pipe(pipe_class_memory);
8022 %}
8023
8024 // Store Float
8025 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8026 %{
8027 match(Set mem (StoreF mem src));
8028
8029 ins_cost(VOLATILE_REF_COST);
8030 format %{ "stlrs $src, $mem\t# float" %}
8031
8032 ins_encode( aarch64_enc_fstlrs(src, mem) );
8033
8034 ins_pipe(pipe_class_memory);
8035 %}
8036
8037 // TODO
8038 // implement storeImmF0 and storeFImmPacked
8039
8040 // Store Double
8041 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8042 %{
8043 match(Set mem (StoreD mem src));
8044
8045 ins_cost(VOLATILE_REF_COST);
8046 format %{ "stlrd $src, $mem\t# double" %}
8047
8048 ins_encode( aarch64_enc_fstlrd(src, mem) );
8049
8050 ins_pipe(pipe_class_memory);
8051 %}
8052
8053 // ---------------- end of volatile loads and stores ----------------
8054
8055 instruct cacheWB(indirect addr)
8056 %{
8057 predicate(VM_Version::supports_data_cache_line_flush());
8058 match(CacheWB addr);
8059
8060 ins_cost(100);
8061 format %{"cache wb $addr" %}
8062 ins_encode %{
8063 assert($addr->index_position() < 0, "should be");
8064 assert($addr$$disp == 0, "should be");
8065 __ cache_wb(Address($addr$$base$$Register, 0));
8066 %}
8067 ins_pipe(pipe_slow); // XXX
8068 %}
8069
8070 instruct cacheWBPreSync()
8071 %{
8072 predicate(VM_Version::supports_data_cache_line_flush());
8073 match(CacheWBPreSync);
8074
8075 ins_cost(100);
8076 format %{"cache wb presync" %}
8077 ins_encode %{
8078 __ cache_wbsync(true);
8079 %}
8080 ins_pipe(pipe_slow); // XXX
8081 %}
8082
8083 instruct cacheWBPostSync()
8084 %{
8085 predicate(VM_Version::supports_data_cache_line_flush());
8086 match(CacheWBPostSync);
8087
8088 ins_cost(100);
8089 format %{"cache wb postsync" %}
8090 ins_encode %{
8091 __ cache_wbsync(false);
8092 %}
8093 ins_pipe(pipe_slow); // XXX
8094 %}
8095
8096 // ============================================================================
8097 // BSWAP Instructions
8098
8099 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8100 match(Set dst (ReverseBytesI src));
8101
8102 ins_cost(INSN_COST);
8103 format %{ "revw $dst, $src" %}
8104
8105 ins_encode %{
8106 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8107 %}
8108
8109 ins_pipe(ialu_reg);
8110 %}
8111
8112 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8113 match(Set dst (ReverseBytesL src));
8114
8115 ins_cost(INSN_COST);
8116 format %{ "rev $dst, $src" %}
8117
8118 ins_encode %{
8119 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8120 %}
8121
8122 ins_pipe(ialu_reg);
8123 %}
8124
8125 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8126 match(Set dst (ReverseBytesUS src));
8127
8128 ins_cost(INSN_COST);
8129 format %{ "rev16w $dst, $src" %}
8130
8131 ins_encode %{
8132 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8133 %}
8134
8135 ins_pipe(ialu_reg);
8136 %}
8137
8138 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8139 match(Set dst (ReverseBytesS src));
8140
8141 ins_cost(INSN_COST);
8142 format %{ "rev16w $dst, $src\n\t"
8143 "sbfmw $dst, $dst, #0, #15" %}
8144
8145 ins_encode %{
8146 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8147 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8148 %}
8149
8150 ins_pipe(ialu_reg);
8151 %}
8152
8153 // ============================================================================
8154 // Zero Count Instructions
8155
8156 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8157 match(Set dst (CountLeadingZerosI src));
8158
8159 ins_cost(INSN_COST);
8160 format %{ "clzw $dst, $src" %}
8161 ins_encode %{
8162 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8163 %}
8164
8165 ins_pipe(ialu_reg);
8166 %}
8167
8168 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8169 match(Set dst (CountLeadingZerosL src));
8170
8171 ins_cost(INSN_COST);
8172 format %{ "clz $dst, $src" %}
8173 ins_encode %{
8174 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8175 %}
8176
8177 ins_pipe(ialu_reg);
8178 %}
8179
8180 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8181 match(Set dst (CountTrailingZerosI src));
8182
8183 ins_cost(INSN_COST * 2);
8184 format %{ "rbitw $dst, $src\n\t"
8185 "clzw $dst, $dst" %}
8186 ins_encode %{
8187 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8188 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8189 %}
8190
8191 ins_pipe(ialu_reg);
8192 %}
8193
8194 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8195 match(Set dst (CountTrailingZerosL src));
8196
8197 ins_cost(INSN_COST * 2);
8198 format %{ "rbit $dst, $src\n\t"
8199 "clz $dst, $dst" %}
8200 ins_encode %{
8201 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8202 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8203 %}
8204
8205 ins_pipe(ialu_reg);
8206 %}
8207
8208 //---------- Population Count Instructions -------------------------------------
8209 //
8210
8211 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8212 match(Set dst (PopCountI src));
8213 effect(TEMP tmp);
8214 ins_cost(INSN_COST * 13);
8215
8216 format %{ "movw $src, $src\n\t"
8217 "mov $tmp, $src\t# vector (1D)\n\t"
8218 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8219 "addv $tmp, $tmp\t# vector (8B)\n\t"
8220 "mov $dst, $tmp\t# vector (1D)" %}
8221 ins_encode %{
8222 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8223 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8224 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8225 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8226 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8227 %}
8228
8229 ins_pipe(pipe_class_default);
8230 %}
8231
8232 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8233 match(Set dst (PopCountI (LoadI mem)));
8234 effect(TEMP tmp);
8235 ins_cost(INSN_COST * 13);
8236
8237 format %{ "ldrs $tmp, $mem\n\t"
8238 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8239 "addv $tmp, $tmp\t# vector (8B)\n\t"
8240 "mov $dst, $tmp\t# vector (1D)" %}
8241 ins_encode %{
8242 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8243 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8244 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8245 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8246 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8247 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8248 %}
8249
8250 ins_pipe(pipe_class_default);
8251 %}
8252
8253 // Note: Long.bitCount(long) returns an int.
8254 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8255 match(Set dst (PopCountL src));
8256 effect(TEMP tmp);
8257 ins_cost(INSN_COST * 13);
8258
8259 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8260 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8261 "addv $tmp, $tmp\t# vector (8B)\n\t"
8262 "mov $dst, $tmp\t# vector (1D)" %}
8263 ins_encode %{
8264 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8265 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8266 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8267 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8268 %}
8269
8270 ins_pipe(pipe_class_default);
8271 %}
8272
8273 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8274 match(Set dst (PopCountL (LoadL mem)));
8275 effect(TEMP tmp);
8276 ins_cost(INSN_COST * 13);
8277
8278 format %{ "ldrd $tmp, $mem\n\t"
8279 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8280 "addv $tmp, $tmp\t# vector (8B)\n\t"
8281 "mov $dst, $tmp\t# vector (1D)" %}
8282 ins_encode %{
8283 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8284 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8285 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8286 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8287 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8288 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8289 %}
8290
8291 ins_pipe(pipe_class_default);
8292 %}
8293
8294 // ============================================================================
8295 // MemBar Instruction
8296
8297 instruct load_fence() %{
8298 match(LoadFence);
8299 ins_cost(VOLATILE_REF_COST);
8300
8301 format %{ "load_fence" %}
8302
8303 ins_encode %{
8304 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8305 %}
8306 ins_pipe(pipe_serial);
8307 %}
8308
8309 instruct unnecessary_membar_acquire() %{
8310 predicate(unnecessary_acquire(n));
8311 match(MemBarAcquire);
8312 ins_cost(0);
8313
8314 format %{ "membar_acquire (elided)" %}
8315
8316 ins_encode %{
8317 __ block_comment("membar_acquire (elided)");
8318 %}
8319
8320 ins_pipe(pipe_class_empty);
8321 %}
8322
8323 instruct membar_acquire() %{
8324 match(MemBarAcquire);
8325 ins_cost(VOLATILE_REF_COST);
8326
8327 format %{ "membar_acquire\n\t"
8328 "dmb ish" %}
8329
8330 ins_encode %{
8331 __ block_comment("membar_acquire");
8332 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8333 %}
8334
8335 ins_pipe(pipe_serial);
8336 %}
8337
8338
8339 instruct membar_acquire_lock() %{
8340 match(MemBarAcquireLock);
8341 ins_cost(VOLATILE_REF_COST);
8342
8343 format %{ "membar_acquire_lock (elided)" %}
8344
8345 ins_encode %{
8346 __ block_comment("membar_acquire_lock (elided)");
8347 %}
8348
8349 ins_pipe(pipe_serial);
8350 %}
8351
8352 instruct store_fence() %{
8353 match(StoreFence);
8354 ins_cost(VOLATILE_REF_COST);
8355
8356 format %{ "store_fence" %}
8357
8358 ins_encode %{
8359 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8360 %}
8361 ins_pipe(pipe_serial);
8362 %}
8363
8364 instruct unnecessary_membar_release() %{
8365 predicate(unnecessary_release(n));
8366 match(MemBarRelease);
8367 ins_cost(0);
8368
8369 format %{ "membar_release (elided)" %}
8370
8371 ins_encode %{
8372 __ block_comment("membar_release (elided)");
8373 %}
8374 ins_pipe(pipe_serial);
8375 %}
8376
8377 instruct membar_release() %{
8378 match(MemBarRelease);
8379 ins_cost(VOLATILE_REF_COST);
8380
8381 format %{ "membar_release\n\t"
8382 "dmb ish" %}
8383
8384 ins_encode %{
8385 __ block_comment("membar_release");
8386 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8387 %}
8388 ins_pipe(pipe_serial);
8389 %}
8390
8391 instruct membar_storestore() %{
8392 match(MemBarStoreStore);
8393 match(StoreStoreFence);
8394 ins_cost(VOLATILE_REF_COST);
8395
8396 format %{ "MEMBAR-store-store" %}
8397
8398 ins_encode %{
8399 __ membar(Assembler::StoreStore);
8400 %}
8401 ins_pipe(pipe_serial);
8402 %}
8403
8404 instruct membar_release_lock() %{
8405 match(MemBarReleaseLock);
8406 ins_cost(VOLATILE_REF_COST);
8407
8408 format %{ "membar_release_lock (elided)" %}
8409
8410 ins_encode %{
8411 __ block_comment("membar_release_lock (elided)");
8412 %}
8413
8414 ins_pipe(pipe_serial);
8415 %}
8416
8417 instruct unnecessary_membar_volatile() %{
8418 predicate(unnecessary_volatile(n));
8419 match(MemBarVolatile);
8420 ins_cost(0);
8421
8422 format %{ "membar_volatile (elided)" %}
8423
8424 ins_encode %{
8425 __ block_comment("membar_volatile (elided)");
8426 %}
8427
8428 ins_pipe(pipe_serial);
8429 %}
8430
8431 instruct membar_volatile() %{
8432 match(MemBarVolatile);
8433 ins_cost(VOLATILE_REF_COST*100);
8434
8435 format %{ "membar_volatile\n\t"
8436 "dmb ish"%}
8437
8438 ins_encode %{
8439 __ block_comment("membar_volatile");
8440 __ membar(Assembler::StoreLoad);
8441 %}
8442
8443 ins_pipe(pipe_serial);
8444 %}
8445
8446 // ============================================================================
8447 // Cast/Convert Instructions
8448
8449 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8450 match(Set dst (CastX2P src));
8451
8452 ins_cost(INSN_COST);
8453 format %{ "mov $dst, $src\t# long -> ptr" %}
8454
8455 ins_encode %{
8456 if ($dst$$reg != $src$$reg) {
8457 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8458 }
8459 %}
8460
8461 ins_pipe(ialu_reg);
8462 %}
8463
8464 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8465 match(Set dst (CastP2X src));
8466
8467 ins_cost(INSN_COST);
8468 format %{ "mov $dst, $src\t# ptr -> long" %}
8469
8470 ins_encode %{
8471 if ($dst$$reg != $src$$reg) {
8472 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8473 }
8474 %}
8475
8476 ins_pipe(ialu_reg);
8477 %}
8478
8479 // Convert oop into int for vectors alignment masking
8480 instruct convP2I(iRegINoSp dst, iRegP src) %{
8481 match(Set dst (ConvL2I (CastP2X src)));
8482
8483 ins_cost(INSN_COST);
8484 format %{ "movw $dst, $src\t# ptr -> int" %}
8485 ins_encode %{
8486 __ movw($dst$$Register, $src$$Register);
8487 %}
8488
8489 ins_pipe(ialu_reg);
8490 %}
8491
8492 // Convert compressed oop into int for vectors alignment masking
8493 // in case of 32bit oops (heap < 4Gb).
8494 instruct convN2I(iRegINoSp dst, iRegN src)
8495 %{
8496 predicate(CompressedOops::shift() == 0);
8497 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8498
8499 ins_cost(INSN_COST);
8500 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8501 ins_encode %{
8502 __ movw($dst$$Register, $src$$Register);
8503 %}
8504
8505 ins_pipe(ialu_reg);
8506 %}
8507
8508
8509 // Convert oop pointer into compressed form
8510 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8511 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8512 match(Set dst (EncodeP src));
8513 effect(KILL cr);
8514 ins_cost(INSN_COST * 3);
8515 format %{ "encode_heap_oop $dst, $src" %}
8516 ins_encode %{
8517 Register s = $src$$Register;
8518 Register d = $dst$$Register;
8519 __ encode_heap_oop(d, s);
8520 %}
8521 ins_pipe(ialu_reg);
8522 %}
8523
8524 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8525 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8526 match(Set dst (EncodeP src));
8527 ins_cost(INSN_COST * 3);
8528 format %{ "encode_heap_oop_not_null $dst, $src" %}
8529 ins_encode %{
8530 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8531 %}
8532 ins_pipe(ialu_reg);
8533 %}
8534
8535 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8536 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8537 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8538 match(Set dst (DecodeN src));
8539 ins_cost(INSN_COST * 3);
8540 format %{ "decode_heap_oop $dst, $src" %}
8541 ins_encode %{
8542 Register s = $src$$Register;
8543 Register d = $dst$$Register;
8544 __ decode_heap_oop(d, s);
8545 %}
8546 ins_pipe(ialu_reg);
8547 %}
8548
8549 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8550 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8551 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8552 match(Set dst (DecodeN src));
8553 ins_cost(INSN_COST * 3);
8554 format %{ "decode_heap_oop_not_null $dst, $src" %}
8555 ins_encode %{
8556 Register s = $src$$Register;
8557 Register d = $dst$$Register;
8558 __ decode_heap_oop_not_null(d, s);
8559 %}
8560 ins_pipe(ialu_reg);
8561 %}
8562
8563 // n.b. AArch64 implementations of encode_klass_not_null and
8564 // decode_klass_not_null do not modify the flags register so, unlike
8565 // Intel, we don't kill CR as a side effect here
8566
8567 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8568 match(Set dst (EncodePKlass src));
8569
8570 ins_cost(INSN_COST * 3);
8571 format %{ "encode_klass_not_null $dst,$src" %}
8572
8573 ins_encode %{
8574 Register src_reg = as_Register($src$$reg);
8575 Register dst_reg = as_Register($dst$$reg);
8576 __ encode_klass_not_null(dst_reg, src_reg);
8577 %}
8578
8579 ins_pipe(ialu_reg);
8580 %}
8581
8582 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8583 match(Set dst (DecodeNKlass src));
8584
8585 ins_cost(INSN_COST * 3);
8586 format %{ "decode_klass_not_null $dst,$src" %}
8587
8588 ins_encode %{
8589 Register src_reg = as_Register($src$$reg);
8590 Register dst_reg = as_Register($dst$$reg);
8591 if (dst_reg != src_reg) {
8592 __ decode_klass_not_null(dst_reg, src_reg);
8593 } else {
8594 __ decode_klass_not_null(dst_reg);
8595 }
8596 %}
8597
8598 ins_pipe(ialu_reg);
8599 %}
8600
8601 instruct checkCastPP(iRegPNoSp dst)
8602 %{
8603 match(Set dst (CheckCastPP dst));
8604
8605 size(0);
8606 format %{ "# checkcastPP of $dst" %}
8607 ins_encode(/* empty encoding */);
8608 ins_pipe(pipe_class_empty);
8609 %}
8610
8611 instruct castPP(iRegPNoSp dst)
8612 %{
8613 match(Set dst (CastPP dst));
8614
8615 size(0);
8616 format %{ "# castPP of $dst" %}
8617 ins_encode(/* empty encoding */);
8618 ins_pipe(pipe_class_empty);
8619 %}
8620
8621 instruct castII(iRegI dst)
8622 %{
8623 match(Set dst (CastII dst));
8624
8625 size(0);
8626 format %{ "# castII of $dst" %}
8627 ins_encode(/* empty encoding */);
8628 ins_cost(0);
8629 ins_pipe(pipe_class_empty);
8630 %}
8631
8632 instruct castLL(iRegL dst)
8633 %{
8634 match(Set dst (CastLL dst));
8635
8636 size(0);
8637 format %{ "# castLL of $dst" %}
8638 ins_encode(/* empty encoding */);
8639 ins_cost(0);
8640 ins_pipe(pipe_class_empty);
8641 %}
8642
8643 instruct castFF(vRegF dst)
8644 %{
8645 match(Set dst (CastFF dst));
8646
8647 size(0);
8648 format %{ "# castFF of $dst" %}
8649 ins_encode(/* empty encoding */);
8650 ins_cost(0);
8651 ins_pipe(pipe_class_empty);
8652 %}
8653
8654 instruct castDD(vRegD dst)
8655 %{
8656 match(Set dst (CastDD dst));
8657
8658 size(0);
8659 format %{ "# castDD of $dst" %}
8660 ins_encode(/* empty encoding */);
8661 ins_cost(0);
8662 ins_pipe(pipe_class_empty);
8663 %}
8664
8665 instruct castVV(vReg dst)
8666 %{
8667 match(Set dst (CastVV dst));
8668
8669 size(0);
8670 format %{ "# castVV of $dst" %}
8671 ins_encode(/* empty encoding */);
8672 ins_cost(0);
8673 ins_pipe(pipe_class_empty);
8674 %}
8675
8676 instruct castVVMask(pRegGov dst)
8677 %{
8678 match(Set dst (CastVV dst));
8679
8680 size(0);
8681 format %{ "# castVV of $dst" %}
8682 ins_encode(/* empty encoding */);
8683 ins_cost(0);
8684 ins_pipe(pipe_class_empty);
8685 %}
8686
8687 // ============================================================================
8688 // Atomic operation instructions
8689 //
8690
8691 // standard CompareAndSwapX when we are using barriers
8692 // these have higher priority than the rules selected by a predicate
8693
8694 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8695 // can't match them
8696
8697 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8698
8699 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8700 ins_cost(2 * VOLATILE_REF_COST);
8701
8702 effect(KILL cr);
8703
8704 format %{
8705 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8706 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8707 %}
8708
8709 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8710 aarch64_enc_cset_eq(res));
8711
8712 ins_pipe(pipe_slow);
8713 %}
8714
8715 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8716
8717 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8718 ins_cost(2 * VOLATILE_REF_COST);
8719
8720 effect(KILL cr);
8721
8722 format %{
8723 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8724 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8725 %}
8726
8727 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8728 aarch64_enc_cset_eq(res));
8729
8730 ins_pipe(pipe_slow);
8731 %}
8732
8733 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8734
8735 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8736 ins_cost(2 * VOLATILE_REF_COST);
8737
8738 effect(KILL cr);
8739
8740 format %{
8741 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8742 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8743 %}
8744
8745 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8746 aarch64_enc_cset_eq(res));
8747
8748 ins_pipe(pipe_slow);
8749 %}
8750
8751 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8752
8753 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8754 ins_cost(2 * VOLATILE_REF_COST);
8755
8756 effect(KILL cr);
8757
8758 format %{
8759 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8760 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8761 %}
8762
8763 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8764 aarch64_enc_cset_eq(res));
8765
8766 ins_pipe(pipe_slow);
8767 %}
8768
8769 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8770
8771 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8772 predicate(n->as_LoadStore()->barrier_data() == 0);
8773 ins_cost(2 * VOLATILE_REF_COST);
8774
8775 effect(KILL cr);
8776
8777 format %{
8778 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8779 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8780 %}
8781
8782 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8783 aarch64_enc_cset_eq(res));
8784
8785 ins_pipe(pipe_slow);
8786 %}
8787
8788 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8789
8790 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8791 ins_cost(2 * VOLATILE_REF_COST);
8792
8793 effect(KILL cr);
8794
8795 format %{
8796 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8797 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8798 %}
8799
8800 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8801 aarch64_enc_cset_eq(res));
8802
8803 ins_pipe(pipe_slow);
8804 %}
8805
8806 // alternative CompareAndSwapX when we are eliding barriers
8807
8808 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8809
8810 predicate(needs_acquiring_load_exclusive(n));
8811 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8812 ins_cost(VOLATILE_REF_COST);
8813
8814 effect(KILL cr);
8815
8816 format %{
8817 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8818 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8819 %}
8820
8821 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8822 aarch64_enc_cset_eq(res));
8823
8824 ins_pipe(pipe_slow);
8825 %}
8826
8827 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8828
8829 predicate(needs_acquiring_load_exclusive(n));
8830 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8831 ins_cost(VOLATILE_REF_COST);
8832
8833 effect(KILL cr);
8834
8835 format %{
8836 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8837 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8838 %}
8839
8840 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8841 aarch64_enc_cset_eq(res));
8842
8843 ins_pipe(pipe_slow);
8844 %}
8845
8846 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8847
8848 predicate(needs_acquiring_load_exclusive(n));
8849 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8850 ins_cost(VOLATILE_REF_COST);
8851
8852 effect(KILL cr);
8853
8854 format %{
8855 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8856 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8857 %}
8858
8859 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8860 aarch64_enc_cset_eq(res));
8861
8862 ins_pipe(pipe_slow);
8863 %}
8864
8865 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8866
8867 predicate(needs_acquiring_load_exclusive(n));
8868 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8869 ins_cost(VOLATILE_REF_COST);
8870
8871 effect(KILL cr);
8872
8873 format %{
8874 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8875 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8876 %}
8877
8878 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8879 aarch64_enc_cset_eq(res));
8880
8881 ins_pipe(pipe_slow);
8882 %}
8883
8884 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8885
8886 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8887 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8888 ins_cost(VOLATILE_REF_COST);
8889
8890 effect(KILL cr);
8891
8892 format %{
8893 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8894 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8895 %}
8896
8897 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8898 aarch64_enc_cset_eq(res));
8899
8900 ins_pipe(pipe_slow);
8901 %}
8902
8903 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8904
8905 predicate(needs_acquiring_load_exclusive(n));
8906 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8907 ins_cost(VOLATILE_REF_COST);
8908
8909 effect(KILL cr);
8910
8911 format %{
8912 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8913 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8914 %}
8915
8916 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8917 aarch64_enc_cset_eq(res));
8918
8919 ins_pipe(pipe_slow);
8920 %}
8921
8922
8923 // ---------------------------------------------------------------------
8924
8925 // BEGIN This section of the file is automatically generated. Do not edit --------------
8926
8927 // Sundry CAS operations. Note that release is always true,
8928 // regardless of the memory ordering of the CAS. This is because we
8929 // need the volatile case to be sequentially consistent but there is
8930 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8931 // can't check the type of memory ordering here, so we always emit a
8932 // STLXR.
8933
8934 // This section is generated from aarch64_ad_cas.m4
8935
8936
8937
8938 // This pattern is generated automatically from cas.m4.
8939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8940 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8941
8942 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8943 ins_cost(2 * VOLATILE_REF_COST);
8944 effect(TEMP_DEF res, KILL cr);
8945 format %{
8946 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8947 %}
8948 ins_encode %{
8949 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8950 Assembler::byte, /*acquire*/ false, /*release*/ true,
8951 /*weak*/ false, $res$$Register);
8952 __ sxtbw($res$$Register, $res$$Register);
8953 %}
8954 ins_pipe(pipe_slow);
8955 %}
8956
8957 // This pattern is generated automatically from cas.m4.
8958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8959 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8960
8961 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8962 ins_cost(2 * VOLATILE_REF_COST);
8963 effect(TEMP_DEF res, KILL cr);
8964 format %{
8965 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8966 %}
8967 ins_encode %{
8968 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8969 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8970 /*weak*/ false, $res$$Register);
8971 __ sxthw($res$$Register, $res$$Register);
8972 %}
8973 ins_pipe(pipe_slow);
8974 %}
8975
8976 // This pattern is generated automatically from cas.m4.
8977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8978 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8979
8980 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8981 ins_cost(2 * VOLATILE_REF_COST);
8982 effect(TEMP_DEF res, KILL cr);
8983 format %{
8984 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8985 %}
8986 ins_encode %{
8987 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8988 Assembler::word, /*acquire*/ false, /*release*/ true,
8989 /*weak*/ false, $res$$Register);
8990 %}
8991 ins_pipe(pipe_slow);
8992 %}
8993
8994 // This pattern is generated automatically from cas.m4.
8995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8996 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8997
8998 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8999 ins_cost(2 * VOLATILE_REF_COST);
9000 effect(TEMP_DEF res, KILL cr);
9001 format %{
9002 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9003 %}
9004 ins_encode %{
9005 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9006 Assembler::xword, /*acquire*/ false, /*release*/ true,
9007 /*weak*/ false, $res$$Register);
9008 %}
9009 ins_pipe(pipe_slow);
9010 %}
9011
9012 // This pattern is generated automatically from cas.m4.
9013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9014 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9015
9016 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9017 ins_cost(2 * VOLATILE_REF_COST);
9018 effect(TEMP_DEF res, KILL cr);
9019 format %{
9020 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9021 %}
9022 ins_encode %{
9023 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9024 Assembler::word, /*acquire*/ false, /*release*/ true,
9025 /*weak*/ false, $res$$Register);
9026 %}
9027 ins_pipe(pipe_slow);
9028 %}
9029
9030 // This pattern is generated automatically from cas.m4.
9031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9032 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9033 predicate(n->as_LoadStore()->barrier_data() == 0);
9034 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9035 ins_cost(2 * VOLATILE_REF_COST);
9036 effect(TEMP_DEF res, KILL cr);
9037 format %{
9038 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9039 %}
9040 ins_encode %{
9041 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9042 Assembler::xword, /*acquire*/ false, /*release*/ true,
9043 /*weak*/ false, $res$$Register);
9044 %}
9045 ins_pipe(pipe_slow);
9046 %}
9047
9048 // This pattern is generated automatically from cas.m4.
9049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9050 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9051 predicate(needs_acquiring_load_exclusive(n));
9052 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9053 ins_cost(VOLATILE_REF_COST);
9054 effect(TEMP_DEF res, KILL cr);
9055 format %{
9056 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9057 %}
9058 ins_encode %{
9059 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9060 Assembler::byte, /*acquire*/ true, /*release*/ true,
9061 /*weak*/ false, $res$$Register);
9062 __ sxtbw($res$$Register, $res$$Register);
9063 %}
9064 ins_pipe(pipe_slow);
9065 %}
9066
9067 // This pattern is generated automatically from cas.m4.
9068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9069 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9070 predicate(needs_acquiring_load_exclusive(n));
9071 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9072 ins_cost(VOLATILE_REF_COST);
9073 effect(TEMP_DEF res, KILL cr);
9074 format %{
9075 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9076 %}
9077 ins_encode %{
9078 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9079 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9080 /*weak*/ false, $res$$Register);
9081 __ sxthw($res$$Register, $res$$Register);
9082 %}
9083 ins_pipe(pipe_slow);
9084 %}
9085
9086 // This pattern is generated automatically from cas.m4.
9087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9088 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9089 predicate(needs_acquiring_load_exclusive(n));
9090 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9091 ins_cost(VOLATILE_REF_COST);
9092 effect(TEMP_DEF res, KILL cr);
9093 format %{
9094 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9095 %}
9096 ins_encode %{
9097 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9098 Assembler::word, /*acquire*/ true, /*release*/ true,
9099 /*weak*/ false, $res$$Register);
9100 %}
9101 ins_pipe(pipe_slow);
9102 %}
9103
9104 // This pattern is generated automatically from cas.m4.
9105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9106 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9107 predicate(needs_acquiring_load_exclusive(n));
9108 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9109 ins_cost(VOLATILE_REF_COST);
9110 effect(TEMP_DEF res, KILL cr);
9111 format %{
9112 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9113 %}
9114 ins_encode %{
9115 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9116 Assembler::xword, /*acquire*/ true, /*release*/ true,
9117 /*weak*/ false, $res$$Register);
9118 %}
9119 ins_pipe(pipe_slow);
9120 %}
9121
9122 // This pattern is generated automatically from cas.m4.
9123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9124 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9125 predicate(needs_acquiring_load_exclusive(n));
9126 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9127 ins_cost(VOLATILE_REF_COST);
9128 effect(TEMP_DEF res, KILL cr);
9129 format %{
9130 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9131 %}
9132 ins_encode %{
9133 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9134 Assembler::word, /*acquire*/ true, /*release*/ true,
9135 /*weak*/ false, $res$$Register);
9136 %}
9137 ins_pipe(pipe_slow);
9138 %}
9139
9140 // This pattern is generated automatically from cas.m4.
9141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9142 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9143 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9144 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9145 ins_cost(VOLATILE_REF_COST);
9146 effect(TEMP_DEF res, KILL cr);
9147 format %{
9148 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9149 %}
9150 ins_encode %{
9151 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9152 Assembler::xword, /*acquire*/ true, /*release*/ true,
9153 /*weak*/ false, $res$$Register);
9154 %}
9155 ins_pipe(pipe_slow);
9156 %}
9157
9158 // This pattern is generated automatically from cas.m4.
9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9160 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9161
9162 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9163 ins_cost(2 * VOLATILE_REF_COST);
9164 effect(KILL cr);
9165 format %{
9166 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9167 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9168 %}
9169 ins_encode %{
9170 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9171 Assembler::byte, /*acquire*/ false, /*release*/ true,
9172 /*weak*/ true, noreg);
9173 __ csetw($res$$Register, Assembler::EQ);
9174 %}
9175 ins_pipe(pipe_slow);
9176 %}
9177
9178 // This pattern is generated automatically from cas.m4.
9179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9180 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9181
9182 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9183 ins_cost(2 * VOLATILE_REF_COST);
9184 effect(KILL cr);
9185 format %{
9186 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9187 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9188 %}
9189 ins_encode %{
9190 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9191 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9192 /*weak*/ true, noreg);
9193 __ csetw($res$$Register, Assembler::EQ);
9194 %}
9195 ins_pipe(pipe_slow);
9196 %}
9197
9198 // This pattern is generated automatically from cas.m4.
9199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9200 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9201
9202 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9203 ins_cost(2 * VOLATILE_REF_COST);
9204 effect(KILL cr);
9205 format %{
9206 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9207 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9208 %}
9209 ins_encode %{
9210 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9211 Assembler::word, /*acquire*/ false, /*release*/ true,
9212 /*weak*/ true, noreg);
9213 __ csetw($res$$Register, Assembler::EQ);
9214 %}
9215 ins_pipe(pipe_slow);
9216 %}
9217
9218 // This pattern is generated automatically from cas.m4.
9219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9220 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9221
9222 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9223 ins_cost(2 * VOLATILE_REF_COST);
9224 effect(KILL cr);
9225 format %{
9226 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9227 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9228 %}
9229 ins_encode %{
9230 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9231 Assembler::xword, /*acquire*/ false, /*release*/ true,
9232 /*weak*/ true, noreg);
9233 __ csetw($res$$Register, Assembler::EQ);
9234 %}
9235 ins_pipe(pipe_slow);
9236 %}
9237
9238 // This pattern is generated automatically from cas.m4.
9239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9240 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9241
9242 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9243 ins_cost(2 * VOLATILE_REF_COST);
9244 effect(KILL cr);
9245 format %{
9246 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9247 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9248 %}
9249 ins_encode %{
9250 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9251 Assembler::word, /*acquire*/ false, /*release*/ true,
9252 /*weak*/ true, noreg);
9253 __ csetw($res$$Register, Assembler::EQ);
9254 %}
9255 ins_pipe(pipe_slow);
9256 %}
9257
9258 // This pattern is generated automatically from cas.m4.
9259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9260 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9261 predicate(n->as_LoadStore()->barrier_data() == 0);
9262 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9263 ins_cost(2 * VOLATILE_REF_COST);
9264 effect(KILL cr);
9265 format %{
9266 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9267 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9268 %}
9269 ins_encode %{
9270 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9271 Assembler::xword, /*acquire*/ false, /*release*/ true,
9272 /*weak*/ true, noreg);
9273 __ csetw($res$$Register, Assembler::EQ);
9274 %}
9275 ins_pipe(pipe_slow);
9276 %}
9277
9278 // This pattern is generated automatically from cas.m4.
9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9280 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9281 predicate(needs_acquiring_load_exclusive(n));
9282 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9283 ins_cost(VOLATILE_REF_COST);
9284 effect(KILL cr);
9285 format %{
9286 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9287 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9288 %}
9289 ins_encode %{
9290 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9291 Assembler::byte, /*acquire*/ true, /*release*/ true,
9292 /*weak*/ true, noreg);
9293 __ csetw($res$$Register, Assembler::EQ);
9294 %}
9295 ins_pipe(pipe_slow);
9296 %}
9297
9298 // This pattern is generated automatically from cas.m4.
9299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9300 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9301 predicate(needs_acquiring_load_exclusive(n));
9302 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9303 ins_cost(VOLATILE_REF_COST);
9304 effect(KILL cr);
9305 format %{
9306 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9307 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9308 %}
9309 ins_encode %{
9310 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9311 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9312 /*weak*/ true, noreg);
9313 __ csetw($res$$Register, Assembler::EQ);
9314 %}
9315 ins_pipe(pipe_slow);
9316 %}
9317
9318 // This pattern is generated automatically from cas.m4.
9319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9320 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9321 predicate(needs_acquiring_load_exclusive(n));
9322 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9323 ins_cost(VOLATILE_REF_COST);
9324 effect(KILL cr);
9325 format %{
9326 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9327 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9328 %}
9329 ins_encode %{
9330 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9331 Assembler::word, /*acquire*/ true, /*release*/ true,
9332 /*weak*/ true, noreg);
9333 __ csetw($res$$Register, Assembler::EQ);
9334 %}
9335 ins_pipe(pipe_slow);
9336 %}
9337
9338 // This pattern is generated automatically from cas.m4.
9339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9340 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9341 predicate(needs_acquiring_load_exclusive(n));
9342 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9343 ins_cost(VOLATILE_REF_COST);
9344 effect(KILL cr);
9345 format %{
9346 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9347 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9348 %}
9349 ins_encode %{
9350 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9351 Assembler::xword, /*acquire*/ true, /*release*/ true,
9352 /*weak*/ true, noreg);
9353 __ csetw($res$$Register, Assembler::EQ);
9354 %}
9355 ins_pipe(pipe_slow);
9356 %}
9357
9358 // This pattern is generated automatically from cas.m4.
9359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9360 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9361 predicate(needs_acquiring_load_exclusive(n));
9362 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9363 ins_cost(VOLATILE_REF_COST);
9364 effect(KILL cr);
9365 format %{
9366 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9367 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9368 %}
9369 ins_encode %{
9370 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9371 Assembler::word, /*acquire*/ true, /*release*/ true,
9372 /*weak*/ true, noreg);
9373 __ csetw($res$$Register, Assembler::EQ);
9374 %}
9375 ins_pipe(pipe_slow);
9376 %}
9377
9378 // This pattern is generated automatically from cas.m4.
9379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9380 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9381 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9382 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9383 ins_cost(VOLATILE_REF_COST);
9384 effect(KILL cr);
9385 format %{
9386 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9387 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9388 %}
9389 ins_encode %{
9390 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9391 Assembler::xword, /*acquire*/ true, /*release*/ true,
9392 /*weak*/ true, noreg);
9393 __ csetw($res$$Register, Assembler::EQ);
9394 %}
9395 ins_pipe(pipe_slow);
9396 %}
9397
9398 // END This section of the file is automatically generated. Do not edit --------------
9399 // ---------------------------------------------------------------------
9400
9401 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9402 match(Set prev (GetAndSetI mem newv));
9403 ins_cost(2 * VOLATILE_REF_COST);
9404 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9405 ins_encode %{
9406 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9407 %}
9408 ins_pipe(pipe_serial);
9409 %}
9410
9411 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9412 match(Set prev (GetAndSetL mem newv));
9413 ins_cost(2 * VOLATILE_REF_COST);
9414 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9415 ins_encode %{
9416 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9417 %}
9418 ins_pipe(pipe_serial);
9419 %}
9420
9421 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9422 match(Set prev (GetAndSetN mem newv));
9423 ins_cost(2 * VOLATILE_REF_COST);
9424 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9425 ins_encode %{
9426 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9427 %}
9428 ins_pipe(pipe_serial);
9429 %}
9430
9431 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9432 predicate(n->as_LoadStore()->barrier_data() == 0);
9433 match(Set prev (GetAndSetP mem newv));
9434 ins_cost(2 * VOLATILE_REF_COST);
9435 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9436 ins_encode %{
9437 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9438 %}
9439 ins_pipe(pipe_serial);
9440 %}
9441
9442 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9443 predicate(needs_acquiring_load_exclusive(n));
9444 match(Set prev (GetAndSetI mem newv));
9445 ins_cost(VOLATILE_REF_COST);
9446 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9447 ins_encode %{
9448 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9449 %}
9450 ins_pipe(pipe_serial);
9451 %}
9452
9453 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9454 predicate(needs_acquiring_load_exclusive(n));
9455 match(Set prev (GetAndSetL mem newv));
9456 ins_cost(VOLATILE_REF_COST);
9457 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9458 ins_encode %{
9459 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9460 %}
9461 ins_pipe(pipe_serial);
9462 %}
9463
9464 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9465 predicate(needs_acquiring_load_exclusive(n));
9466 match(Set prev (GetAndSetN mem newv));
9467 ins_cost(VOLATILE_REF_COST);
9468 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9469 ins_encode %{
9470 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9471 %}
9472 ins_pipe(pipe_serial);
9473 %}
9474
9475 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9476 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9477 match(Set prev (GetAndSetP mem newv));
9478 ins_cost(VOLATILE_REF_COST);
9479 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9480 ins_encode %{
9481 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9482 %}
9483 ins_pipe(pipe_serial);
9484 %}
9485
9486
9487 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9488 match(Set newval (GetAndAddL mem incr));
9489 ins_cost(2 * VOLATILE_REF_COST + 1);
9490 format %{ "get_and_addL $newval, [$mem], $incr" %}
9491 ins_encode %{
9492 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9493 %}
9494 ins_pipe(pipe_serial);
9495 %}
9496
9497 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9498 predicate(n->as_LoadStore()->result_not_used());
9499 match(Set dummy (GetAndAddL mem incr));
9500 ins_cost(2 * VOLATILE_REF_COST);
9501 format %{ "get_and_addL [$mem], $incr" %}
9502 ins_encode %{
9503 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9504 %}
9505 ins_pipe(pipe_serial);
9506 %}
9507
9508 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9509 match(Set newval (GetAndAddL mem incr));
9510 ins_cost(2 * VOLATILE_REF_COST + 1);
9511 format %{ "get_and_addL $newval, [$mem], $incr" %}
9512 ins_encode %{
9513 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9514 %}
9515 ins_pipe(pipe_serial);
9516 %}
9517
9518 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9519 predicate(n->as_LoadStore()->result_not_used());
9520 match(Set dummy (GetAndAddL mem incr));
9521 ins_cost(2 * VOLATILE_REF_COST);
9522 format %{ "get_and_addL [$mem], $incr" %}
9523 ins_encode %{
9524 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9525 %}
9526 ins_pipe(pipe_serial);
9527 %}
9528
9529 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9530 match(Set newval (GetAndAddI mem incr));
9531 ins_cost(2 * VOLATILE_REF_COST + 1);
9532 format %{ "get_and_addI $newval, [$mem], $incr" %}
9533 ins_encode %{
9534 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9535 %}
9536 ins_pipe(pipe_serial);
9537 %}
9538
9539 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9540 predicate(n->as_LoadStore()->result_not_used());
9541 match(Set dummy (GetAndAddI mem incr));
9542 ins_cost(2 * VOLATILE_REF_COST);
9543 format %{ "get_and_addI [$mem], $incr" %}
9544 ins_encode %{
9545 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9546 %}
9547 ins_pipe(pipe_serial);
9548 %}
9549
9550 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9551 match(Set newval (GetAndAddI mem incr));
9552 ins_cost(2 * VOLATILE_REF_COST + 1);
9553 format %{ "get_and_addI $newval, [$mem], $incr" %}
9554 ins_encode %{
9555 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9556 %}
9557 ins_pipe(pipe_serial);
9558 %}
9559
9560 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9561 predicate(n->as_LoadStore()->result_not_used());
9562 match(Set dummy (GetAndAddI mem incr));
9563 ins_cost(2 * VOLATILE_REF_COST);
9564 format %{ "get_and_addI [$mem], $incr" %}
9565 ins_encode %{
9566 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9567 %}
9568 ins_pipe(pipe_serial);
9569 %}
9570
9571 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9572 predicate(needs_acquiring_load_exclusive(n));
9573 match(Set newval (GetAndAddL mem incr));
9574 ins_cost(VOLATILE_REF_COST + 1);
9575 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9576 ins_encode %{
9577 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9578 %}
9579 ins_pipe(pipe_serial);
9580 %}
9581
9582 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9583 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9584 match(Set dummy (GetAndAddL mem incr));
9585 ins_cost(VOLATILE_REF_COST);
9586 format %{ "get_and_addL_acq [$mem], $incr" %}
9587 ins_encode %{
9588 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9589 %}
9590 ins_pipe(pipe_serial);
9591 %}
9592
9593 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9594 predicate(needs_acquiring_load_exclusive(n));
9595 match(Set newval (GetAndAddL mem incr));
9596 ins_cost(VOLATILE_REF_COST + 1);
9597 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9598 ins_encode %{
9599 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9600 %}
9601 ins_pipe(pipe_serial);
9602 %}
9603
9604 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9605 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9606 match(Set dummy (GetAndAddL mem incr));
9607 ins_cost(VOLATILE_REF_COST);
9608 format %{ "get_and_addL_acq [$mem], $incr" %}
9609 ins_encode %{
9610 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9611 %}
9612 ins_pipe(pipe_serial);
9613 %}
9614
9615 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9616 predicate(needs_acquiring_load_exclusive(n));
9617 match(Set newval (GetAndAddI mem incr));
9618 ins_cost(VOLATILE_REF_COST + 1);
9619 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9620 ins_encode %{
9621 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9622 %}
9623 ins_pipe(pipe_serial);
9624 %}
9625
9626 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9627 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9628 match(Set dummy (GetAndAddI mem incr));
9629 ins_cost(VOLATILE_REF_COST);
9630 format %{ "get_and_addI_acq [$mem], $incr" %}
9631 ins_encode %{
9632 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9633 %}
9634 ins_pipe(pipe_serial);
9635 %}
9636
9637 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9638 predicate(needs_acquiring_load_exclusive(n));
9639 match(Set newval (GetAndAddI mem incr));
9640 ins_cost(VOLATILE_REF_COST + 1);
9641 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9642 ins_encode %{
9643 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9644 %}
9645 ins_pipe(pipe_serial);
9646 %}
9647
9648 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9649 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9650 match(Set dummy (GetAndAddI mem incr));
9651 ins_cost(VOLATILE_REF_COST);
9652 format %{ "get_and_addI_acq [$mem], $incr" %}
9653 ins_encode %{
9654 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9655 %}
9656 ins_pipe(pipe_serial);
9657 %}
9658
9659 // Manifest a CmpU result in an integer register.
9660 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9661 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9662 %{
9663 match(Set dst (CmpU3 src1 src2));
9664 effect(KILL flags);
9665
9666 ins_cost(INSN_COST * 3);
9667 format %{
9668 "cmpw $src1, $src2\n\t"
9669 "csetw $dst, ne\n\t"
9670 "cnegw $dst, lo\t# CmpU3(reg)"
9671 %}
9672 ins_encode %{
9673 __ cmpw($src1$$Register, $src2$$Register);
9674 __ csetw($dst$$Register, Assembler::NE);
9675 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9676 %}
9677
9678 ins_pipe(pipe_class_default);
9679 %}
9680
9681 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9682 %{
9683 match(Set dst (CmpU3 src1 src2));
9684 effect(KILL flags);
9685
9686 ins_cost(INSN_COST * 3);
9687 format %{
9688 "subsw zr, $src1, $src2\n\t"
9689 "csetw $dst, ne\n\t"
9690 "cnegw $dst, lo\t# CmpU3(imm)"
9691 %}
9692 ins_encode %{
9693 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9694 __ csetw($dst$$Register, Assembler::NE);
9695 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9696 %}
9697
9698 ins_pipe(pipe_class_default);
9699 %}
9700
9701 // Manifest a CmpUL result in an integer register.
9702 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9703 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9704 %{
9705 match(Set dst (CmpUL3 src1 src2));
9706 effect(KILL flags);
9707
9708 ins_cost(INSN_COST * 3);
9709 format %{
9710 "cmp $src1, $src2\n\t"
9711 "csetw $dst, ne\n\t"
9712 "cnegw $dst, lo\t# CmpUL3(reg)"
9713 %}
9714 ins_encode %{
9715 __ cmp($src1$$Register, $src2$$Register);
9716 __ csetw($dst$$Register, Assembler::NE);
9717 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9718 %}
9719
9720 ins_pipe(pipe_class_default);
9721 %}
9722
9723 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9724 %{
9725 match(Set dst (CmpUL3 src1 src2));
9726 effect(KILL flags);
9727
9728 ins_cost(INSN_COST * 3);
9729 format %{
9730 "subs zr, $src1, $src2\n\t"
9731 "csetw $dst, ne\n\t"
9732 "cnegw $dst, lo\t# CmpUL3(imm)"
9733 %}
9734 ins_encode %{
9735 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9736 __ csetw($dst$$Register, Assembler::NE);
9737 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9738 %}
9739
9740 ins_pipe(pipe_class_default);
9741 %}
9742
9743 // Manifest a CmpL result in an integer register.
9744 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9745 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9746 %{
9747 match(Set dst (CmpL3 src1 src2));
9748 effect(KILL flags);
9749
9750 ins_cost(INSN_COST * 3);
9751 format %{
9752 "cmp $src1, $src2\n\t"
9753 "csetw $dst, ne\n\t"
9754 "cnegw $dst, lt\t# CmpL3(reg)"
9755 %}
9756 ins_encode %{
9757 __ cmp($src1$$Register, $src2$$Register);
9758 __ csetw($dst$$Register, Assembler::NE);
9759 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9760 %}
9761
9762 ins_pipe(pipe_class_default);
9763 %}
9764
9765 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9766 %{
9767 match(Set dst (CmpL3 src1 src2));
9768 effect(KILL flags);
9769
9770 ins_cost(INSN_COST * 3);
9771 format %{
9772 "subs zr, $src1, $src2\n\t"
9773 "csetw $dst, ne\n\t"
9774 "cnegw $dst, lt\t# CmpL3(imm)"
9775 %}
9776 ins_encode %{
9777 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9778 __ csetw($dst$$Register, Assembler::NE);
9779 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9780 %}
9781
9782 ins_pipe(pipe_class_default);
9783 %}
9784
9785 // ============================================================================
9786 // Conditional Move Instructions
9787
9788 // n.b. we have identical rules for both a signed compare op (cmpOp)
9789 // and an unsigned compare op (cmpOpU). it would be nice if we could
9790 // define an op class which merged both inputs and use it to type the
9791 // argument to a single rule. unfortunatelyt his fails because the
9792 // opclass does not live up to the COND_INTER interface of its
9793 // component operands. When the generic code tries to negate the
9794 // operand it ends up running the generci Machoper::negate method
9795 // which throws a ShouldNotHappen. So, we have to provide two flavours
9796 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9797
9798 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9799 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9800
9801 ins_cost(INSN_COST * 2);
9802 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9803
9804 ins_encode %{
9805 __ cselw(as_Register($dst$$reg),
9806 as_Register($src2$$reg),
9807 as_Register($src1$$reg),
9808 (Assembler::Condition)$cmp$$cmpcode);
9809 %}
9810
9811 ins_pipe(icond_reg_reg);
9812 %}
9813
9814 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9815 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9816
9817 ins_cost(INSN_COST * 2);
9818 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9819
9820 ins_encode %{
9821 __ cselw(as_Register($dst$$reg),
9822 as_Register($src2$$reg),
9823 as_Register($src1$$reg),
9824 (Assembler::Condition)$cmp$$cmpcode);
9825 %}
9826
9827 ins_pipe(icond_reg_reg);
9828 %}
9829
9830 // special cases where one arg is zero
9831
9832 // n.b. this is selected in preference to the rule above because it
9833 // avoids loading constant 0 into a source register
9834
9835 // TODO
9836 // we ought only to be able to cull one of these variants as the ideal
9837 // transforms ought always to order the zero consistently (to left/right?)
9838
9839 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9840 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9841
9842 ins_cost(INSN_COST * 2);
9843 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9844
9845 ins_encode %{
9846 __ cselw(as_Register($dst$$reg),
9847 as_Register($src$$reg),
9848 zr,
9849 (Assembler::Condition)$cmp$$cmpcode);
9850 %}
9851
9852 ins_pipe(icond_reg);
9853 %}
9854
9855 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9856 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9857
9858 ins_cost(INSN_COST * 2);
9859 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9860
9861 ins_encode %{
9862 __ cselw(as_Register($dst$$reg),
9863 as_Register($src$$reg),
9864 zr,
9865 (Assembler::Condition)$cmp$$cmpcode);
9866 %}
9867
9868 ins_pipe(icond_reg);
9869 %}
9870
9871 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9872 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9873
9874 ins_cost(INSN_COST * 2);
9875 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9876
9877 ins_encode %{
9878 __ cselw(as_Register($dst$$reg),
9879 zr,
9880 as_Register($src$$reg),
9881 (Assembler::Condition)$cmp$$cmpcode);
9882 %}
9883
9884 ins_pipe(icond_reg);
9885 %}
9886
9887 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9888 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9889
9890 ins_cost(INSN_COST * 2);
9891 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9892
9893 ins_encode %{
9894 __ cselw(as_Register($dst$$reg),
9895 zr,
9896 as_Register($src$$reg),
9897 (Assembler::Condition)$cmp$$cmpcode);
9898 %}
9899
9900 ins_pipe(icond_reg);
9901 %}
9902
9903 // special case for creating a boolean 0 or 1
9904
9905 // n.b. this is selected in preference to the rule above because it
9906 // avoids loading constants 0 and 1 into a source register
9907
9908 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9909 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9910
9911 ins_cost(INSN_COST * 2);
9912 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9913
9914 ins_encode %{
9915 // equivalently
9916 // cset(as_Register($dst$$reg),
9917 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9918 __ csincw(as_Register($dst$$reg),
9919 zr,
9920 zr,
9921 (Assembler::Condition)$cmp$$cmpcode);
9922 %}
9923
9924 ins_pipe(icond_none);
9925 %}
9926
9927 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9928 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9929
9930 ins_cost(INSN_COST * 2);
9931 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9932
9933 ins_encode %{
9934 // equivalently
9935 // cset(as_Register($dst$$reg),
9936 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9937 __ csincw(as_Register($dst$$reg),
9938 zr,
9939 zr,
9940 (Assembler::Condition)$cmp$$cmpcode);
9941 %}
9942
9943 ins_pipe(icond_none);
9944 %}
9945
9946 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9947 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9948
9949 ins_cost(INSN_COST * 2);
9950 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9951
9952 ins_encode %{
9953 __ csel(as_Register($dst$$reg),
9954 as_Register($src2$$reg),
9955 as_Register($src1$$reg),
9956 (Assembler::Condition)$cmp$$cmpcode);
9957 %}
9958
9959 ins_pipe(icond_reg_reg);
9960 %}
9961
9962 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9963 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9964
9965 ins_cost(INSN_COST * 2);
9966 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9967
9968 ins_encode %{
9969 __ csel(as_Register($dst$$reg),
9970 as_Register($src2$$reg),
9971 as_Register($src1$$reg),
9972 (Assembler::Condition)$cmp$$cmpcode);
9973 %}
9974
9975 ins_pipe(icond_reg_reg);
9976 %}
9977
9978 // special cases where one arg is zero
9979
9980 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9981 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9982
9983 ins_cost(INSN_COST * 2);
9984 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9985
9986 ins_encode %{
9987 __ csel(as_Register($dst$$reg),
9988 zr,
9989 as_Register($src$$reg),
9990 (Assembler::Condition)$cmp$$cmpcode);
9991 %}
9992
9993 ins_pipe(icond_reg);
9994 %}
9995
9996 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9997 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9998
9999 ins_cost(INSN_COST * 2);
10000 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10001
10002 ins_encode %{
10003 __ csel(as_Register($dst$$reg),
10004 zr,
10005 as_Register($src$$reg),
10006 (Assembler::Condition)$cmp$$cmpcode);
10007 %}
10008
10009 ins_pipe(icond_reg);
10010 %}
10011
10012 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10013 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10014
10015 ins_cost(INSN_COST * 2);
10016 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10017
10018 ins_encode %{
10019 __ csel(as_Register($dst$$reg),
10020 as_Register($src$$reg),
10021 zr,
10022 (Assembler::Condition)$cmp$$cmpcode);
10023 %}
10024
10025 ins_pipe(icond_reg);
10026 %}
10027
10028 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10029 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10030
10031 ins_cost(INSN_COST * 2);
10032 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10033
10034 ins_encode %{
10035 __ csel(as_Register($dst$$reg),
10036 as_Register($src$$reg),
10037 zr,
10038 (Assembler::Condition)$cmp$$cmpcode);
10039 %}
10040
10041 ins_pipe(icond_reg);
10042 %}
10043
10044 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10045 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10046
10047 ins_cost(INSN_COST * 2);
10048 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10049
10050 ins_encode %{
10051 __ csel(as_Register($dst$$reg),
10052 as_Register($src2$$reg),
10053 as_Register($src1$$reg),
10054 (Assembler::Condition)$cmp$$cmpcode);
10055 %}
10056
10057 ins_pipe(icond_reg_reg);
10058 %}
10059
10060 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10061 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10062
10063 ins_cost(INSN_COST * 2);
10064 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10065
10066 ins_encode %{
10067 __ csel(as_Register($dst$$reg),
10068 as_Register($src2$$reg),
10069 as_Register($src1$$reg),
10070 (Assembler::Condition)$cmp$$cmpcode);
10071 %}
10072
10073 ins_pipe(icond_reg_reg);
10074 %}
10075
10076 // special cases where one arg is zero
10077
10078 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10079 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10080
10081 ins_cost(INSN_COST * 2);
10082 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10083
10084 ins_encode %{
10085 __ csel(as_Register($dst$$reg),
10086 zr,
10087 as_Register($src$$reg),
10088 (Assembler::Condition)$cmp$$cmpcode);
10089 %}
10090
10091 ins_pipe(icond_reg);
10092 %}
10093
10094 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10095 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10096
10097 ins_cost(INSN_COST * 2);
10098 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10099
10100 ins_encode %{
10101 __ csel(as_Register($dst$$reg),
10102 zr,
10103 as_Register($src$$reg),
10104 (Assembler::Condition)$cmp$$cmpcode);
10105 %}
10106
10107 ins_pipe(icond_reg);
10108 %}
10109
10110 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10111 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10112
10113 ins_cost(INSN_COST * 2);
10114 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10115
10116 ins_encode %{
10117 __ csel(as_Register($dst$$reg),
10118 as_Register($src$$reg),
10119 zr,
10120 (Assembler::Condition)$cmp$$cmpcode);
10121 %}
10122
10123 ins_pipe(icond_reg);
10124 %}
10125
10126 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10127 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10128
10129 ins_cost(INSN_COST * 2);
10130 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10131
10132 ins_encode %{
10133 __ csel(as_Register($dst$$reg),
10134 as_Register($src$$reg),
10135 zr,
10136 (Assembler::Condition)$cmp$$cmpcode);
10137 %}
10138
10139 ins_pipe(icond_reg);
10140 %}
10141
10142 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10143 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10144
10145 ins_cost(INSN_COST * 2);
10146 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10147
10148 ins_encode %{
10149 __ cselw(as_Register($dst$$reg),
10150 as_Register($src2$$reg),
10151 as_Register($src1$$reg),
10152 (Assembler::Condition)$cmp$$cmpcode);
10153 %}
10154
10155 ins_pipe(icond_reg_reg);
10156 %}
10157
10158 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10159 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10160
10161 ins_cost(INSN_COST * 2);
10162 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10163
10164 ins_encode %{
10165 __ cselw(as_Register($dst$$reg),
10166 as_Register($src2$$reg),
10167 as_Register($src1$$reg),
10168 (Assembler::Condition)$cmp$$cmpcode);
10169 %}
10170
10171 ins_pipe(icond_reg_reg);
10172 %}
10173
10174 // special cases where one arg is zero
10175
10176 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10177 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10178
10179 ins_cost(INSN_COST * 2);
10180 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10181
10182 ins_encode %{
10183 __ cselw(as_Register($dst$$reg),
10184 zr,
10185 as_Register($src$$reg),
10186 (Assembler::Condition)$cmp$$cmpcode);
10187 %}
10188
10189 ins_pipe(icond_reg);
10190 %}
10191
10192 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10193 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10194
10195 ins_cost(INSN_COST * 2);
10196 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10197
10198 ins_encode %{
10199 __ cselw(as_Register($dst$$reg),
10200 zr,
10201 as_Register($src$$reg),
10202 (Assembler::Condition)$cmp$$cmpcode);
10203 %}
10204
10205 ins_pipe(icond_reg);
10206 %}
10207
10208 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10209 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10210
10211 ins_cost(INSN_COST * 2);
10212 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10213
10214 ins_encode %{
10215 __ cselw(as_Register($dst$$reg),
10216 as_Register($src$$reg),
10217 zr,
10218 (Assembler::Condition)$cmp$$cmpcode);
10219 %}
10220
10221 ins_pipe(icond_reg);
10222 %}
10223
10224 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10225 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10226
10227 ins_cost(INSN_COST * 2);
10228 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10229
10230 ins_encode %{
10231 __ cselw(as_Register($dst$$reg),
10232 as_Register($src$$reg),
10233 zr,
10234 (Assembler::Condition)$cmp$$cmpcode);
10235 %}
10236
10237 ins_pipe(icond_reg);
10238 %}
10239
10240 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10241 %{
10242 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10243
10244 ins_cost(INSN_COST * 3);
10245
10246 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10247 ins_encode %{
10248 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10249 __ fcsels(as_FloatRegister($dst$$reg),
10250 as_FloatRegister($src2$$reg),
10251 as_FloatRegister($src1$$reg),
10252 cond);
10253 %}
10254
10255 ins_pipe(fp_cond_reg_reg_s);
10256 %}
10257
10258 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10259 %{
10260 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10261
10262 ins_cost(INSN_COST * 3);
10263
10264 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10265 ins_encode %{
10266 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10267 __ fcsels(as_FloatRegister($dst$$reg),
10268 as_FloatRegister($src2$$reg),
10269 as_FloatRegister($src1$$reg),
10270 cond);
10271 %}
10272
10273 ins_pipe(fp_cond_reg_reg_s);
10274 %}
10275
10276 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10277 %{
10278 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10279
10280 ins_cost(INSN_COST * 3);
10281
10282 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10283 ins_encode %{
10284 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10285 __ fcseld(as_FloatRegister($dst$$reg),
10286 as_FloatRegister($src2$$reg),
10287 as_FloatRegister($src1$$reg),
10288 cond);
10289 %}
10290
10291 ins_pipe(fp_cond_reg_reg_d);
10292 %}
10293
10294 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10295 %{
10296 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10297
10298 ins_cost(INSN_COST * 3);
10299
10300 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10301 ins_encode %{
10302 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10303 __ fcseld(as_FloatRegister($dst$$reg),
10304 as_FloatRegister($src2$$reg),
10305 as_FloatRegister($src1$$reg),
10306 cond);
10307 %}
10308
10309 ins_pipe(fp_cond_reg_reg_d);
10310 %}
10311
10312 // ============================================================================
10313 // Arithmetic Instructions
10314 //
10315
10316 // Integer Addition
10317
10318 // TODO
10319 // these currently employ operations which do not set CR and hence are
10320 // not flagged as killing CR but we would like to isolate the cases
10321 // where we want to set flags from those where we don't. need to work
10322 // out how to do that.
10323
10324 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10325 match(Set dst (AddI src1 src2));
10326
10327 ins_cost(INSN_COST);
10328 format %{ "addw $dst, $src1, $src2" %}
10329
10330 ins_encode %{
10331 __ addw(as_Register($dst$$reg),
10332 as_Register($src1$$reg),
10333 as_Register($src2$$reg));
10334 %}
10335
10336 ins_pipe(ialu_reg_reg);
10337 %}
10338
10339 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10340 match(Set dst (AddI src1 src2));
10341
10342 ins_cost(INSN_COST);
10343 format %{ "addw $dst, $src1, $src2" %}
10344
10345 // use opcode to indicate that this is an add not a sub
10346 opcode(0x0);
10347
10348 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10349
10350 ins_pipe(ialu_reg_imm);
10351 %}
10352
10353 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10354 match(Set dst (AddI (ConvL2I src1) src2));
10355
10356 ins_cost(INSN_COST);
10357 format %{ "addw $dst, $src1, $src2" %}
10358
10359 // use opcode to indicate that this is an add not a sub
10360 opcode(0x0);
10361
10362 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10363
10364 ins_pipe(ialu_reg_imm);
10365 %}
10366
10367 // Pointer Addition
10368 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10369 match(Set dst (AddP src1 src2));
10370
10371 ins_cost(INSN_COST);
10372 format %{ "add $dst, $src1, $src2\t# ptr" %}
10373
10374 ins_encode %{
10375 __ add(as_Register($dst$$reg),
10376 as_Register($src1$$reg),
10377 as_Register($src2$$reg));
10378 %}
10379
10380 ins_pipe(ialu_reg_reg);
10381 %}
10382
10383 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10384 match(Set dst (AddP src1 (ConvI2L src2)));
10385
10386 ins_cost(1.9 * INSN_COST);
10387 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10388
10389 ins_encode %{
10390 __ add(as_Register($dst$$reg),
10391 as_Register($src1$$reg),
10392 as_Register($src2$$reg), ext::sxtw);
10393 %}
10394
10395 ins_pipe(ialu_reg_reg);
10396 %}
10397
10398 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10399 match(Set dst (AddP src1 (LShiftL src2 scale)));
10400
10401 ins_cost(1.9 * INSN_COST);
10402 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10403
10404 ins_encode %{
10405 __ lea(as_Register($dst$$reg),
10406 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10407 Address::lsl($scale$$constant)));
10408 %}
10409
10410 ins_pipe(ialu_reg_reg_shift);
10411 %}
10412
10413 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10414 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10415
10416 ins_cost(1.9 * INSN_COST);
10417 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10418
10419 ins_encode %{
10420 __ lea(as_Register($dst$$reg),
10421 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10422 Address::sxtw($scale$$constant)));
10423 %}
10424
10425 ins_pipe(ialu_reg_reg_shift);
10426 %}
10427
10428 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10429 match(Set dst (LShiftL (ConvI2L src) scale));
10430
10431 ins_cost(INSN_COST);
10432 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10433
10434 ins_encode %{
10435 __ sbfiz(as_Register($dst$$reg),
10436 as_Register($src$$reg),
10437 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10438 %}
10439
10440 ins_pipe(ialu_reg_shift);
10441 %}
10442
10443 // Pointer Immediate Addition
10444 // n.b. this needs to be more expensive than using an indirect memory
10445 // operand
10446 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10447 match(Set dst (AddP src1 src2));
10448
10449 ins_cost(INSN_COST);
10450 format %{ "add $dst, $src1, $src2\t# ptr" %}
10451
10452 // use opcode to indicate that this is an add not a sub
10453 opcode(0x0);
10454
10455 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10456
10457 ins_pipe(ialu_reg_imm);
10458 %}
10459
10460 // Long Addition
10461 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10462
10463 match(Set dst (AddL src1 src2));
10464
10465 ins_cost(INSN_COST);
10466 format %{ "add $dst, $src1, $src2" %}
10467
10468 ins_encode %{
10469 __ add(as_Register($dst$$reg),
10470 as_Register($src1$$reg),
10471 as_Register($src2$$reg));
10472 %}
10473
10474 ins_pipe(ialu_reg_reg);
10475 %}
10476
10477 // No constant pool entries requiredLong Immediate Addition.
10478 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10479 match(Set dst (AddL src1 src2));
10480
10481 ins_cost(INSN_COST);
10482 format %{ "add $dst, $src1, $src2" %}
10483
10484 // use opcode to indicate that this is an add not a sub
10485 opcode(0x0);
10486
10487 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10488
10489 ins_pipe(ialu_reg_imm);
10490 %}
10491
10492 // Integer Subtraction
10493 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10494 match(Set dst (SubI src1 src2));
10495
10496 ins_cost(INSN_COST);
10497 format %{ "subw $dst, $src1, $src2" %}
10498
10499 ins_encode %{
10500 __ subw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg));
10503 %}
10504
10505 ins_pipe(ialu_reg_reg);
10506 %}
10507
10508 // Immediate Subtraction
10509 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10510 match(Set dst (SubI src1 src2));
10511
10512 ins_cost(INSN_COST);
10513 format %{ "subw $dst, $src1, $src2" %}
10514
10515 // use opcode to indicate that this is a sub not an add
10516 opcode(0x1);
10517
10518 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10519
10520 ins_pipe(ialu_reg_imm);
10521 %}
10522
10523 // Long Subtraction
10524 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10525
10526 match(Set dst (SubL src1 src2));
10527
10528 ins_cost(INSN_COST);
10529 format %{ "sub $dst, $src1, $src2" %}
10530
10531 ins_encode %{
10532 __ sub(as_Register($dst$$reg),
10533 as_Register($src1$$reg),
10534 as_Register($src2$$reg));
10535 %}
10536
10537 ins_pipe(ialu_reg_reg);
10538 %}
10539
10540 // No constant pool entries requiredLong Immediate Subtraction.
10541 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10542 match(Set dst (SubL src1 src2));
10543
10544 ins_cost(INSN_COST);
10545 format %{ "sub$dst, $src1, $src2" %}
10546
10547 // use opcode to indicate that this is a sub not an add
10548 opcode(0x1);
10549
10550 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10551
10552 ins_pipe(ialu_reg_imm);
10553 %}
10554
10555 // Integer Negation (special case for sub)
10556
10557 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10558 match(Set dst (SubI zero src));
10559
10560 ins_cost(INSN_COST);
10561 format %{ "negw $dst, $src\t# int" %}
10562
10563 ins_encode %{
10564 __ negw(as_Register($dst$$reg),
10565 as_Register($src$$reg));
10566 %}
10567
10568 ins_pipe(ialu_reg);
10569 %}
10570
10571 // Long Negation
10572
10573 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10574 match(Set dst (SubL zero src));
10575
10576 ins_cost(INSN_COST);
10577 format %{ "neg $dst, $src\t# long" %}
10578
10579 ins_encode %{
10580 __ neg(as_Register($dst$$reg),
10581 as_Register($src$$reg));
10582 %}
10583
10584 ins_pipe(ialu_reg);
10585 %}
10586
10587 // Integer Multiply
10588
10589 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10590 match(Set dst (MulI src1 src2));
10591
10592 ins_cost(INSN_COST * 3);
10593 format %{ "mulw $dst, $src1, $src2" %}
10594
10595 ins_encode %{
10596 __ mulw(as_Register($dst$$reg),
10597 as_Register($src1$$reg),
10598 as_Register($src2$$reg));
10599 %}
10600
10601 ins_pipe(imul_reg_reg);
10602 %}
10603
10604 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10605 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10606
10607 ins_cost(INSN_COST * 3);
10608 format %{ "smull $dst, $src1, $src2" %}
10609
10610 ins_encode %{
10611 __ smull(as_Register($dst$$reg),
10612 as_Register($src1$$reg),
10613 as_Register($src2$$reg));
10614 %}
10615
10616 ins_pipe(imul_reg_reg);
10617 %}
10618
10619 // Long Multiply
10620
10621 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10622 match(Set dst (MulL src1 src2));
10623
10624 ins_cost(INSN_COST * 5);
10625 format %{ "mul $dst, $src1, $src2" %}
10626
10627 ins_encode %{
10628 __ mul(as_Register($dst$$reg),
10629 as_Register($src1$$reg),
10630 as_Register($src2$$reg));
10631 %}
10632
10633 ins_pipe(lmul_reg_reg);
10634 %}
10635
10636 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10637 %{
10638 match(Set dst (MulHiL src1 src2));
10639
10640 ins_cost(INSN_COST * 7);
10641 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10642
10643 ins_encode %{
10644 __ smulh(as_Register($dst$$reg),
10645 as_Register($src1$$reg),
10646 as_Register($src2$$reg));
10647 %}
10648
10649 ins_pipe(lmul_reg_reg);
10650 %}
10651
10652 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10653 %{
10654 match(Set dst (UMulHiL src1 src2));
10655
10656 ins_cost(INSN_COST * 7);
10657 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10658
10659 ins_encode %{
10660 __ umulh(as_Register($dst$$reg),
10661 as_Register($src1$$reg),
10662 as_Register($src2$$reg));
10663 %}
10664
10665 ins_pipe(lmul_reg_reg);
10666 %}
10667
10668 // Combined Integer Multiply & Add/Sub
10669
10670 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10671 match(Set dst (AddI src3 (MulI src1 src2)));
10672
10673 ins_cost(INSN_COST * 3);
10674 format %{ "madd $dst, $src1, $src2, $src3" %}
10675
10676 ins_encode %{
10677 __ maddw(as_Register($dst$$reg),
10678 as_Register($src1$$reg),
10679 as_Register($src2$$reg),
10680 as_Register($src3$$reg));
10681 %}
10682
10683 ins_pipe(imac_reg_reg);
10684 %}
10685
10686 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10687 match(Set dst (SubI src3 (MulI src1 src2)));
10688
10689 ins_cost(INSN_COST * 3);
10690 format %{ "msub $dst, $src1, $src2, $src3" %}
10691
10692 ins_encode %{
10693 __ msubw(as_Register($dst$$reg),
10694 as_Register($src1$$reg),
10695 as_Register($src2$$reg),
10696 as_Register($src3$$reg));
10697 %}
10698
10699 ins_pipe(imac_reg_reg);
10700 %}
10701
10702 // Combined Integer Multiply & Neg
10703
10704 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10705 match(Set dst (MulI (SubI zero src1) src2));
10706
10707 ins_cost(INSN_COST * 3);
10708 format %{ "mneg $dst, $src1, $src2" %}
10709
10710 ins_encode %{
10711 __ mnegw(as_Register($dst$$reg),
10712 as_Register($src1$$reg),
10713 as_Register($src2$$reg));
10714 %}
10715
10716 ins_pipe(imac_reg_reg);
10717 %}
10718
10719 // Combined Long Multiply & Add/Sub
10720
10721 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10722 match(Set dst (AddL src3 (MulL src1 src2)));
10723
10724 ins_cost(INSN_COST * 5);
10725 format %{ "madd $dst, $src1, $src2, $src3" %}
10726
10727 ins_encode %{
10728 __ madd(as_Register($dst$$reg),
10729 as_Register($src1$$reg),
10730 as_Register($src2$$reg),
10731 as_Register($src3$$reg));
10732 %}
10733
10734 ins_pipe(lmac_reg_reg);
10735 %}
10736
10737 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10738 match(Set dst (SubL src3 (MulL src1 src2)));
10739
10740 ins_cost(INSN_COST * 5);
10741 format %{ "msub $dst, $src1, $src2, $src3" %}
10742
10743 ins_encode %{
10744 __ msub(as_Register($dst$$reg),
10745 as_Register($src1$$reg),
10746 as_Register($src2$$reg),
10747 as_Register($src3$$reg));
10748 %}
10749
10750 ins_pipe(lmac_reg_reg);
10751 %}
10752
10753 // Combined Long Multiply & Neg
10754
10755 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10756 match(Set dst (MulL (SubL zero src1) src2));
10757
10758 ins_cost(INSN_COST * 5);
10759 format %{ "mneg $dst, $src1, $src2" %}
10760
10761 ins_encode %{
10762 __ mneg(as_Register($dst$$reg),
10763 as_Register($src1$$reg),
10764 as_Register($src2$$reg));
10765 %}
10766
10767 ins_pipe(lmac_reg_reg);
10768 %}
10769
10770 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10771
10772 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10773 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10774
10775 ins_cost(INSN_COST * 3);
10776 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10777
10778 ins_encode %{
10779 __ smaddl(as_Register($dst$$reg),
10780 as_Register($src1$$reg),
10781 as_Register($src2$$reg),
10782 as_Register($src3$$reg));
10783 %}
10784
10785 ins_pipe(imac_reg_reg);
10786 %}
10787
10788 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10789 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10790
10791 ins_cost(INSN_COST * 3);
10792 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10793
10794 ins_encode %{
10795 __ smsubl(as_Register($dst$$reg),
10796 as_Register($src1$$reg),
10797 as_Register($src2$$reg),
10798 as_Register($src3$$reg));
10799 %}
10800
10801 ins_pipe(imac_reg_reg);
10802 %}
10803
10804 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10805 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10806
10807 ins_cost(INSN_COST * 3);
10808 format %{ "smnegl $dst, $src1, $src2" %}
10809
10810 ins_encode %{
10811 __ smnegl(as_Register($dst$$reg),
10812 as_Register($src1$$reg),
10813 as_Register($src2$$reg));
10814 %}
10815
10816 ins_pipe(imac_reg_reg);
10817 %}
10818
10819 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10820
10821 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10822 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10823
10824 ins_cost(INSN_COST * 5);
10825 format %{ "mulw rscratch1, $src1, $src2\n\t"
10826 "maddw $dst, $src3, $src4, rscratch1" %}
10827
10828 ins_encode %{
10829 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10830 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10831
10832 ins_pipe(imac_reg_reg);
10833 %}
10834
10835 // Integer Divide
10836
10837 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10838 match(Set dst (DivI src1 src2));
10839
10840 ins_cost(INSN_COST * 19);
10841 format %{ "sdivw $dst, $src1, $src2" %}
10842
10843 ins_encode(aarch64_enc_divw(dst, src1, src2));
10844 ins_pipe(idiv_reg_reg);
10845 %}
10846
10847 // Long Divide
10848
10849 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10850 match(Set dst (DivL src1 src2));
10851
10852 ins_cost(INSN_COST * 35);
10853 format %{ "sdiv $dst, $src1, $src2" %}
10854
10855 ins_encode(aarch64_enc_div(dst, src1, src2));
10856 ins_pipe(ldiv_reg_reg);
10857 %}
10858
10859 // Integer Remainder
10860
10861 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10862 match(Set dst (ModI src1 src2));
10863
10864 ins_cost(INSN_COST * 22);
10865 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10866 "msubw $dst, rscratch1, $src2, $src1" %}
10867
10868 ins_encode(aarch64_enc_modw(dst, src1, src2));
10869 ins_pipe(idiv_reg_reg);
10870 %}
10871
10872 // Long Remainder
10873
10874 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10875 match(Set dst (ModL src1 src2));
10876
10877 ins_cost(INSN_COST * 38);
10878 format %{ "sdiv rscratch1, $src1, $src2\n"
10879 "msub $dst, rscratch1, $src2, $src1" %}
10880
10881 ins_encode(aarch64_enc_mod(dst, src1, src2));
10882 ins_pipe(ldiv_reg_reg);
10883 %}
10884
10885 // Unsigned Integer Divide
10886
10887 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10888 match(Set dst (UDivI src1 src2));
10889
10890 ins_cost(INSN_COST * 19);
10891 format %{ "udivw $dst, $src1, $src2" %}
10892
10893 ins_encode %{
10894 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10895 %}
10896
10897 ins_pipe(idiv_reg_reg);
10898 %}
10899
10900 // Unsigned Long Divide
10901
10902 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10903 match(Set dst (UDivL src1 src2));
10904
10905 ins_cost(INSN_COST * 35);
10906 format %{ "udiv $dst, $src1, $src2" %}
10907
10908 ins_encode %{
10909 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10910 %}
10911
10912 ins_pipe(ldiv_reg_reg);
10913 %}
10914
10915 // Unsigned Integer Remainder
10916
10917 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10918 match(Set dst (UModI src1 src2));
10919
10920 ins_cost(INSN_COST * 22);
10921 format %{ "udivw rscratch1, $src1, $src2\n\t"
10922 "msubw $dst, rscratch1, $src2, $src1" %}
10923
10924 ins_encode %{
10925 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10926 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10927 %}
10928
10929 ins_pipe(idiv_reg_reg);
10930 %}
10931
10932 // Unsigned Long Remainder
10933
10934 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10935 match(Set dst (UModL src1 src2));
10936
10937 ins_cost(INSN_COST * 38);
10938 format %{ "udiv rscratch1, $src1, $src2\n"
10939 "msub $dst, rscratch1, $src2, $src1" %}
10940
10941 ins_encode %{
10942 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10943 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10944 %}
10945
10946 ins_pipe(ldiv_reg_reg);
10947 %}
10948
10949 // Integer Shifts
10950
10951 // Shift Left Register
10952 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10953 match(Set dst (LShiftI src1 src2));
10954
10955 ins_cost(INSN_COST * 2);
10956 format %{ "lslvw $dst, $src1, $src2" %}
10957
10958 ins_encode %{
10959 __ lslvw(as_Register($dst$$reg),
10960 as_Register($src1$$reg),
10961 as_Register($src2$$reg));
10962 %}
10963
10964 ins_pipe(ialu_reg_reg_vshift);
10965 %}
10966
10967 // Shift Left Immediate
10968 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10969 match(Set dst (LShiftI src1 src2));
10970
10971 ins_cost(INSN_COST);
10972 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10973
10974 ins_encode %{
10975 __ lslw(as_Register($dst$$reg),
10976 as_Register($src1$$reg),
10977 $src2$$constant & 0x1f);
10978 %}
10979
10980 ins_pipe(ialu_reg_shift);
10981 %}
10982
10983 // Shift Right Logical Register
10984 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10985 match(Set dst (URShiftI src1 src2));
10986
10987 ins_cost(INSN_COST * 2);
10988 format %{ "lsrvw $dst, $src1, $src2" %}
10989
10990 ins_encode %{
10991 __ lsrvw(as_Register($dst$$reg),
10992 as_Register($src1$$reg),
10993 as_Register($src2$$reg));
10994 %}
10995
10996 ins_pipe(ialu_reg_reg_vshift);
10997 %}
10998
10999 // Shift Right Logical Immediate
11000 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11001 match(Set dst (URShiftI src1 src2));
11002
11003 ins_cost(INSN_COST);
11004 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11005
11006 ins_encode %{
11007 __ lsrw(as_Register($dst$$reg),
11008 as_Register($src1$$reg),
11009 $src2$$constant & 0x1f);
11010 %}
11011
11012 ins_pipe(ialu_reg_shift);
11013 %}
11014
11015 // Shift Right Arithmetic Register
11016 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11017 match(Set dst (RShiftI src1 src2));
11018
11019 ins_cost(INSN_COST * 2);
11020 format %{ "asrvw $dst, $src1, $src2" %}
11021
11022 ins_encode %{
11023 __ asrvw(as_Register($dst$$reg),
11024 as_Register($src1$$reg),
11025 as_Register($src2$$reg));
11026 %}
11027
11028 ins_pipe(ialu_reg_reg_vshift);
11029 %}
11030
11031 // Shift Right Arithmetic Immediate
11032 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11033 match(Set dst (RShiftI src1 src2));
11034
11035 ins_cost(INSN_COST);
11036 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11037
11038 ins_encode %{
11039 __ asrw(as_Register($dst$$reg),
11040 as_Register($src1$$reg),
11041 $src2$$constant & 0x1f);
11042 %}
11043
11044 ins_pipe(ialu_reg_shift);
11045 %}
11046
11047 // Combined Int Mask and Right Shift (using UBFM)
11048 // TODO
11049
11050 // Long Shifts
11051
11052 // Shift Left Register
11053 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11054 match(Set dst (LShiftL src1 src2));
11055
11056 ins_cost(INSN_COST * 2);
11057 format %{ "lslv $dst, $src1, $src2" %}
11058
11059 ins_encode %{
11060 __ lslv(as_Register($dst$$reg),
11061 as_Register($src1$$reg),
11062 as_Register($src2$$reg));
11063 %}
11064
11065 ins_pipe(ialu_reg_reg_vshift);
11066 %}
11067
11068 // Shift Left Immediate
11069 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11070 match(Set dst (LShiftL src1 src2));
11071
11072 ins_cost(INSN_COST);
11073 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11074
11075 ins_encode %{
11076 __ lsl(as_Register($dst$$reg),
11077 as_Register($src1$$reg),
11078 $src2$$constant & 0x3f);
11079 %}
11080
11081 ins_pipe(ialu_reg_shift);
11082 %}
11083
11084 // Shift Right Logical Register
11085 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11086 match(Set dst (URShiftL src1 src2));
11087
11088 ins_cost(INSN_COST * 2);
11089 format %{ "lsrv $dst, $src1, $src2" %}
11090
11091 ins_encode %{
11092 __ lsrv(as_Register($dst$$reg),
11093 as_Register($src1$$reg),
11094 as_Register($src2$$reg));
11095 %}
11096
11097 ins_pipe(ialu_reg_reg_vshift);
11098 %}
11099
11100 // Shift Right Logical Immediate
11101 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11102 match(Set dst (URShiftL src1 src2));
11103
11104 ins_cost(INSN_COST);
11105 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11106
11107 ins_encode %{
11108 __ lsr(as_Register($dst$$reg),
11109 as_Register($src1$$reg),
11110 $src2$$constant & 0x3f);
11111 %}
11112
11113 ins_pipe(ialu_reg_shift);
11114 %}
11115
11116 // A special-case pattern for card table stores.
11117 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11118 match(Set dst (URShiftL (CastP2X src1) src2));
11119
11120 ins_cost(INSN_COST);
11121 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11122
11123 ins_encode %{
11124 __ lsr(as_Register($dst$$reg),
11125 as_Register($src1$$reg),
11126 $src2$$constant & 0x3f);
11127 %}
11128
11129 ins_pipe(ialu_reg_shift);
11130 %}
11131
11132 // Shift Right Arithmetic Register
11133 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11134 match(Set dst (RShiftL src1 src2));
11135
11136 ins_cost(INSN_COST * 2);
11137 format %{ "asrv $dst, $src1, $src2" %}
11138
11139 ins_encode %{
11140 __ asrv(as_Register($dst$$reg),
11141 as_Register($src1$$reg),
11142 as_Register($src2$$reg));
11143 %}
11144
11145 ins_pipe(ialu_reg_reg_vshift);
11146 %}
11147
11148 // Shift Right Arithmetic Immediate
11149 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11150 match(Set dst (RShiftL src1 src2));
11151
11152 ins_cost(INSN_COST);
11153 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11154
11155 ins_encode %{
11156 __ asr(as_Register($dst$$reg),
11157 as_Register($src1$$reg),
11158 $src2$$constant & 0x3f);
11159 %}
11160
11161 ins_pipe(ialu_reg_shift);
11162 %}
11163
11164 // BEGIN This section of the file is automatically generated. Do not edit --------------
11165 // This section is generated from aarch64_ad.m4
11166
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 instruct regL_not_reg(iRegLNoSp dst,
11170 iRegL src1, immL_M1 m1,
11171 rFlagsReg cr) %{
11172 match(Set dst (XorL src1 m1));
11173 ins_cost(INSN_COST);
11174 format %{ "eon $dst, $src1, zr" %}
11175
11176 ins_encode %{
11177 __ eon(as_Register($dst$$reg),
11178 as_Register($src1$$reg),
11179 zr,
11180 Assembler::LSL, 0);
11181 %}
11182
11183 ins_pipe(ialu_reg);
11184 %}
11185
11186 // This pattern is automatically generated from aarch64_ad.m4
11187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11188 instruct regI_not_reg(iRegINoSp dst,
11189 iRegIorL2I src1, immI_M1 m1,
11190 rFlagsReg cr) %{
11191 match(Set dst (XorI src1 m1));
11192 ins_cost(INSN_COST);
11193 format %{ "eonw $dst, $src1, zr" %}
11194
11195 ins_encode %{
11196 __ eonw(as_Register($dst$$reg),
11197 as_Register($src1$$reg),
11198 zr,
11199 Assembler::LSL, 0);
11200 %}
11201
11202 ins_pipe(ialu_reg);
11203 %}
11204
11205 // This pattern is automatically generated from aarch64_ad.m4
11206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11207 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11208 immI0 zero, iRegIorL2I src1, immI src2) %{
11209 match(Set dst (SubI zero (URShiftI src1 src2)));
11210
11211 ins_cost(1.9 * INSN_COST);
11212 format %{ "negw $dst, $src1, LSR $src2" %}
11213
11214 ins_encode %{
11215 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11216 Assembler::LSR, $src2$$constant & 0x1f);
11217 %}
11218
11219 ins_pipe(ialu_reg_shift);
11220 %}
11221
11222 // This pattern is automatically generated from aarch64_ad.m4
11223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11224 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11225 immI0 zero, iRegIorL2I src1, immI src2) %{
11226 match(Set dst (SubI zero (RShiftI src1 src2)));
11227
11228 ins_cost(1.9 * INSN_COST);
11229 format %{ "negw $dst, $src1, ASR $src2" %}
11230
11231 ins_encode %{
11232 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11233 Assembler::ASR, $src2$$constant & 0x1f);
11234 %}
11235
11236 ins_pipe(ialu_reg_shift);
11237 %}
11238
11239 // This pattern is automatically generated from aarch64_ad.m4
11240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11241 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11242 immI0 zero, iRegIorL2I src1, immI src2) %{
11243 match(Set dst (SubI zero (LShiftI src1 src2)));
11244
11245 ins_cost(1.9 * INSN_COST);
11246 format %{ "negw $dst, $src1, LSL $src2" %}
11247
11248 ins_encode %{
11249 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11250 Assembler::LSL, $src2$$constant & 0x1f);
11251 %}
11252
11253 ins_pipe(ialu_reg_shift);
11254 %}
11255
11256 // This pattern is automatically generated from aarch64_ad.m4
11257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11258 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11259 immL0 zero, iRegL src1, immI src2) %{
11260 match(Set dst (SubL zero (URShiftL src1 src2)));
11261
11262 ins_cost(1.9 * INSN_COST);
11263 format %{ "neg $dst, $src1, LSR $src2" %}
11264
11265 ins_encode %{
11266 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11267 Assembler::LSR, $src2$$constant & 0x3f);
11268 %}
11269
11270 ins_pipe(ialu_reg_shift);
11271 %}
11272
11273 // This pattern is automatically generated from aarch64_ad.m4
11274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11275 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11276 immL0 zero, iRegL src1, immI src2) %{
11277 match(Set dst (SubL zero (RShiftL src1 src2)));
11278
11279 ins_cost(1.9 * INSN_COST);
11280 format %{ "neg $dst, $src1, ASR $src2" %}
11281
11282 ins_encode %{
11283 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11284 Assembler::ASR, $src2$$constant & 0x3f);
11285 %}
11286
11287 ins_pipe(ialu_reg_shift);
11288 %}
11289
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11293 immL0 zero, iRegL src1, immI src2) %{
11294 match(Set dst (SubL zero (LShiftL src1 src2)));
11295
11296 ins_cost(1.9 * INSN_COST);
11297 format %{ "neg $dst, $src1, LSL $src2" %}
11298
11299 ins_encode %{
11300 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11301 Assembler::LSL, $src2$$constant & 0x3f);
11302 %}
11303
11304 ins_pipe(ialu_reg_shift);
11305 %}
11306
11307 // This pattern is automatically generated from aarch64_ad.m4
11308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11309 instruct AndI_reg_not_reg(iRegINoSp dst,
11310 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11311 match(Set dst (AndI src1 (XorI src2 m1)));
11312 ins_cost(INSN_COST);
11313 format %{ "bicw $dst, $src1, $src2" %}
11314
11315 ins_encode %{
11316 __ bicw(as_Register($dst$$reg),
11317 as_Register($src1$$reg),
11318 as_Register($src2$$reg),
11319 Assembler::LSL, 0);
11320 %}
11321
11322 ins_pipe(ialu_reg_reg);
11323 %}
11324
11325 // This pattern is automatically generated from aarch64_ad.m4
11326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11327 instruct AndL_reg_not_reg(iRegLNoSp dst,
11328 iRegL src1, iRegL src2, immL_M1 m1) %{
11329 match(Set dst (AndL src1 (XorL src2 m1)));
11330 ins_cost(INSN_COST);
11331 format %{ "bic $dst, $src1, $src2" %}
11332
11333 ins_encode %{
11334 __ bic(as_Register($dst$$reg),
11335 as_Register($src1$$reg),
11336 as_Register($src2$$reg),
11337 Assembler::LSL, 0);
11338 %}
11339
11340 ins_pipe(ialu_reg_reg);
11341 %}
11342
11343 // This pattern is automatically generated from aarch64_ad.m4
11344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11345 instruct OrI_reg_not_reg(iRegINoSp dst,
11346 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11347 match(Set dst (OrI src1 (XorI src2 m1)));
11348 ins_cost(INSN_COST);
11349 format %{ "ornw $dst, $src1, $src2" %}
11350
11351 ins_encode %{
11352 __ ornw(as_Register($dst$$reg),
11353 as_Register($src1$$reg),
11354 as_Register($src2$$reg),
11355 Assembler::LSL, 0);
11356 %}
11357
11358 ins_pipe(ialu_reg_reg);
11359 %}
11360
11361 // This pattern is automatically generated from aarch64_ad.m4
11362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11363 instruct OrL_reg_not_reg(iRegLNoSp dst,
11364 iRegL src1, iRegL src2, immL_M1 m1) %{
11365 match(Set dst (OrL src1 (XorL src2 m1)));
11366 ins_cost(INSN_COST);
11367 format %{ "orn $dst, $src1, $src2" %}
11368
11369 ins_encode %{
11370 __ orn(as_Register($dst$$reg),
11371 as_Register($src1$$reg),
11372 as_Register($src2$$reg),
11373 Assembler::LSL, 0);
11374 %}
11375
11376 ins_pipe(ialu_reg_reg);
11377 %}
11378
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 instruct XorI_reg_not_reg(iRegINoSp dst,
11382 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11383 match(Set dst (XorI m1 (XorI src2 src1)));
11384 ins_cost(INSN_COST);
11385 format %{ "eonw $dst, $src1, $src2" %}
11386
11387 ins_encode %{
11388 __ eonw(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg),
11391 Assembler::LSL, 0);
11392 %}
11393
11394 ins_pipe(ialu_reg_reg);
11395 %}
11396
11397 // This pattern is automatically generated from aarch64_ad.m4
11398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11399 instruct XorL_reg_not_reg(iRegLNoSp dst,
11400 iRegL src1, iRegL src2, immL_M1 m1) %{
11401 match(Set dst (XorL m1 (XorL src2 src1)));
11402 ins_cost(INSN_COST);
11403 format %{ "eon $dst, $src1, $src2" %}
11404
11405 ins_encode %{
11406 __ eon(as_Register($dst$$reg),
11407 as_Register($src1$$reg),
11408 as_Register($src2$$reg),
11409 Assembler::LSL, 0);
11410 %}
11411
11412 ins_pipe(ialu_reg_reg);
11413 %}
11414
11415 // This pattern is automatically generated from aarch64_ad.m4
11416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11417 // val & (-1 ^ (val >>> shift)) ==> bicw
11418 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11419 iRegIorL2I src1, iRegIorL2I src2,
11420 immI src3, immI_M1 src4) %{
11421 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11422 ins_cost(1.9 * INSN_COST);
11423 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11424
11425 ins_encode %{
11426 __ bicw(as_Register($dst$$reg),
11427 as_Register($src1$$reg),
11428 as_Register($src2$$reg),
11429 Assembler::LSR,
11430 $src3$$constant & 0x1f);
11431 %}
11432
11433 ins_pipe(ialu_reg_reg_shift);
11434 %}
11435
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 // val & (-1 ^ (val >>> shift)) ==> bic
11439 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11440 iRegL src1, iRegL src2,
11441 immI src3, immL_M1 src4) %{
11442 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11443 ins_cost(1.9 * INSN_COST);
11444 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11445
11446 ins_encode %{
11447 __ bic(as_Register($dst$$reg),
11448 as_Register($src1$$reg),
11449 as_Register($src2$$reg),
11450 Assembler::LSR,
11451 $src3$$constant & 0x3f);
11452 %}
11453
11454 ins_pipe(ialu_reg_reg_shift);
11455 %}
11456
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459 // val & (-1 ^ (val >> shift)) ==> bicw
11460 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11461 iRegIorL2I src1, iRegIorL2I src2,
11462 immI src3, immI_M1 src4) %{
11463 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11464 ins_cost(1.9 * INSN_COST);
11465 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11466
11467 ins_encode %{
11468 __ bicw(as_Register($dst$$reg),
11469 as_Register($src1$$reg),
11470 as_Register($src2$$reg),
11471 Assembler::ASR,
11472 $src3$$constant & 0x1f);
11473 %}
11474
11475 ins_pipe(ialu_reg_reg_shift);
11476 %}
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 // val & (-1 ^ (val >> shift)) ==> bic
11481 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11482 iRegL src1, iRegL src2,
11483 immI src3, immL_M1 src4) %{
11484 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11485 ins_cost(1.9 * INSN_COST);
11486 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11487
11488 ins_encode %{
11489 __ bic(as_Register($dst$$reg),
11490 as_Register($src1$$reg),
11491 as_Register($src2$$reg),
11492 Assembler::ASR,
11493 $src3$$constant & 0x3f);
11494 %}
11495
11496 ins_pipe(ialu_reg_reg_shift);
11497 %}
11498
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 // val & (-1 ^ (val ror shift)) ==> bicw
11502 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11503 iRegIorL2I src1, iRegIorL2I src2,
11504 immI src3, immI_M1 src4) %{
11505 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11506 ins_cost(1.9 * INSN_COST);
11507 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11508
11509 ins_encode %{
11510 __ bicw(as_Register($dst$$reg),
11511 as_Register($src1$$reg),
11512 as_Register($src2$$reg),
11513 Assembler::ROR,
11514 $src3$$constant & 0x1f);
11515 %}
11516
11517 ins_pipe(ialu_reg_reg_shift);
11518 %}
11519
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522 // val & (-1 ^ (val ror shift)) ==> bic
11523 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11524 iRegL src1, iRegL src2,
11525 immI src3, immL_M1 src4) %{
11526 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11527 ins_cost(1.9 * INSN_COST);
11528 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11529
11530 ins_encode %{
11531 __ bic(as_Register($dst$$reg),
11532 as_Register($src1$$reg),
11533 as_Register($src2$$reg),
11534 Assembler::ROR,
11535 $src3$$constant & 0x3f);
11536 %}
11537
11538 ins_pipe(ialu_reg_reg_shift);
11539 %}
11540
11541 // This pattern is automatically generated from aarch64_ad.m4
11542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11543 // val & (-1 ^ (val << shift)) ==> bicw
11544 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11545 iRegIorL2I src1, iRegIorL2I src2,
11546 immI src3, immI_M1 src4) %{
11547 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11548 ins_cost(1.9 * INSN_COST);
11549 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11550
11551 ins_encode %{
11552 __ bicw(as_Register($dst$$reg),
11553 as_Register($src1$$reg),
11554 as_Register($src2$$reg),
11555 Assembler::LSL,
11556 $src3$$constant & 0x1f);
11557 %}
11558
11559 ins_pipe(ialu_reg_reg_shift);
11560 %}
11561
11562 // This pattern is automatically generated from aarch64_ad.m4
11563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11564 // val & (-1 ^ (val << shift)) ==> bic
11565 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11566 iRegL src1, iRegL src2,
11567 immI src3, immL_M1 src4) %{
11568 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11569 ins_cost(1.9 * INSN_COST);
11570 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11571
11572 ins_encode %{
11573 __ bic(as_Register($dst$$reg),
11574 as_Register($src1$$reg),
11575 as_Register($src2$$reg),
11576 Assembler::LSL,
11577 $src3$$constant & 0x3f);
11578 %}
11579
11580 ins_pipe(ialu_reg_reg_shift);
11581 %}
11582
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11586 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11587 iRegIorL2I src1, iRegIorL2I src2,
11588 immI src3, immI_M1 src4) %{
11589 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11590 ins_cost(1.9 * INSN_COST);
11591 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11592
11593 ins_encode %{
11594 __ eonw(as_Register($dst$$reg),
11595 as_Register($src1$$reg),
11596 as_Register($src2$$reg),
11597 Assembler::LSR,
11598 $src3$$constant & 0x1f);
11599 %}
11600
11601 ins_pipe(ialu_reg_reg_shift);
11602 %}
11603
11604 // This pattern is automatically generated from aarch64_ad.m4
11605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11606 // val ^ (-1 ^ (val >>> shift)) ==> eon
11607 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11608 iRegL src1, iRegL src2,
11609 immI src3, immL_M1 src4) %{
11610 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11611 ins_cost(1.9 * INSN_COST);
11612 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11613
11614 ins_encode %{
11615 __ eon(as_Register($dst$$reg),
11616 as_Register($src1$$reg),
11617 as_Register($src2$$reg),
11618 Assembler::LSR,
11619 $src3$$constant & 0x3f);
11620 %}
11621
11622 ins_pipe(ialu_reg_reg_shift);
11623 %}
11624
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 // val ^ (-1 ^ (val >> shift)) ==> eonw
11628 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11629 iRegIorL2I src1, iRegIorL2I src2,
11630 immI src3, immI_M1 src4) %{
11631 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11632 ins_cost(1.9 * INSN_COST);
11633 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11634
11635 ins_encode %{
11636 __ eonw(as_Register($dst$$reg),
11637 as_Register($src1$$reg),
11638 as_Register($src2$$reg),
11639 Assembler::ASR,
11640 $src3$$constant & 0x1f);
11641 %}
11642
11643 ins_pipe(ialu_reg_reg_shift);
11644 %}
11645
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648 // val ^ (-1 ^ (val >> shift)) ==> eon
11649 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11650 iRegL src1, iRegL src2,
11651 immI src3, immL_M1 src4) %{
11652 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11653 ins_cost(1.9 * INSN_COST);
11654 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11655
11656 ins_encode %{
11657 __ eon(as_Register($dst$$reg),
11658 as_Register($src1$$reg),
11659 as_Register($src2$$reg),
11660 Assembler::ASR,
11661 $src3$$constant & 0x3f);
11662 %}
11663
11664 ins_pipe(ialu_reg_reg_shift);
11665 %}
11666
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669 // val ^ (-1 ^ (val ror shift)) ==> eonw
11670 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11671 iRegIorL2I src1, iRegIorL2I src2,
11672 immI src3, immI_M1 src4) %{
11673 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11674 ins_cost(1.9 * INSN_COST);
11675 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11676
11677 ins_encode %{
11678 __ eonw(as_Register($dst$$reg),
11679 as_Register($src1$$reg),
11680 as_Register($src2$$reg),
11681 Assembler::ROR,
11682 $src3$$constant & 0x1f);
11683 %}
11684
11685 ins_pipe(ialu_reg_reg_shift);
11686 %}
11687
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 // val ^ (-1 ^ (val ror shift)) ==> eon
11691 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11692 iRegL src1, iRegL src2,
11693 immI src3, immL_M1 src4) %{
11694 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11695 ins_cost(1.9 * INSN_COST);
11696 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11697
11698 ins_encode %{
11699 __ eon(as_Register($dst$$reg),
11700 as_Register($src1$$reg),
11701 as_Register($src2$$reg),
11702 Assembler::ROR,
11703 $src3$$constant & 0x3f);
11704 %}
11705
11706 ins_pipe(ialu_reg_reg_shift);
11707 %}
11708
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 // val ^ (-1 ^ (val << shift)) ==> eonw
11712 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11713 iRegIorL2I src1, iRegIorL2I src2,
11714 immI src3, immI_M1 src4) %{
11715 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11716 ins_cost(1.9 * INSN_COST);
11717 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11718
11719 ins_encode %{
11720 __ eonw(as_Register($dst$$reg),
11721 as_Register($src1$$reg),
11722 as_Register($src2$$reg),
11723 Assembler::LSL,
11724 $src3$$constant & 0x1f);
11725 %}
11726
11727 ins_pipe(ialu_reg_reg_shift);
11728 %}
11729
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732 // val ^ (-1 ^ (val << shift)) ==> eon
11733 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11734 iRegL src1, iRegL src2,
11735 immI src3, immL_M1 src4) %{
11736 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11737 ins_cost(1.9 * INSN_COST);
11738 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11739
11740 ins_encode %{
11741 __ eon(as_Register($dst$$reg),
11742 as_Register($src1$$reg),
11743 as_Register($src2$$reg),
11744 Assembler::LSL,
11745 $src3$$constant & 0x3f);
11746 %}
11747
11748 ins_pipe(ialu_reg_reg_shift);
11749 %}
11750
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 // val | (-1 ^ (val >>> shift)) ==> ornw
11754 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11755 iRegIorL2I src1, iRegIorL2I src2,
11756 immI src3, immI_M1 src4) %{
11757 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11758 ins_cost(1.9 * INSN_COST);
11759 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11760
11761 ins_encode %{
11762 __ ornw(as_Register($dst$$reg),
11763 as_Register($src1$$reg),
11764 as_Register($src2$$reg),
11765 Assembler::LSR,
11766 $src3$$constant & 0x1f);
11767 %}
11768
11769 ins_pipe(ialu_reg_reg_shift);
11770 %}
11771
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 // val | (-1 ^ (val >>> shift)) ==> orn
11775 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11776 iRegL src1, iRegL src2,
11777 immI src3, immL_M1 src4) %{
11778 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11779 ins_cost(1.9 * INSN_COST);
11780 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11781
11782 ins_encode %{
11783 __ orn(as_Register($dst$$reg),
11784 as_Register($src1$$reg),
11785 as_Register($src2$$reg),
11786 Assembler::LSR,
11787 $src3$$constant & 0x3f);
11788 %}
11789
11790 ins_pipe(ialu_reg_reg_shift);
11791 %}
11792
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 // val | (-1 ^ (val >> shift)) ==> ornw
11796 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11797 iRegIorL2I src1, iRegIorL2I src2,
11798 immI src3, immI_M1 src4) %{
11799 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11800 ins_cost(1.9 * INSN_COST);
11801 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11802
11803 ins_encode %{
11804 __ ornw(as_Register($dst$$reg),
11805 as_Register($src1$$reg),
11806 as_Register($src2$$reg),
11807 Assembler::ASR,
11808 $src3$$constant & 0x1f);
11809 %}
11810
11811 ins_pipe(ialu_reg_reg_shift);
11812 %}
11813
11814 // This pattern is automatically generated from aarch64_ad.m4
11815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11816 // val | (-1 ^ (val >> shift)) ==> orn
11817 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11818 iRegL src1, iRegL src2,
11819 immI src3, immL_M1 src4) %{
11820 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11821 ins_cost(1.9 * INSN_COST);
11822 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11823
11824 ins_encode %{
11825 __ orn(as_Register($dst$$reg),
11826 as_Register($src1$$reg),
11827 as_Register($src2$$reg),
11828 Assembler::ASR,
11829 $src3$$constant & 0x3f);
11830 %}
11831
11832 ins_pipe(ialu_reg_reg_shift);
11833 %}
11834
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 // val | (-1 ^ (val ror shift)) ==> ornw
11838 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11839 iRegIorL2I src1, iRegIorL2I src2,
11840 immI src3, immI_M1 src4) %{
11841 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11842 ins_cost(1.9 * INSN_COST);
11843 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11844
11845 ins_encode %{
11846 __ ornw(as_Register($dst$$reg),
11847 as_Register($src1$$reg),
11848 as_Register($src2$$reg),
11849 Assembler::ROR,
11850 $src3$$constant & 0x1f);
11851 %}
11852
11853 ins_pipe(ialu_reg_reg_shift);
11854 %}
11855
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 // val | (-1 ^ (val ror shift)) ==> orn
11859 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11860 iRegL src1, iRegL src2,
11861 immI src3, immL_M1 src4) %{
11862 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11863 ins_cost(1.9 * INSN_COST);
11864 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11865
11866 ins_encode %{
11867 __ orn(as_Register($dst$$reg),
11868 as_Register($src1$$reg),
11869 as_Register($src2$$reg),
11870 Assembler::ROR,
11871 $src3$$constant & 0x3f);
11872 %}
11873
11874 ins_pipe(ialu_reg_reg_shift);
11875 %}
11876
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 // val | (-1 ^ (val << shift)) ==> ornw
11880 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11881 iRegIorL2I src1, iRegIorL2I src2,
11882 immI src3, immI_M1 src4) %{
11883 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11884 ins_cost(1.9 * INSN_COST);
11885 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11886
11887 ins_encode %{
11888 __ ornw(as_Register($dst$$reg),
11889 as_Register($src1$$reg),
11890 as_Register($src2$$reg),
11891 Assembler::LSL,
11892 $src3$$constant & 0x1f);
11893 %}
11894
11895 ins_pipe(ialu_reg_reg_shift);
11896 %}
11897
11898 // This pattern is automatically generated from aarch64_ad.m4
11899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11900 // val | (-1 ^ (val << shift)) ==> orn
11901 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11902 iRegL src1, iRegL src2,
11903 immI src3, immL_M1 src4) %{
11904 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11905 ins_cost(1.9 * INSN_COST);
11906 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11907
11908 ins_encode %{
11909 __ orn(as_Register($dst$$reg),
11910 as_Register($src1$$reg),
11911 as_Register($src2$$reg),
11912 Assembler::LSL,
11913 $src3$$constant & 0x3f);
11914 %}
11915
11916 ins_pipe(ialu_reg_reg_shift);
11917 %}
11918
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11922 iRegIorL2I src1, iRegIorL2I src2,
11923 immI src3) %{
11924 match(Set dst (AndI src1 (URShiftI src2 src3)));
11925
11926 ins_cost(1.9 * INSN_COST);
11927 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11928
11929 ins_encode %{
11930 __ andw(as_Register($dst$$reg),
11931 as_Register($src1$$reg),
11932 as_Register($src2$$reg),
11933 Assembler::LSR,
11934 $src3$$constant & 0x1f);
11935 %}
11936
11937 ins_pipe(ialu_reg_reg_shift);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11943 iRegL src1, iRegL src2,
11944 immI src3) %{
11945 match(Set dst (AndL src1 (URShiftL src2 src3)));
11946
11947 ins_cost(1.9 * INSN_COST);
11948 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11949
11950 ins_encode %{
11951 __ andr(as_Register($dst$$reg),
11952 as_Register($src1$$reg),
11953 as_Register($src2$$reg),
11954 Assembler::LSR,
11955 $src3$$constant & 0x3f);
11956 %}
11957
11958 ins_pipe(ialu_reg_reg_shift);
11959 %}
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11964 iRegIorL2I src1, iRegIorL2I src2,
11965 immI src3) %{
11966 match(Set dst (AndI src1 (RShiftI src2 src3)));
11967
11968 ins_cost(1.9 * INSN_COST);
11969 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11970
11971 ins_encode %{
11972 __ andw(as_Register($dst$$reg),
11973 as_Register($src1$$reg),
11974 as_Register($src2$$reg),
11975 Assembler::ASR,
11976 $src3$$constant & 0x1f);
11977 %}
11978
11979 ins_pipe(ialu_reg_reg_shift);
11980 %}
11981
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11985 iRegL src1, iRegL src2,
11986 immI src3) %{
11987 match(Set dst (AndL src1 (RShiftL src2 src3)));
11988
11989 ins_cost(1.9 * INSN_COST);
11990 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11991
11992 ins_encode %{
11993 __ andr(as_Register($dst$$reg),
11994 as_Register($src1$$reg),
11995 as_Register($src2$$reg),
11996 Assembler::ASR,
11997 $src3$$constant & 0x3f);
11998 %}
11999
12000 ins_pipe(ialu_reg_reg_shift);
12001 %}
12002
12003 // This pattern is automatically generated from aarch64_ad.m4
12004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12005 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12006 iRegIorL2I src1, iRegIorL2I src2,
12007 immI src3) %{
12008 match(Set dst (AndI src1 (LShiftI src2 src3)));
12009
12010 ins_cost(1.9 * INSN_COST);
12011 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12012
12013 ins_encode %{
12014 __ andw(as_Register($dst$$reg),
12015 as_Register($src1$$reg),
12016 as_Register($src2$$reg),
12017 Assembler::LSL,
12018 $src3$$constant & 0x1f);
12019 %}
12020
12021 ins_pipe(ialu_reg_reg_shift);
12022 %}
12023
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12027 iRegL src1, iRegL src2,
12028 immI src3) %{
12029 match(Set dst (AndL src1 (LShiftL src2 src3)));
12030
12031 ins_cost(1.9 * INSN_COST);
12032 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12033
12034 ins_encode %{
12035 __ andr(as_Register($dst$$reg),
12036 as_Register($src1$$reg),
12037 as_Register($src2$$reg),
12038 Assembler::LSL,
12039 $src3$$constant & 0x3f);
12040 %}
12041
12042 ins_pipe(ialu_reg_reg_shift);
12043 %}
12044
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12048 iRegIorL2I src1, iRegIorL2I src2,
12049 immI src3) %{
12050 match(Set dst (AndI src1 (RotateRight src2 src3)));
12051
12052 ins_cost(1.9 * INSN_COST);
12053 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12054
12055 ins_encode %{
12056 __ andw(as_Register($dst$$reg),
12057 as_Register($src1$$reg),
12058 as_Register($src2$$reg),
12059 Assembler::ROR,
12060 $src3$$constant & 0x1f);
12061 %}
12062
12063 ins_pipe(ialu_reg_reg_shift);
12064 %}
12065
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12069 iRegL src1, iRegL src2,
12070 immI src3) %{
12071 match(Set dst (AndL src1 (RotateRight src2 src3)));
12072
12073 ins_cost(1.9 * INSN_COST);
12074 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12075
12076 ins_encode %{
12077 __ andr(as_Register($dst$$reg),
12078 as_Register($src1$$reg),
12079 as_Register($src2$$reg),
12080 Assembler::ROR,
12081 $src3$$constant & 0x3f);
12082 %}
12083
12084 ins_pipe(ialu_reg_reg_shift);
12085 %}
12086
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12090 iRegIorL2I src1, iRegIorL2I src2,
12091 immI src3) %{
12092 match(Set dst (XorI src1 (URShiftI src2 src3)));
12093
12094 ins_cost(1.9 * INSN_COST);
12095 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12096
12097 ins_encode %{
12098 __ eorw(as_Register($dst$$reg),
12099 as_Register($src1$$reg),
12100 as_Register($src2$$reg),
12101 Assembler::LSR,
12102 $src3$$constant & 0x1f);
12103 %}
12104
12105 ins_pipe(ialu_reg_reg_shift);
12106 %}
12107
12108 // This pattern is automatically generated from aarch64_ad.m4
12109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12110 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12111 iRegL src1, iRegL src2,
12112 immI src3) %{
12113 match(Set dst (XorL src1 (URShiftL src2 src3)));
12114
12115 ins_cost(1.9 * INSN_COST);
12116 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12117
12118 ins_encode %{
12119 __ eor(as_Register($dst$$reg),
12120 as_Register($src1$$reg),
12121 as_Register($src2$$reg),
12122 Assembler::LSR,
12123 $src3$$constant & 0x3f);
12124 %}
12125
12126 ins_pipe(ialu_reg_reg_shift);
12127 %}
12128
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12132 iRegIorL2I src1, iRegIorL2I src2,
12133 immI src3) %{
12134 match(Set dst (XorI src1 (RShiftI src2 src3)));
12135
12136 ins_cost(1.9 * INSN_COST);
12137 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12138
12139 ins_encode %{
12140 __ eorw(as_Register($dst$$reg),
12141 as_Register($src1$$reg),
12142 as_Register($src2$$reg),
12143 Assembler::ASR,
12144 $src3$$constant & 0x1f);
12145 %}
12146
12147 ins_pipe(ialu_reg_reg_shift);
12148 %}
12149
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12153 iRegL src1, iRegL src2,
12154 immI src3) %{
12155 match(Set dst (XorL src1 (RShiftL src2 src3)));
12156
12157 ins_cost(1.9 * INSN_COST);
12158 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12159
12160 ins_encode %{
12161 __ eor(as_Register($dst$$reg),
12162 as_Register($src1$$reg),
12163 as_Register($src2$$reg),
12164 Assembler::ASR,
12165 $src3$$constant & 0x3f);
12166 %}
12167
12168 ins_pipe(ialu_reg_reg_shift);
12169 %}
12170
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12174 iRegIorL2I src1, iRegIorL2I src2,
12175 immI src3) %{
12176 match(Set dst (XorI src1 (LShiftI src2 src3)));
12177
12178 ins_cost(1.9 * INSN_COST);
12179 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12180
12181 ins_encode %{
12182 __ eorw(as_Register($dst$$reg),
12183 as_Register($src1$$reg),
12184 as_Register($src2$$reg),
12185 Assembler::LSL,
12186 $src3$$constant & 0x1f);
12187 %}
12188
12189 ins_pipe(ialu_reg_reg_shift);
12190 %}
12191
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12195 iRegL src1, iRegL src2,
12196 immI src3) %{
12197 match(Set dst (XorL src1 (LShiftL src2 src3)));
12198
12199 ins_cost(1.9 * INSN_COST);
12200 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12201
12202 ins_encode %{
12203 __ eor(as_Register($dst$$reg),
12204 as_Register($src1$$reg),
12205 as_Register($src2$$reg),
12206 Assembler::LSL,
12207 $src3$$constant & 0x3f);
12208 %}
12209
12210 ins_pipe(ialu_reg_reg_shift);
12211 %}
12212
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12216 iRegIorL2I src1, iRegIorL2I src2,
12217 immI src3) %{
12218 match(Set dst (XorI src1 (RotateRight src2 src3)));
12219
12220 ins_cost(1.9 * INSN_COST);
12221 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12222
12223 ins_encode %{
12224 __ eorw(as_Register($dst$$reg),
12225 as_Register($src1$$reg),
12226 as_Register($src2$$reg),
12227 Assembler::ROR,
12228 $src3$$constant & 0x1f);
12229 %}
12230
12231 ins_pipe(ialu_reg_reg_shift);
12232 %}
12233
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12237 iRegL src1, iRegL src2,
12238 immI src3) %{
12239 match(Set dst (XorL src1 (RotateRight src2 src3)));
12240
12241 ins_cost(1.9 * INSN_COST);
12242 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12243
12244 ins_encode %{
12245 __ eor(as_Register($dst$$reg),
12246 as_Register($src1$$reg),
12247 as_Register($src2$$reg),
12248 Assembler::ROR,
12249 $src3$$constant & 0x3f);
12250 %}
12251
12252 ins_pipe(ialu_reg_reg_shift);
12253 %}
12254
12255 // This pattern is automatically generated from aarch64_ad.m4
12256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12257 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12258 iRegIorL2I src1, iRegIorL2I src2,
12259 immI src3) %{
12260 match(Set dst (OrI src1 (URShiftI src2 src3)));
12261
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12264
12265 ins_encode %{
12266 __ orrw(as_Register($dst$$reg),
12267 as_Register($src1$$reg),
12268 as_Register($src2$$reg),
12269 Assembler::LSR,
12270 $src3$$constant & 0x1f);
12271 %}
12272
12273 ins_pipe(ialu_reg_reg_shift);
12274 %}
12275
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12279 iRegL src1, iRegL src2,
12280 immI src3) %{
12281 match(Set dst (OrL src1 (URShiftL src2 src3)));
12282
12283 ins_cost(1.9 * INSN_COST);
12284 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12285
12286 ins_encode %{
12287 __ orr(as_Register($dst$$reg),
12288 as_Register($src1$$reg),
12289 as_Register($src2$$reg),
12290 Assembler::LSR,
12291 $src3$$constant & 0x3f);
12292 %}
12293
12294 ins_pipe(ialu_reg_reg_shift);
12295 %}
12296
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12300 iRegIorL2I src1, iRegIorL2I src2,
12301 immI src3) %{
12302 match(Set dst (OrI src1 (RShiftI src2 src3)));
12303
12304 ins_cost(1.9 * INSN_COST);
12305 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12306
12307 ins_encode %{
12308 __ orrw(as_Register($dst$$reg),
12309 as_Register($src1$$reg),
12310 as_Register($src2$$reg),
12311 Assembler::ASR,
12312 $src3$$constant & 0x1f);
12313 %}
12314
12315 ins_pipe(ialu_reg_reg_shift);
12316 %}
12317
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12321 iRegL src1, iRegL src2,
12322 immI src3) %{
12323 match(Set dst (OrL src1 (RShiftL src2 src3)));
12324
12325 ins_cost(1.9 * INSN_COST);
12326 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12327
12328 ins_encode %{
12329 __ orr(as_Register($dst$$reg),
12330 as_Register($src1$$reg),
12331 as_Register($src2$$reg),
12332 Assembler::ASR,
12333 $src3$$constant & 0x3f);
12334 %}
12335
12336 ins_pipe(ialu_reg_reg_shift);
12337 %}
12338
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12342 iRegIorL2I src1, iRegIorL2I src2,
12343 immI src3) %{
12344 match(Set dst (OrI src1 (LShiftI src2 src3)));
12345
12346 ins_cost(1.9 * INSN_COST);
12347 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12348
12349 ins_encode %{
12350 __ orrw(as_Register($dst$$reg),
12351 as_Register($src1$$reg),
12352 as_Register($src2$$reg),
12353 Assembler::LSL,
12354 $src3$$constant & 0x1f);
12355 %}
12356
12357 ins_pipe(ialu_reg_reg_shift);
12358 %}
12359
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12363 iRegL src1, iRegL src2,
12364 immI src3) %{
12365 match(Set dst (OrL src1 (LShiftL src2 src3)));
12366
12367 ins_cost(1.9 * INSN_COST);
12368 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12369
12370 ins_encode %{
12371 __ orr(as_Register($dst$$reg),
12372 as_Register($src1$$reg),
12373 as_Register($src2$$reg),
12374 Assembler::LSL,
12375 $src3$$constant & 0x3f);
12376 %}
12377
12378 ins_pipe(ialu_reg_reg_shift);
12379 %}
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12384 iRegIorL2I src1, iRegIorL2I src2,
12385 immI src3) %{
12386 match(Set dst (OrI src1 (RotateRight src2 src3)));
12387
12388 ins_cost(1.9 * INSN_COST);
12389 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12390
12391 ins_encode %{
12392 __ orrw(as_Register($dst$$reg),
12393 as_Register($src1$$reg),
12394 as_Register($src2$$reg),
12395 Assembler::ROR,
12396 $src3$$constant & 0x1f);
12397 %}
12398
12399 ins_pipe(ialu_reg_reg_shift);
12400 %}
12401
12402 // This pattern is automatically generated from aarch64_ad.m4
12403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12404 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12405 iRegL src1, iRegL src2,
12406 immI src3) %{
12407 match(Set dst (OrL src1 (RotateRight src2 src3)));
12408
12409 ins_cost(1.9 * INSN_COST);
12410 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12411
12412 ins_encode %{
12413 __ orr(as_Register($dst$$reg),
12414 as_Register($src1$$reg),
12415 as_Register($src2$$reg),
12416 Assembler::ROR,
12417 $src3$$constant & 0x3f);
12418 %}
12419
12420 ins_pipe(ialu_reg_reg_shift);
12421 %}
12422
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12426 iRegIorL2I src1, iRegIorL2I src2,
12427 immI src3) %{
12428 match(Set dst (AddI src1 (URShiftI src2 src3)));
12429
12430 ins_cost(1.9 * INSN_COST);
12431 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12432
12433 ins_encode %{
12434 __ addw(as_Register($dst$$reg),
12435 as_Register($src1$$reg),
12436 as_Register($src2$$reg),
12437 Assembler::LSR,
12438 $src3$$constant & 0x1f);
12439 %}
12440
12441 ins_pipe(ialu_reg_reg_shift);
12442 %}
12443
12444 // This pattern is automatically generated from aarch64_ad.m4
12445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12446 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12447 iRegL src1, iRegL src2,
12448 immI src3) %{
12449 match(Set dst (AddL src1 (URShiftL src2 src3)));
12450
12451 ins_cost(1.9 * INSN_COST);
12452 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12453
12454 ins_encode %{
12455 __ add(as_Register($dst$$reg),
12456 as_Register($src1$$reg),
12457 as_Register($src2$$reg),
12458 Assembler::LSR,
12459 $src3$$constant & 0x3f);
12460 %}
12461
12462 ins_pipe(ialu_reg_reg_shift);
12463 %}
12464
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12468 iRegIorL2I src1, iRegIorL2I src2,
12469 immI src3) %{
12470 match(Set dst (AddI src1 (RShiftI src2 src3)));
12471
12472 ins_cost(1.9 * INSN_COST);
12473 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12474
12475 ins_encode %{
12476 __ addw(as_Register($dst$$reg),
12477 as_Register($src1$$reg),
12478 as_Register($src2$$reg),
12479 Assembler::ASR,
12480 $src3$$constant & 0x1f);
12481 %}
12482
12483 ins_pipe(ialu_reg_reg_shift);
12484 %}
12485
12486 // This pattern is automatically generated from aarch64_ad.m4
12487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12488 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12489 iRegL src1, iRegL src2,
12490 immI src3) %{
12491 match(Set dst (AddL src1 (RShiftL src2 src3)));
12492
12493 ins_cost(1.9 * INSN_COST);
12494 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12495
12496 ins_encode %{
12497 __ add(as_Register($dst$$reg),
12498 as_Register($src1$$reg),
12499 as_Register($src2$$reg),
12500 Assembler::ASR,
12501 $src3$$constant & 0x3f);
12502 %}
12503
12504 ins_pipe(ialu_reg_reg_shift);
12505 %}
12506
12507 // This pattern is automatically generated from aarch64_ad.m4
12508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12509 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12510 iRegIorL2I src1, iRegIorL2I src2,
12511 immI src3) %{
12512 match(Set dst (AddI src1 (LShiftI src2 src3)));
12513
12514 ins_cost(1.9 * INSN_COST);
12515 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12516
12517 ins_encode %{
12518 __ addw(as_Register($dst$$reg),
12519 as_Register($src1$$reg),
12520 as_Register($src2$$reg),
12521 Assembler::LSL,
12522 $src3$$constant & 0x1f);
12523 %}
12524
12525 ins_pipe(ialu_reg_reg_shift);
12526 %}
12527
12528 // This pattern is automatically generated from aarch64_ad.m4
12529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12530 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12531 iRegL src1, iRegL src2,
12532 immI src3) %{
12533 match(Set dst (AddL src1 (LShiftL src2 src3)));
12534
12535 ins_cost(1.9 * INSN_COST);
12536 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12537
12538 ins_encode %{
12539 __ add(as_Register($dst$$reg),
12540 as_Register($src1$$reg),
12541 as_Register($src2$$reg),
12542 Assembler::LSL,
12543 $src3$$constant & 0x3f);
12544 %}
12545
12546 ins_pipe(ialu_reg_reg_shift);
12547 %}
12548
12549 // This pattern is automatically generated from aarch64_ad.m4
12550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12551 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12552 iRegIorL2I src1, iRegIorL2I src2,
12553 immI src3) %{
12554 match(Set dst (SubI src1 (URShiftI src2 src3)));
12555
12556 ins_cost(1.9 * INSN_COST);
12557 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12558
12559 ins_encode %{
12560 __ subw(as_Register($dst$$reg),
12561 as_Register($src1$$reg),
12562 as_Register($src2$$reg),
12563 Assembler::LSR,
12564 $src3$$constant & 0x1f);
12565 %}
12566
12567 ins_pipe(ialu_reg_reg_shift);
12568 %}
12569
12570 // This pattern is automatically generated from aarch64_ad.m4
12571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12572 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12573 iRegL src1, iRegL src2,
12574 immI src3) %{
12575 match(Set dst (SubL src1 (URShiftL src2 src3)));
12576
12577 ins_cost(1.9 * INSN_COST);
12578 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12579
12580 ins_encode %{
12581 __ sub(as_Register($dst$$reg),
12582 as_Register($src1$$reg),
12583 as_Register($src2$$reg),
12584 Assembler::LSR,
12585 $src3$$constant & 0x3f);
12586 %}
12587
12588 ins_pipe(ialu_reg_reg_shift);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12594 iRegIorL2I src1, iRegIorL2I src2,
12595 immI src3) %{
12596 match(Set dst (SubI src1 (RShiftI src2 src3)));
12597
12598 ins_cost(1.9 * INSN_COST);
12599 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12600
12601 ins_encode %{
12602 __ subw(as_Register($dst$$reg),
12603 as_Register($src1$$reg),
12604 as_Register($src2$$reg),
12605 Assembler::ASR,
12606 $src3$$constant & 0x1f);
12607 %}
12608
12609 ins_pipe(ialu_reg_reg_shift);
12610 %}
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12615 iRegL src1, iRegL src2,
12616 immI src3) %{
12617 match(Set dst (SubL src1 (RShiftL src2 src3)));
12618
12619 ins_cost(1.9 * INSN_COST);
12620 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12621
12622 ins_encode %{
12623 __ sub(as_Register($dst$$reg),
12624 as_Register($src1$$reg),
12625 as_Register($src2$$reg),
12626 Assembler::ASR,
12627 $src3$$constant & 0x3f);
12628 %}
12629
12630 ins_pipe(ialu_reg_reg_shift);
12631 %}
12632
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12636 iRegIorL2I src1, iRegIorL2I src2,
12637 immI src3) %{
12638 match(Set dst (SubI src1 (LShiftI src2 src3)));
12639
12640 ins_cost(1.9 * INSN_COST);
12641 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12642
12643 ins_encode %{
12644 __ subw(as_Register($dst$$reg),
12645 as_Register($src1$$reg),
12646 as_Register($src2$$reg),
12647 Assembler::LSL,
12648 $src3$$constant & 0x1f);
12649 %}
12650
12651 ins_pipe(ialu_reg_reg_shift);
12652 %}
12653
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12657 iRegL src1, iRegL src2,
12658 immI src3) %{
12659 match(Set dst (SubL src1 (LShiftL src2 src3)));
12660
12661 ins_cost(1.9 * INSN_COST);
12662 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12663
12664 ins_encode %{
12665 __ sub(as_Register($dst$$reg),
12666 as_Register($src1$$reg),
12667 as_Register($src2$$reg),
12668 Assembler::LSL,
12669 $src3$$constant & 0x3f);
12670 %}
12671
12672 ins_pipe(ialu_reg_reg_shift);
12673 %}
12674
12675 // This pattern is automatically generated from aarch64_ad.m4
12676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12677
12678 // Shift Left followed by Shift Right.
12679 // This idiom is used by the compiler for the i2b bytecode etc.
12680 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12681 %{
12682 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12683 ins_cost(INSN_COST * 2);
12684 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12685 ins_encode %{
12686 int lshift = $lshift_count$$constant & 63;
12687 int rshift = $rshift_count$$constant & 63;
12688 int s = 63 - lshift;
12689 int r = (rshift - lshift) & 63;
12690 __ sbfm(as_Register($dst$$reg),
12691 as_Register($src$$reg),
12692 r, s);
12693 %}
12694
12695 ins_pipe(ialu_reg_shift);
12696 %}
12697
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700
12701 // Shift Left followed by Shift Right.
12702 // This idiom is used by the compiler for the i2b bytecode etc.
12703 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12704 %{
12705 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12706 ins_cost(INSN_COST * 2);
12707 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12708 ins_encode %{
12709 int lshift = $lshift_count$$constant & 31;
12710 int rshift = $rshift_count$$constant & 31;
12711 int s = 31 - lshift;
12712 int r = (rshift - lshift) & 31;
12713 __ sbfmw(as_Register($dst$$reg),
12714 as_Register($src$$reg),
12715 r, s);
12716 %}
12717
12718 ins_pipe(ialu_reg_shift);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723
12724 // Shift Left followed by Shift Right.
12725 // This idiom is used by the compiler for the i2b bytecode etc.
12726 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12727 %{
12728 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12729 ins_cost(INSN_COST * 2);
12730 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12731 ins_encode %{
12732 int lshift = $lshift_count$$constant & 63;
12733 int rshift = $rshift_count$$constant & 63;
12734 int s = 63 - lshift;
12735 int r = (rshift - lshift) & 63;
12736 __ ubfm(as_Register($dst$$reg),
12737 as_Register($src$$reg),
12738 r, s);
12739 %}
12740
12741 ins_pipe(ialu_reg_shift);
12742 %}
12743
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746
12747 // Shift Left followed by Shift Right.
12748 // This idiom is used by the compiler for the i2b bytecode etc.
12749 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12750 %{
12751 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12752 ins_cost(INSN_COST * 2);
12753 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12754 ins_encode %{
12755 int lshift = $lshift_count$$constant & 31;
12756 int rshift = $rshift_count$$constant & 31;
12757 int s = 31 - lshift;
12758 int r = (rshift - lshift) & 31;
12759 __ ubfmw(as_Register($dst$$reg),
12760 as_Register($src$$reg),
12761 r, s);
12762 %}
12763
12764 ins_pipe(ialu_reg_shift);
12765 %}
12766
12767 // Bitfield extract with shift & mask
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12772 %{
12773 match(Set dst (AndI (URShiftI src rshift) mask));
12774 // Make sure we are not going to exceed what ubfxw can do.
12775 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12776
12777 ins_cost(INSN_COST);
12778 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12779 ins_encode %{
12780 int rshift = $rshift$$constant & 31;
12781 intptr_t mask = $mask$$constant;
12782 int width = exact_log2(mask+1);
12783 __ ubfxw(as_Register($dst$$reg),
12784 as_Register($src$$reg), rshift, width);
12785 %}
12786 ins_pipe(ialu_reg_shift);
12787 %}
12788
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12792 %{
12793 match(Set dst (AndL (URShiftL src rshift) mask));
12794 // Make sure we are not going to exceed what ubfx can do.
12795 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12796
12797 ins_cost(INSN_COST);
12798 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12799 ins_encode %{
12800 int rshift = $rshift$$constant & 63;
12801 intptr_t mask = $mask$$constant;
12802 int width = exact_log2_long(mask+1);
12803 __ ubfx(as_Register($dst$$reg),
12804 as_Register($src$$reg), rshift, width);
12805 %}
12806 ins_pipe(ialu_reg_shift);
12807 %}
12808
12809
12810 // This pattern is automatically generated from aarch64_ad.m4
12811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12812
12813 // We can use ubfx when extending an And with a mask when we know mask
12814 // is positive. We know that because immI_bitmask guarantees it.
12815 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12816 %{
12817 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12818 // Make sure we are not going to exceed what ubfxw can do.
12819 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12820
12821 ins_cost(INSN_COST * 2);
12822 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12823 ins_encode %{
12824 int rshift = $rshift$$constant & 31;
12825 intptr_t mask = $mask$$constant;
12826 int width = exact_log2(mask+1);
12827 __ ubfx(as_Register($dst$$reg),
12828 as_Register($src$$reg), rshift, width);
12829 %}
12830 ins_pipe(ialu_reg_shift);
12831 %}
12832
12833
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836
12837 // We can use ubfiz when masking by a positive number and then left shifting the result.
12838 // We know that the mask is positive because immI_bitmask guarantees it.
12839 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12840 %{
12841 match(Set dst (LShiftI (AndI src mask) lshift));
12842 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12843
12844 ins_cost(INSN_COST);
12845 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12846 ins_encode %{
12847 int lshift = $lshift$$constant & 31;
12848 intptr_t mask = $mask$$constant;
12849 int width = exact_log2(mask+1);
12850 __ ubfizw(as_Register($dst$$reg),
12851 as_Register($src$$reg), lshift, width);
12852 %}
12853 ins_pipe(ialu_reg_shift);
12854 %}
12855
12856 // This pattern is automatically generated from aarch64_ad.m4
12857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12858
12859 // We can use ubfiz when masking by a positive number and then left shifting the result.
12860 // We know that the mask is positive because immL_bitmask guarantees it.
12861 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12862 %{
12863 match(Set dst (LShiftL (AndL src mask) lshift));
12864 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12865
12866 ins_cost(INSN_COST);
12867 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12868 ins_encode %{
12869 int lshift = $lshift$$constant & 63;
12870 intptr_t mask = $mask$$constant;
12871 int width = exact_log2_long(mask+1);
12872 __ ubfiz(as_Register($dst$$reg),
12873 as_Register($src$$reg), lshift, width);
12874 %}
12875 ins_pipe(ialu_reg_shift);
12876 %}
12877
12878 // This pattern is automatically generated from aarch64_ad.m4
12879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12880
12881 // We can use ubfiz when masking by a positive number and then left shifting the result.
12882 // We know that the mask is positive because immI_bitmask guarantees it.
12883 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12884 %{
12885 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12886 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12887
12888 ins_cost(INSN_COST);
12889 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12890 ins_encode %{
12891 int lshift = $lshift$$constant & 31;
12892 intptr_t mask = $mask$$constant;
12893 int width = exact_log2(mask+1);
12894 __ ubfizw(as_Register($dst$$reg),
12895 as_Register($src$$reg), lshift, width);
12896 %}
12897 ins_pipe(ialu_reg_shift);
12898 %}
12899
12900 // This pattern is automatically generated from aarch64_ad.m4
12901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12902
12903 // We can use ubfiz when masking by a positive number and then left shifting the result.
12904 // We know that the mask is positive because immL_bitmask guarantees it.
12905 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12906 %{
12907 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12908 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12909
12910 ins_cost(INSN_COST);
12911 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12912 ins_encode %{
12913 int lshift = $lshift$$constant & 63;
12914 intptr_t mask = $mask$$constant;
12915 int width = exact_log2_long(mask+1);
12916 __ ubfiz(as_Register($dst$$reg),
12917 as_Register($src$$reg), lshift, width);
12918 %}
12919 ins_pipe(ialu_reg_shift);
12920 %}
12921
12922
12923 // This pattern is automatically generated from aarch64_ad.m4
12924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12925
12926 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12927 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12928 %{
12929 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12930 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12931
12932 ins_cost(INSN_COST);
12933 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12934 ins_encode %{
12935 int lshift = $lshift$$constant & 63;
12936 intptr_t mask = $mask$$constant;
12937 int width = exact_log2(mask+1);
12938 __ ubfiz(as_Register($dst$$reg),
12939 as_Register($src$$reg), lshift, width);
12940 %}
12941 ins_pipe(ialu_reg_shift);
12942 %}
12943
12944 // This pattern is automatically generated from aarch64_ad.m4
12945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12946
12947 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12948 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12949 %{
12950 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12951 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12952
12953 ins_cost(INSN_COST);
12954 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12955 ins_encode %{
12956 int lshift = $lshift$$constant & 31;
12957 intptr_t mask = $mask$$constant;
12958 int width = exact_log2(mask+1);
12959 __ ubfiz(as_Register($dst$$reg),
12960 as_Register($src$$reg), lshift, width);
12961 %}
12962 ins_pipe(ialu_reg_shift);
12963 %}
12964
12965 // This pattern is automatically generated from aarch64_ad.m4
12966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12967
12968 // Can skip int2long conversions after AND with small bitmask
12969 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12970 %{
12971 match(Set dst (ConvI2L (AndI src msk)));
12972 ins_cost(INSN_COST);
12973 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12974 ins_encode %{
12975 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12976 %}
12977 ins_pipe(ialu_reg_shift);
12978 %}
12979
12980
12981 // Rotations
12982
12983 // This pattern is automatically generated from aarch64_ad.m4
12984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12985 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12986 %{
12987 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12988 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12989
12990 ins_cost(INSN_COST);
12991 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12992
12993 ins_encode %{
12994 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12995 $rshift$$constant & 63);
12996 %}
12997 ins_pipe(ialu_reg_reg_extr);
12998 %}
12999
13000
13001 // This pattern is automatically generated from aarch64_ad.m4
13002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13003 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13004 %{
13005 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13006 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13007
13008 ins_cost(INSN_COST);
13009 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13010
13011 ins_encode %{
13012 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13013 $rshift$$constant & 31);
13014 %}
13015 ins_pipe(ialu_reg_reg_extr);
13016 %}
13017
13018
13019 // This pattern is automatically generated from aarch64_ad.m4
13020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13021 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13022 %{
13023 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13024 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13025
13026 ins_cost(INSN_COST);
13027 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13028
13029 ins_encode %{
13030 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13031 $rshift$$constant & 63);
13032 %}
13033 ins_pipe(ialu_reg_reg_extr);
13034 %}
13035
13036
13037 // This pattern is automatically generated from aarch64_ad.m4
13038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13039 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13040 %{
13041 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13042 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13043
13044 ins_cost(INSN_COST);
13045 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13046
13047 ins_encode %{
13048 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13049 $rshift$$constant & 31);
13050 %}
13051 ins_pipe(ialu_reg_reg_extr);
13052 %}
13053
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13057 %{
13058 match(Set dst (RotateRight src shift));
13059
13060 ins_cost(INSN_COST);
13061 format %{ "ror $dst, $src, $shift" %}
13062
13063 ins_encode %{
13064 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13065 $shift$$constant & 0x1f);
13066 %}
13067 ins_pipe(ialu_reg_reg_vshift);
13068 %}
13069
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13073 %{
13074 match(Set dst (RotateRight src shift));
13075
13076 ins_cost(INSN_COST);
13077 format %{ "ror $dst, $src, $shift" %}
13078
13079 ins_encode %{
13080 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13081 $shift$$constant & 0x3f);
13082 %}
13083 ins_pipe(ialu_reg_reg_vshift);
13084 %}
13085
13086 // This pattern is automatically generated from aarch64_ad.m4
13087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13088 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13089 %{
13090 match(Set dst (RotateRight src shift));
13091
13092 ins_cost(INSN_COST);
13093 format %{ "ror $dst, $src, $shift" %}
13094
13095 ins_encode %{
13096 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13097 %}
13098 ins_pipe(ialu_reg_reg_vshift);
13099 %}
13100
13101 // This pattern is automatically generated from aarch64_ad.m4
13102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13103 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13104 %{
13105 match(Set dst (RotateRight src shift));
13106
13107 ins_cost(INSN_COST);
13108 format %{ "ror $dst, $src, $shift" %}
13109
13110 ins_encode %{
13111 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13112 %}
13113 ins_pipe(ialu_reg_reg_vshift);
13114 %}
13115
13116 // This pattern is automatically generated from aarch64_ad.m4
13117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13118 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13119 %{
13120 match(Set dst (RotateLeft src shift));
13121
13122 ins_cost(INSN_COST);
13123 format %{ "rol $dst, $src, $shift" %}
13124
13125 ins_encode %{
13126 __ subw(rscratch1, zr, as_Register($shift$$reg));
13127 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13128 %}
13129 ins_pipe(ialu_reg_reg_vshift);
13130 %}
13131
13132 // This pattern is automatically generated from aarch64_ad.m4
13133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13134 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13135 %{
13136 match(Set dst (RotateLeft src shift));
13137
13138 ins_cost(INSN_COST);
13139 format %{ "rol $dst, $src, $shift" %}
13140
13141 ins_encode %{
13142 __ subw(rscratch1, zr, as_Register($shift$$reg));
13143 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13144 %}
13145 ins_pipe(ialu_reg_reg_vshift);
13146 %}
13147
13148
13149 // Add/subtract (extended)
13150
13151 // This pattern is automatically generated from aarch64_ad.m4
13152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13153 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13154 %{
13155 match(Set dst (AddL src1 (ConvI2L src2)));
13156 ins_cost(INSN_COST);
13157 format %{ "add $dst, $src1, $src2, sxtw" %}
13158
13159 ins_encode %{
13160 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13161 as_Register($src2$$reg), ext::sxtw);
13162 %}
13163 ins_pipe(ialu_reg_reg);
13164 %}
13165
13166 // This pattern is automatically generated from aarch64_ad.m4
13167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13168 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13169 %{
13170 match(Set dst (SubL src1 (ConvI2L src2)));
13171 ins_cost(INSN_COST);
13172 format %{ "sub $dst, $src1, $src2, sxtw" %}
13173
13174 ins_encode %{
13175 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13176 as_Register($src2$$reg), ext::sxtw);
13177 %}
13178 ins_pipe(ialu_reg_reg);
13179 %}
13180
13181 // This pattern is automatically generated from aarch64_ad.m4
13182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13183 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13184 %{
13185 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13186 ins_cost(INSN_COST);
13187 format %{ "add $dst, $src1, $src2, sxth" %}
13188
13189 ins_encode %{
13190 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13191 as_Register($src2$$reg), ext::sxth);
13192 %}
13193 ins_pipe(ialu_reg_reg);
13194 %}
13195
13196 // This pattern is automatically generated from aarch64_ad.m4
13197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13198 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13199 %{
13200 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13201 ins_cost(INSN_COST);
13202 format %{ "add $dst, $src1, $src2, sxtb" %}
13203
13204 ins_encode %{
13205 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13206 as_Register($src2$$reg), ext::sxtb);
13207 %}
13208 ins_pipe(ialu_reg_reg);
13209 %}
13210
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13214 %{
13215 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13216 ins_cost(INSN_COST);
13217 format %{ "add $dst, $src1, $src2, uxtb" %}
13218
13219 ins_encode %{
13220 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13221 as_Register($src2$$reg), ext::uxtb);
13222 %}
13223 ins_pipe(ialu_reg_reg);
13224 %}
13225
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13229 %{
13230 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13231 ins_cost(INSN_COST);
13232 format %{ "add $dst, $src1, $src2, sxth" %}
13233
13234 ins_encode %{
13235 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13236 as_Register($src2$$reg), ext::sxth);
13237 %}
13238 ins_pipe(ialu_reg_reg);
13239 %}
13240
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13244 %{
13245 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13246 ins_cost(INSN_COST);
13247 format %{ "add $dst, $src1, $src2, sxtw" %}
13248
13249 ins_encode %{
13250 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13251 as_Register($src2$$reg), ext::sxtw);
13252 %}
13253 ins_pipe(ialu_reg_reg);
13254 %}
13255
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13259 %{
13260 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13261 ins_cost(INSN_COST);
13262 format %{ "add $dst, $src1, $src2, sxtb" %}
13263
13264 ins_encode %{
13265 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13266 as_Register($src2$$reg), ext::sxtb);
13267 %}
13268 ins_pipe(ialu_reg_reg);
13269 %}
13270
13271 // This pattern is automatically generated from aarch64_ad.m4
13272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13273 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13274 %{
13275 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13276 ins_cost(INSN_COST);
13277 format %{ "add $dst, $src1, $src2, uxtb" %}
13278
13279 ins_encode %{
13280 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13281 as_Register($src2$$reg), ext::uxtb);
13282 %}
13283 ins_pipe(ialu_reg_reg);
13284 %}
13285
13286 // This pattern is automatically generated from aarch64_ad.m4
13287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13288 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13289 %{
13290 match(Set dst (AddI src1 (AndI src2 mask)));
13291 ins_cost(INSN_COST);
13292 format %{ "addw $dst, $src1, $src2, uxtb" %}
13293
13294 ins_encode %{
13295 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13296 as_Register($src2$$reg), ext::uxtb);
13297 %}
13298 ins_pipe(ialu_reg_reg);
13299 %}
13300
13301 // This pattern is automatically generated from aarch64_ad.m4
13302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13303 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13304 %{
13305 match(Set dst (AddI src1 (AndI src2 mask)));
13306 ins_cost(INSN_COST);
13307 format %{ "addw $dst, $src1, $src2, uxth" %}
13308
13309 ins_encode %{
13310 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13311 as_Register($src2$$reg), ext::uxth);
13312 %}
13313 ins_pipe(ialu_reg_reg);
13314 %}
13315
13316 // This pattern is automatically generated from aarch64_ad.m4
13317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13318 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13319 %{
13320 match(Set dst (AddL src1 (AndL src2 mask)));
13321 ins_cost(INSN_COST);
13322 format %{ "add $dst, $src1, $src2, uxtb" %}
13323
13324 ins_encode %{
13325 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13326 as_Register($src2$$reg), ext::uxtb);
13327 %}
13328 ins_pipe(ialu_reg_reg);
13329 %}
13330
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13334 %{
13335 match(Set dst (AddL src1 (AndL src2 mask)));
13336 ins_cost(INSN_COST);
13337 format %{ "add $dst, $src1, $src2, uxth" %}
13338
13339 ins_encode %{
13340 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13341 as_Register($src2$$reg), ext::uxth);
13342 %}
13343 ins_pipe(ialu_reg_reg);
13344 %}
13345
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13349 %{
13350 match(Set dst (AddL src1 (AndL src2 mask)));
13351 ins_cost(INSN_COST);
13352 format %{ "add $dst, $src1, $src2, uxtw" %}
13353
13354 ins_encode %{
13355 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13356 as_Register($src2$$reg), ext::uxtw);
13357 %}
13358 ins_pipe(ialu_reg_reg);
13359 %}
13360
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13364 %{
13365 match(Set dst (SubI src1 (AndI src2 mask)));
13366 ins_cost(INSN_COST);
13367 format %{ "subw $dst, $src1, $src2, uxtb" %}
13368
13369 ins_encode %{
13370 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13371 as_Register($src2$$reg), ext::uxtb);
13372 %}
13373 ins_pipe(ialu_reg_reg);
13374 %}
13375
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13379 %{
13380 match(Set dst (SubI src1 (AndI src2 mask)));
13381 ins_cost(INSN_COST);
13382 format %{ "subw $dst, $src1, $src2, uxth" %}
13383
13384 ins_encode %{
13385 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13386 as_Register($src2$$reg), ext::uxth);
13387 %}
13388 ins_pipe(ialu_reg_reg);
13389 %}
13390
13391 // This pattern is automatically generated from aarch64_ad.m4
13392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13393 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13394 %{
13395 match(Set dst (SubL src1 (AndL src2 mask)));
13396 ins_cost(INSN_COST);
13397 format %{ "sub $dst, $src1, $src2, uxtb" %}
13398
13399 ins_encode %{
13400 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13401 as_Register($src2$$reg), ext::uxtb);
13402 %}
13403 ins_pipe(ialu_reg_reg);
13404 %}
13405
13406 // This pattern is automatically generated from aarch64_ad.m4
13407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13408 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13409 %{
13410 match(Set dst (SubL src1 (AndL src2 mask)));
13411 ins_cost(INSN_COST);
13412 format %{ "sub $dst, $src1, $src2, uxth" %}
13413
13414 ins_encode %{
13415 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13416 as_Register($src2$$reg), ext::uxth);
13417 %}
13418 ins_pipe(ialu_reg_reg);
13419 %}
13420
13421 // This pattern is automatically generated from aarch64_ad.m4
13422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13423 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13424 %{
13425 match(Set dst (SubL src1 (AndL src2 mask)));
13426 ins_cost(INSN_COST);
13427 format %{ "sub $dst, $src1, $src2, uxtw" %}
13428
13429 ins_encode %{
13430 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13431 as_Register($src2$$reg), ext::uxtw);
13432 %}
13433 ins_pipe(ialu_reg_reg);
13434 %}
13435
13436
13437 // This pattern is automatically generated from aarch64_ad.m4
13438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13439 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13440 %{
13441 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13442 ins_cost(1.9 * INSN_COST);
13443 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13444
13445 ins_encode %{
13446 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13447 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13448 %}
13449 ins_pipe(ialu_reg_reg_shift);
13450 %}
13451
13452 // This pattern is automatically generated from aarch64_ad.m4
13453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13454 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13455 %{
13456 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13457 ins_cost(1.9 * INSN_COST);
13458 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13459
13460 ins_encode %{
13461 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13462 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13463 %}
13464 ins_pipe(ialu_reg_reg_shift);
13465 %}
13466
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13470 %{
13471 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13472 ins_cost(1.9 * INSN_COST);
13473 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13474
13475 ins_encode %{
13476 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13477 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13478 %}
13479 ins_pipe(ialu_reg_reg_shift);
13480 %}
13481
13482 // This pattern is automatically generated from aarch64_ad.m4
13483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13484 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13485 %{
13486 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13487 ins_cost(1.9 * INSN_COST);
13488 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13489
13490 ins_encode %{
13491 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13492 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13493 %}
13494 ins_pipe(ialu_reg_reg_shift);
13495 %}
13496
13497 // This pattern is automatically generated from aarch64_ad.m4
13498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13499 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13500 %{
13501 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13502 ins_cost(1.9 * INSN_COST);
13503 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13504
13505 ins_encode %{
13506 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13507 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13508 %}
13509 ins_pipe(ialu_reg_reg_shift);
13510 %}
13511
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13515 %{
13516 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13517 ins_cost(1.9 * INSN_COST);
13518 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13519
13520 ins_encode %{
13521 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13522 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13523 %}
13524 ins_pipe(ialu_reg_reg_shift);
13525 %}
13526
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13530 %{
13531 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13532 ins_cost(1.9 * INSN_COST);
13533 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13534
13535 ins_encode %{
13536 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13537 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13538 %}
13539 ins_pipe(ialu_reg_reg_shift);
13540 %}
13541
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13545 %{
13546 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13547 ins_cost(1.9 * INSN_COST);
13548 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13549
13550 ins_encode %{
13551 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13552 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13553 %}
13554 ins_pipe(ialu_reg_reg_shift);
13555 %}
13556
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13560 %{
13561 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13562 ins_cost(1.9 * INSN_COST);
13563 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13564
13565 ins_encode %{
13566 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13567 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13568 %}
13569 ins_pipe(ialu_reg_reg_shift);
13570 %}
13571
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13575 %{
13576 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13577 ins_cost(1.9 * INSN_COST);
13578 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13579
13580 ins_encode %{
13581 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13582 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13583 %}
13584 ins_pipe(ialu_reg_reg_shift);
13585 %}
13586
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13590 %{
13591 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13592 ins_cost(1.9 * INSN_COST);
13593 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13594
13595 ins_encode %{
13596 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13597 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13598 %}
13599 ins_pipe(ialu_reg_reg_shift);
13600 %}
13601
13602 // This pattern is automatically generated from aarch64_ad.m4
13603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13604 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13605 %{
13606 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13607 ins_cost(1.9 * INSN_COST);
13608 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13609
13610 ins_encode %{
13611 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13612 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13613 %}
13614 ins_pipe(ialu_reg_reg_shift);
13615 %}
13616
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13620 %{
13621 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13622 ins_cost(1.9 * INSN_COST);
13623 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13624
13625 ins_encode %{
13626 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13627 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13628 %}
13629 ins_pipe(ialu_reg_reg_shift);
13630 %}
13631
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13635 %{
13636 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13637 ins_cost(1.9 * INSN_COST);
13638 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13639
13640 ins_encode %{
13641 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13642 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13643 %}
13644 ins_pipe(ialu_reg_reg_shift);
13645 %}
13646
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13650 %{
13651 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13652 ins_cost(1.9 * INSN_COST);
13653 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13654
13655 ins_encode %{
13656 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13657 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13658 %}
13659 ins_pipe(ialu_reg_reg_shift);
13660 %}
13661
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13665 %{
13666 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13667 ins_cost(1.9 * INSN_COST);
13668 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13669
13670 ins_encode %{
13671 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13672 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13673 %}
13674 ins_pipe(ialu_reg_reg_shift);
13675 %}
13676
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13680 %{
13681 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13682 ins_cost(1.9 * INSN_COST);
13683 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13684
13685 ins_encode %{
13686 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13687 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13688 %}
13689 ins_pipe(ialu_reg_reg_shift);
13690 %}
13691
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13695 %{
13696 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13697 ins_cost(1.9 * INSN_COST);
13698 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13699
13700 ins_encode %{
13701 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13702 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13703 %}
13704 ins_pipe(ialu_reg_reg_shift);
13705 %}
13706
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13710 %{
13711 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13712 ins_cost(1.9 * INSN_COST);
13713 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13714
13715 ins_encode %{
13716 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13717 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13718 %}
13719 ins_pipe(ialu_reg_reg_shift);
13720 %}
13721
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13725 %{
13726 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13727 ins_cost(1.9 * INSN_COST);
13728 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13729
13730 ins_encode %{
13731 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13732 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13733 %}
13734 ins_pipe(ialu_reg_reg_shift);
13735 %}
13736
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13740 %{
13741 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13742 ins_cost(1.9 * INSN_COST);
13743 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13744
13745 ins_encode %{
13746 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13747 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13748 %}
13749 ins_pipe(ialu_reg_reg_shift);
13750 %}
13751
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13755 %{
13756 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13757 ins_cost(1.9 * INSN_COST);
13758 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13759
13760 ins_encode %{
13761 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13762 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13763 %}
13764 ins_pipe(ialu_reg_reg_shift);
13765 %}
13766
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13770 %{
13771 effect(DEF dst, USE src1, USE src2, USE cr);
13772 ins_cost(INSN_COST * 2);
13773 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13774
13775 ins_encode %{
13776 __ cselw($dst$$Register,
13777 $src1$$Register,
13778 $src2$$Register,
13779 Assembler::LT);
13780 %}
13781 ins_pipe(icond_reg_reg);
13782 %}
13783
13784 // This pattern is automatically generated from aarch64_ad.m4
13785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13786 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13787 %{
13788 effect(DEF dst, USE src1, USE src2, USE cr);
13789 ins_cost(INSN_COST * 2);
13790 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13791
13792 ins_encode %{
13793 __ cselw($dst$$Register,
13794 $src1$$Register,
13795 $src2$$Register,
13796 Assembler::GT);
13797 %}
13798 ins_pipe(icond_reg_reg);
13799 %}
13800
13801 // This pattern is automatically generated from aarch64_ad.m4
13802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13803 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13804 %{
13805 effect(DEF dst, USE src1, USE cr);
13806 ins_cost(INSN_COST * 2);
13807 format %{ "cselw $dst, $src1, zr lt\t" %}
13808
13809 ins_encode %{
13810 __ cselw($dst$$Register,
13811 $src1$$Register,
13812 zr,
13813 Assembler::LT);
13814 %}
13815 ins_pipe(icond_reg);
13816 %}
13817
13818 // This pattern is automatically generated from aarch64_ad.m4
13819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13820 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13821 %{
13822 effect(DEF dst, USE src1, USE cr);
13823 ins_cost(INSN_COST * 2);
13824 format %{ "cselw $dst, $src1, zr gt\t" %}
13825
13826 ins_encode %{
13827 __ cselw($dst$$Register,
13828 $src1$$Register,
13829 zr,
13830 Assembler::GT);
13831 %}
13832 ins_pipe(icond_reg);
13833 %}
13834
13835 // This pattern is automatically generated from aarch64_ad.m4
13836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13837 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13838 %{
13839 effect(DEF dst, USE src1, USE cr);
13840 ins_cost(INSN_COST * 2);
13841 format %{ "csincw $dst, $src1, zr le\t" %}
13842
13843 ins_encode %{
13844 __ csincw($dst$$Register,
13845 $src1$$Register,
13846 zr,
13847 Assembler::LE);
13848 %}
13849 ins_pipe(icond_reg);
13850 %}
13851
13852 // This pattern is automatically generated from aarch64_ad.m4
13853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13854 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13855 %{
13856 effect(DEF dst, USE src1, USE cr);
13857 ins_cost(INSN_COST * 2);
13858 format %{ "csincw $dst, $src1, zr gt\t" %}
13859
13860 ins_encode %{
13861 __ csincw($dst$$Register,
13862 $src1$$Register,
13863 zr,
13864 Assembler::GT);
13865 %}
13866 ins_pipe(icond_reg);
13867 %}
13868
13869 // This pattern is automatically generated from aarch64_ad.m4
13870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13871 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13872 %{
13873 effect(DEF dst, USE src1, USE cr);
13874 ins_cost(INSN_COST * 2);
13875 format %{ "csinvw $dst, $src1, zr lt\t" %}
13876
13877 ins_encode %{
13878 __ csinvw($dst$$Register,
13879 $src1$$Register,
13880 zr,
13881 Assembler::LT);
13882 %}
13883 ins_pipe(icond_reg);
13884 %}
13885
13886 // This pattern is automatically generated from aarch64_ad.m4
13887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13888 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13889 %{
13890 effect(DEF dst, USE src1, USE cr);
13891 ins_cost(INSN_COST * 2);
13892 format %{ "csinvw $dst, $src1, zr ge\t" %}
13893
13894 ins_encode %{
13895 __ csinvw($dst$$Register,
13896 $src1$$Register,
13897 zr,
13898 Assembler::GE);
13899 %}
13900 ins_pipe(icond_reg);
13901 %}
13902
13903 // This pattern is automatically generated from aarch64_ad.m4
13904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13905 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13906 %{
13907 match(Set dst (MinI src imm));
13908 ins_cost(INSN_COST * 3);
13909 expand %{
13910 rFlagsReg cr;
13911 compI_reg_imm0(cr, src);
13912 cmovI_reg_imm0_lt(dst, src, cr);
13913 %}
13914 %}
13915
13916 // This pattern is automatically generated from aarch64_ad.m4
13917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13918 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13919 %{
13920 match(Set dst (MinI imm src));
13921 ins_cost(INSN_COST * 3);
13922 expand %{
13923 rFlagsReg cr;
13924 compI_reg_imm0(cr, src);
13925 cmovI_reg_imm0_lt(dst, src, cr);
13926 %}
13927 %}
13928
13929 // This pattern is automatically generated from aarch64_ad.m4
13930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13931 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13932 %{
13933 match(Set dst (MinI src imm));
13934 ins_cost(INSN_COST * 3);
13935 expand %{
13936 rFlagsReg cr;
13937 compI_reg_imm0(cr, src);
13938 cmovI_reg_imm1_le(dst, src, cr);
13939 %}
13940 %}
13941
13942 // This pattern is automatically generated from aarch64_ad.m4
13943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13944 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13945 %{
13946 match(Set dst (MinI imm src));
13947 ins_cost(INSN_COST * 3);
13948 expand %{
13949 rFlagsReg cr;
13950 compI_reg_imm0(cr, src);
13951 cmovI_reg_imm1_le(dst, src, cr);
13952 %}
13953 %}
13954
13955 // This pattern is automatically generated from aarch64_ad.m4
13956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13957 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13958 %{
13959 match(Set dst (MinI src imm));
13960 ins_cost(INSN_COST * 3);
13961 expand %{
13962 rFlagsReg cr;
13963 compI_reg_imm0(cr, src);
13964 cmovI_reg_immM1_lt(dst, src, cr);
13965 %}
13966 %}
13967
13968 // This pattern is automatically generated from aarch64_ad.m4
13969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13970 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13971 %{
13972 match(Set dst (MinI imm src));
13973 ins_cost(INSN_COST * 3);
13974 expand %{
13975 rFlagsReg cr;
13976 compI_reg_imm0(cr, src);
13977 cmovI_reg_immM1_lt(dst, src, cr);
13978 %}
13979 %}
13980
13981 // This pattern is automatically generated from aarch64_ad.m4
13982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13983 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13984 %{
13985 match(Set dst (MaxI src imm));
13986 ins_cost(INSN_COST * 3);
13987 expand %{
13988 rFlagsReg cr;
13989 compI_reg_imm0(cr, src);
13990 cmovI_reg_imm0_gt(dst, src, cr);
13991 %}
13992 %}
13993
13994 // This pattern is automatically generated from aarch64_ad.m4
13995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13996 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13997 %{
13998 match(Set dst (MaxI imm src));
13999 ins_cost(INSN_COST * 3);
14000 expand %{
14001 rFlagsReg cr;
14002 compI_reg_imm0(cr, src);
14003 cmovI_reg_imm0_gt(dst, src, cr);
14004 %}
14005 %}
14006
14007 // This pattern is automatically generated from aarch64_ad.m4
14008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14009 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14010 %{
14011 match(Set dst (MaxI src imm));
14012 ins_cost(INSN_COST * 3);
14013 expand %{
14014 rFlagsReg cr;
14015 compI_reg_imm0(cr, src);
14016 cmovI_reg_imm1_gt(dst, src, cr);
14017 %}
14018 %}
14019
14020 // This pattern is automatically generated from aarch64_ad.m4
14021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14022 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14023 %{
14024 match(Set dst (MaxI imm src));
14025 ins_cost(INSN_COST * 3);
14026 expand %{
14027 rFlagsReg cr;
14028 compI_reg_imm0(cr, src);
14029 cmovI_reg_imm1_gt(dst, src, cr);
14030 %}
14031 %}
14032
14033 // This pattern is automatically generated from aarch64_ad.m4
14034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14035 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14036 %{
14037 match(Set dst (MaxI src imm));
14038 ins_cost(INSN_COST * 3);
14039 expand %{
14040 rFlagsReg cr;
14041 compI_reg_imm0(cr, src);
14042 cmovI_reg_immM1_ge(dst, src, cr);
14043 %}
14044 %}
14045
14046 // This pattern is automatically generated from aarch64_ad.m4
14047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14048 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14049 %{
14050 match(Set dst (MaxI imm src));
14051 ins_cost(INSN_COST * 3);
14052 expand %{
14053 rFlagsReg cr;
14054 compI_reg_imm0(cr, src);
14055 cmovI_reg_immM1_ge(dst, src, cr);
14056 %}
14057 %}
14058
14059
14060
14061 // END This section of the file is automatically generated. Do not edit --------------
14062
14063
14064 // ============================================================================
14065 // Floating Point Arithmetic Instructions
14066
14067 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14068 match(Set dst (AddF src1 src2));
14069
14070 ins_cost(INSN_COST * 5);
14071 format %{ "fadds $dst, $src1, $src2" %}
14072
14073 ins_encode %{
14074 __ fadds(as_FloatRegister($dst$$reg),
14075 as_FloatRegister($src1$$reg),
14076 as_FloatRegister($src2$$reg));
14077 %}
14078
14079 ins_pipe(fp_dop_reg_reg_s);
14080 %}
14081
14082 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14083 match(Set dst (AddD src1 src2));
14084
14085 ins_cost(INSN_COST * 5);
14086 format %{ "faddd $dst, $src1, $src2" %}
14087
14088 ins_encode %{
14089 __ faddd(as_FloatRegister($dst$$reg),
14090 as_FloatRegister($src1$$reg),
14091 as_FloatRegister($src2$$reg));
14092 %}
14093
14094 ins_pipe(fp_dop_reg_reg_d);
14095 %}
14096
14097 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14098 match(Set dst (SubF src1 src2));
14099
14100 ins_cost(INSN_COST * 5);
14101 format %{ "fsubs $dst, $src1, $src2" %}
14102
14103 ins_encode %{
14104 __ fsubs(as_FloatRegister($dst$$reg),
14105 as_FloatRegister($src1$$reg),
14106 as_FloatRegister($src2$$reg));
14107 %}
14108
14109 ins_pipe(fp_dop_reg_reg_s);
14110 %}
14111
14112 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14113 match(Set dst (SubD src1 src2));
14114
14115 ins_cost(INSN_COST * 5);
14116 format %{ "fsubd $dst, $src1, $src2" %}
14117
14118 ins_encode %{
14119 __ fsubd(as_FloatRegister($dst$$reg),
14120 as_FloatRegister($src1$$reg),
14121 as_FloatRegister($src2$$reg));
14122 %}
14123
14124 ins_pipe(fp_dop_reg_reg_d);
14125 %}
14126
14127 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14128 match(Set dst (MulF src1 src2));
14129
14130 ins_cost(INSN_COST * 6);
14131 format %{ "fmuls $dst, $src1, $src2" %}
14132
14133 ins_encode %{
14134 __ fmuls(as_FloatRegister($dst$$reg),
14135 as_FloatRegister($src1$$reg),
14136 as_FloatRegister($src2$$reg));
14137 %}
14138
14139 ins_pipe(fp_dop_reg_reg_s);
14140 %}
14141
14142 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14143 match(Set dst (MulD src1 src2));
14144
14145 ins_cost(INSN_COST * 6);
14146 format %{ "fmuld $dst, $src1, $src2" %}
14147
14148 ins_encode %{
14149 __ fmuld(as_FloatRegister($dst$$reg),
14150 as_FloatRegister($src1$$reg),
14151 as_FloatRegister($src2$$reg));
14152 %}
14153
14154 ins_pipe(fp_dop_reg_reg_d);
14155 %}
14156
14157 // src1 * src2 + src3
14158 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14159 predicate(UseFMA);
14160 match(Set dst (FmaF src3 (Binary src1 src2)));
14161
14162 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14163
14164 ins_encode %{
14165 __ fmadds(as_FloatRegister($dst$$reg),
14166 as_FloatRegister($src1$$reg),
14167 as_FloatRegister($src2$$reg),
14168 as_FloatRegister($src3$$reg));
14169 %}
14170
14171 ins_pipe(pipe_class_default);
14172 %}
14173
14174 // src1 * src2 + src3
14175 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14176 predicate(UseFMA);
14177 match(Set dst (FmaD src3 (Binary src1 src2)));
14178
14179 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14180
14181 ins_encode %{
14182 __ fmaddd(as_FloatRegister($dst$$reg),
14183 as_FloatRegister($src1$$reg),
14184 as_FloatRegister($src2$$reg),
14185 as_FloatRegister($src3$$reg));
14186 %}
14187
14188 ins_pipe(pipe_class_default);
14189 %}
14190
14191 // -src1 * src2 + src3
14192 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14193 predicate(UseFMA);
14194 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14195 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14196
14197 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14198
14199 ins_encode %{
14200 __ fmsubs(as_FloatRegister($dst$$reg),
14201 as_FloatRegister($src1$$reg),
14202 as_FloatRegister($src2$$reg),
14203 as_FloatRegister($src3$$reg));
14204 %}
14205
14206 ins_pipe(pipe_class_default);
14207 %}
14208
14209 // -src1 * src2 + src3
14210 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14211 predicate(UseFMA);
14212 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14213 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14214
14215 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14216
14217 ins_encode %{
14218 __ fmsubd(as_FloatRegister($dst$$reg),
14219 as_FloatRegister($src1$$reg),
14220 as_FloatRegister($src2$$reg),
14221 as_FloatRegister($src3$$reg));
14222 %}
14223
14224 ins_pipe(pipe_class_default);
14225 %}
14226
14227 // -src1 * src2 - src3
14228 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14229 predicate(UseFMA);
14230 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14231 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14232
14233 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14234
14235 ins_encode %{
14236 __ fnmadds(as_FloatRegister($dst$$reg),
14237 as_FloatRegister($src1$$reg),
14238 as_FloatRegister($src2$$reg),
14239 as_FloatRegister($src3$$reg));
14240 %}
14241
14242 ins_pipe(pipe_class_default);
14243 %}
14244
14245 // -src1 * src2 - src3
14246 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14247 predicate(UseFMA);
14248 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14249 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14250
14251 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14252
14253 ins_encode %{
14254 __ fnmaddd(as_FloatRegister($dst$$reg),
14255 as_FloatRegister($src1$$reg),
14256 as_FloatRegister($src2$$reg),
14257 as_FloatRegister($src3$$reg));
14258 %}
14259
14260 ins_pipe(pipe_class_default);
14261 %}
14262
14263 // src1 * src2 - src3
14264 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14265 predicate(UseFMA);
14266 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14267
14268 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14269
14270 ins_encode %{
14271 __ fnmsubs(as_FloatRegister($dst$$reg),
14272 as_FloatRegister($src1$$reg),
14273 as_FloatRegister($src2$$reg),
14274 as_FloatRegister($src3$$reg));
14275 %}
14276
14277 ins_pipe(pipe_class_default);
14278 %}
14279
14280 // src1 * src2 - src3
14281 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14282 predicate(UseFMA);
14283 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14284
14285 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14286
14287 ins_encode %{
14288 // n.b. insn name should be fnmsubd
14289 __ fnmsub(as_FloatRegister($dst$$reg),
14290 as_FloatRegister($src1$$reg),
14291 as_FloatRegister($src2$$reg),
14292 as_FloatRegister($src3$$reg));
14293 %}
14294
14295 ins_pipe(pipe_class_default);
14296 %}
14297
14298
14299 // Math.max(FF)F
14300 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14301 match(Set dst (MaxF src1 src2));
14302
14303 format %{ "fmaxs $dst, $src1, $src2" %}
14304 ins_encode %{
14305 __ fmaxs(as_FloatRegister($dst$$reg),
14306 as_FloatRegister($src1$$reg),
14307 as_FloatRegister($src2$$reg));
14308 %}
14309
14310 ins_pipe(fp_dop_reg_reg_s);
14311 %}
14312
14313 // Math.min(FF)F
14314 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14315 match(Set dst (MinF src1 src2));
14316
14317 format %{ "fmins $dst, $src1, $src2" %}
14318 ins_encode %{
14319 __ fmins(as_FloatRegister($dst$$reg),
14320 as_FloatRegister($src1$$reg),
14321 as_FloatRegister($src2$$reg));
14322 %}
14323
14324 ins_pipe(fp_dop_reg_reg_s);
14325 %}
14326
14327 // Math.max(DD)D
14328 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14329 match(Set dst (MaxD src1 src2));
14330
14331 format %{ "fmaxd $dst, $src1, $src2" %}
14332 ins_encode %{
14333 __ fmaxd(as_FloatRegister($dst$$reg),
14334 as_FloatRegister($src1$$reg),
14335 as_FloatRegister($src2$$reg));
14336 %}
14337
14338 ins_pipe(fp_dop_reg_reg_d);
14339 %}
14340
14341 // Math.min(DD)D
14342 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14343 match(Set dst (MinD src1 src2));
14344
14345 format %{ "fmind $dst, $src1, $src2" %}
14346 ins_encode %{
14347 __ fmind(as_FloatRegister($dst$$reg),
14348 as_FloatRegister($src1$$reg),
14349 as_FloatRegister($src2$$reg));
14350 %}
14351
14352 ins_pipe(fp_dop_reg_reg_d);
14353 %}
14354
14355
14356 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14357 match(Set dst (DivF src1 src2));
14358
14359 ins_cost(INSN_COST * 18);
14360 format %{ "fdivs $dst, $src1, $src2" %}
14361
14362 ins_encode %{
14363 __ fdivs(as_FloatRegister($dst$$reg),
14364 as_FloatRegister($src1$$reg),
14365 as_FloatRegister($src2$$reg));
14366 %}
14367
14368 ins_pipe(fp_div_s);
14369 %}
14370
14371 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14372 match(Set dst (DivD src1 src2));
14373
14374 ins_cost(INSN_COST * 32);
14375 format %{ "fdivd $dst, $src1, $src2" %}
14376
14377 ins_encode %{
14378 __ fdivd(as_FloatRegister($dst$$reg),
14379 as_FloatRegister($src1$$reg),
14380 as_FloatRegister($src2$$reg));
14381 %}
14382
14383 ins_pipe(fp_div_d);
14384 %}
14385
14386 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14387 match(Set dst (NegF src));
14388
14389 ins_cost(INSN_COST * 3);
14390 format %{ "fneg $dst, $src" %}
14391
14392 ins_encode %{
14393 __ fnegs(as_FloatRegister($dst$$reg),
14394 as_FloatRegister($src$$reg));
14395 %}
14396
14397 ins_pipe(fp_uop_s);
14398 %}
14399
14400 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14401 match(Set dst (NegD src));
14402
14403 ins_cost(INSN_COST * 3);
14404 format %{ "fnegd $dst, $src" %}
14405
14406 ins_encode %{
14407 __ fnegd(as_FloatRegister($dst$$reg),
14408 as_FloatRegister($src$$reg));
14409 %}
14410
14411 ins_pipe(fp_uop_d);
14412 %}
14413
14414 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14415 %{
14416 match(Set dst (AbsI src));
14417
14418 effect(KILL cr);
14419 ins_cost(INSN_COST * 2);
14420 format %{ "cmpw $src, zr\n\t"
14421 "cnegw $dst, $src, Assembler::LT\t# int abs"
14422 %}
14423
14424 ins_encode %{
14425 __ cmpw(as_Register($src$$reg), zr);
14426 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14427 %}
14428 ins_pipe(pipe_class_default);
14429 %}
14430
14431 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14432 %{
14433 match(Set dst (AbsL src));
14434
14435 effect(KILL cr);
14436 ins_cost(INSN_COST * 2);
14437 format %{ "cmp $src, zr\n\t"
14438 "cneg $dst, $src, Assembler::LT\t# long abs"
14439 %}
14440
14441 ins_encode %{
14442 __ cmp(as_Register($src$$reg), zr);
14443 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14444 %}
14445 ins_pipe(pipe_class_default);
14446 %}
14447
14448 instruct absF_reg(vRegF dst, vRegF src) %{
14449 match(Set dst (AbsF src));
14450
14451 ins_cost(INSN_COST * 3);
14452 format %{ "fabss $dst, $src" %}
14453 ins_encode %{
14454 __ fabss(as_FloatRegister($dst$$reg),
14455 as_FloatRegister($src$$reg));
14456 %}
14457
14458 ins_pipe(fp_uop_s);
14459 %}
14460
14461 instruct absD_reg(vRegD dst, vRegD src) %{
14462 match(Set dst (AbsD src));
14463
14464 ins_cost(INSN_COST * 3);
14465 format %{ "fabsd $dst, $src" %}
14466 ins_encode %{
14467 __ fabsd(as_FloatRegister($dst$$reg),
14468 as_FloatRegister($src$$reg));
14469 %}
14470
14471 ins_pipe(fp_uop_d);
14472 %}
14473
14474 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14475 match(Set dst (AbsF (SubF src1 src2)));
14476
14477 ins_cost(INSN_COST * 3);
14478 format %{ "fabds $dst, $src1, $src2" %}
14479 ins_encode %{
14480 __ fabds(as_FloatRegister($dst$$reg),
14481 as_FloatRegister($src1$$reg),
14482 as_FloatRegister($src2$$reg));
14483 %}
14484
14485 ins_pipe(fp_uop_s);
14486 %}
14487
14488 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14489 match(Set dst (AbsD (SubD src1 src2)));
14490
14491 ins_cost(INSN_COST * 3);
14492 format %{ "fabdd $dst, $src1, $src2" %}
14493 ins_encode %{
14494 __ fabdd(as_FloatRegister($dst$$reg),
14495 as_FloatRegister($src1$$reg),
14496 as_FloatRegister($src2$$reg));
14497 %}
14498
14499 ins_pipe(fp_uop_d);
14500 %}
14501
14502 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14503 match(Set dst (SqrtD src));
14504
14505 ins_cost(INSN_COST * 50);
14506 format %{ "fsqrtd $dst, $src" %}
14507 ins_encode %{
14508 __ fsqrtd(as_FloatRegister($dst$$reg),
14509 as_FloatRegister($src$$reg));
14510 %}
14511
14512 ins_pipe(fp_div_s);
14513 %}
14514
14515 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14516 match(Set dst (SqrtF src));
14517
14518 ins_cost(INSN_COST * 50);
14519 format %{ "fsqrts $dst, $src" %}
14520 ins_encode %{
14521 __ fsqrts(as_FloatRegister($dst$$reg),
14522 as_FloatRegister($src$$reg));
14523 %}
14524
14525 ins_pipe(fp_div_d);
14526 %}
14527
14528 // Math.rint, floor, ceil
14529 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14530 match(Set dst (RoundDoubleMode src rmode));
14531 format %{ "frint $dst, $src, $rmode" %}
14532 ins_encode %{
14533 switch ($rmode$$constant) {
14534 case RoundDoubleModeNode::rmode_rint:
14535 __ frintnd(as_FloatRegister($dst$$reg),
14536 as_FloatRegister($src$$reg));
14537 break;
14538 case RoundDoubleModeNode::rmode_floor:
14539 __ frintmd(as_FloatRegister($dst$$reg),
14540 as_FloatRegister($src$$reg));
14541 break;
14542 case RoundDoubleModeNode::rmode_ceil:
14543 __ frintpd(as_FloatRegister($dst$$reg),
14544 as_FloatRegister($src$$reg));
14545 break;
14546 }
14547 %}
14548 ins_pipe(fp_uop_d);
14549 %}
14550
14551 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14552 match(Set dst (CopySignD src1 (Binary src2 zero)));
14553 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14554 format %{ "CopySignD $dst $src1 $src2" %}
14555 ins_encode %{
14556 FloatRegister dst = as_FloatRegister($dst$$reg),
14557 src1 = as_FloatRegister($src1$$reg),
14558 src2 = as_FloatRegister($src2$$reg),
14559 zero = as_FloatRegister($zero$$reg);
14560 __ fnegd(dst, zero);
14561 __ bsl(dst, __ T8B, src2, src1);
14562 %}
14563 ins_pipe(fp_uop_d);
14564 %}
14565
14566 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14567 match(Set dst (CopySignF src1 src2));
14568 effect(TEMP_DEF dst, USE src1, USE src2);
14569 format %{ "CopySignF $dst $src1 $src2" %}
14570 ins_encode %{
14571 FloatRegister dst = as_FloatRegister($dst$$reg),
14572 src1 = as_FloatRegister($src1$$reg),
14573 src2 = as_FloatRegister($src2$$reg);
14574 __ movi(dst, __ T2S, 0x80, 24);
14575 __ bsl(dst, __ T8B, src2, src1);
14576 %}
14577 ins_pipe(fp_uop_d);
14578 %}
14579
14580 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14581 match(Set dst (SignumD src (Binary zero one)));
14582 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14583 format %{ "signumD $dst, $src" %}
14584 ins_encode %{
14585 FloatRegister src = as_FloatRegister($src$$reg),
14586 dst = as_FloatRegister($dst$$reg),
14587 zero = as_FloatRegister($zero$$reg),
14588 one = as_FloatRegister($one$$reg);
14589 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14590 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14591 // Bit selection instruction gets bit from "one" for each enabled bit in
14592 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14593 // NaN the whole "src" will be copied because "dst" is zero. For all other
14594 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14595 // from "src", and all other bits are copied from 1.0.
14596 __ bsl(dst, __ T8B, one, src);
14597 %}
14598 ins_pipe(fp_uop_d);
14599 %}
14600
14601 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14602 match(Set dst (SignumF src (Binary zero one)));
14603 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14604 format %{ "signumF $dst, $src" %}
14605 ins_encode %{
14606 FloatRegister src = as_FloatRegister($src$$reg),
14607 dst = as_FloatRegister($dst$$reg),
14608 zero = as_FloatRegister($zero$$reg),
14609 one = as_FloatRegister($one$$reg);
14610 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14611 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14612 // Bit selection instruction gets bit from "one" for each enabled bit in
14613 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14614 // NaN the whole "src" will be copied because "dst" is zero. For all other
14615 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14616 // from "src", and all other bits are copied from 1.0.
14617 __ bsl(dst, __ T8B, one, src);
14618 %}
14619 ins_pipe(fp_uop_d);
14620 %}
14621
14622 instruct onspinwait() %{
14623 match(OnSpinWait);
14624 ins_cost(INSN_COST);
14625
14626 format %{ "onspinwait" %}
14627
14628 ins_encode %{
14629 __ spin_wait();
14630 %}
14631 ins_pipe(pipe_class_empty);
14632 %}
14633
14634 // ============================================================================
14635 // Logical Instructions
14636
14637 // Integer Logical Instructions
14638
14639 // And Instructions
14640
14641
14642 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14643 match(Set dst (AndI src1 src2));
14644
14645 format %{ "andw $dst, $src1, $src2\t# int" %}
14646
14647 ins_cost(INSN_COST);
14648 ins_encode %{
14649 __ andw(as_Register($dst$$reg),
14650 as_Register($src1$$reg),
14651 as_Register($src2$$reg));
14652 %}
14653
14654 ins_pipe(ialu_reg_reg);
14655 %}
14656
14657 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14658 match(Set dst (AndI src1 src2));
14659
14660 format %{ "andsw $dst, $src1, $src2\t# int" %}
14661
14662 ins_cost(INSN_COST);
14663 ins_encode %{
14664 __ andw(as_Register($dst$$reg),
14665 as_Register($src1$$reg),
14666 (uint64_t)($src2$$constant));
14667 %}
14668
14669 ins_pipe(ialu_reg_imm);
14670 %}
14671
14672 // Or Instructions
14673
14674 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14675 match(Set dst (OrI src1 src2));
14676
14677 format %{ "orrw $dst, $src1, $src2\t# int" %}
14678
14679 ins_cost(INSN_COST);
14680 ins_encode %{
14681 __ orrw(as_Register($dst$$reg),
14682 as_Register($src1$$reg),
14683 as_Register($src2$$reg));
14684 %}
14685
14686 ins_pipe(ialu_reg_reg);
14687 %}
14688
14689 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14690 match(Set dst (OrI src1 src2));
14691
14692 format %{ "orrw $dst, $src1, $src2\t# int" %}
14693
14694 ins_cost(INSN_COST);
14695 ins_encode %{
14696 __ orrw(as_Register($dst$$reg),
14697 as_Register($src1$$reg),
14698 (uint64_t)($src2$$constant));
14699 %}
14700
14701 ins_pipe(ialu_reg_imm);
14702 %}
14703
14704 // Xor Instructions
14705
14706 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14707 match(Set dst (XorI src1 src2));
14708
14709 format %{ "eorw $dst, $src1, $src2\t# int" %}
14710
14711 ins_cost(INSN_COST);
14712 ins_encode %{
14713 __ eorw(as_Register($dst$$reg),
14714 as_Register($src1$$reg),
14715 as_Register($src2$$reg));
14716 %}
14717
14718 ins_pipe(ialu_reg_reg);
14719 %}
14720
14721 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14722 match(Set dst (XorI src1 src2));
14723
14724 format %{ "eorw $dst, $src1, $src2\t# int" %}
14725
14726 ins_cost(INSN_COST);
14727 ins_encode %{
14728 __ eorw(as_Register($dst$$reg),
14729 as_Register($src1$$reg),
14730 (uint64_t)($src2$$constant));
14731 %}
14732
14733 ins_pipe(ialu_reg_imm);
14734 %}
14735
14736 // Long Logical Instructions
14737 // TODO
14738
14739 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14740 match(Set dst (AndL src1 src2));
14741
14742 format %{ "and $dst, $src1, $src2\t# int" %}
14743
14744 ins_cost(INSN_COST);
14745 ins_encode %{
14746 __ andr(as_Register($dst$$reg),
14747 as_Register($src1$$reg),
14748 as_Register($src2$$reg));
14749 %}
14750
14751 ins_pipe(ialu_reg_reg);
14752 %}
14753
14754 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14755 match(Set dst (AndL src1 src2));
14756
14757 format %{ "and $dst, $src1, $src2\t# int" %}
14758
14759 ins_cost(INSN_COST);
14760 ins_encode %{
14761 __ andr(as_Register($dst$$reg),
14762 as_Register($src1$$reg),
14763 (uint64_t)($src2$$constant));
14764 %}
14765
14766 ins_pipe(ialu_reg_imm);
14767 %}
14768
14769 // Or Instructions
14770
14771 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14772 match(Set dst (OrL src1 src2));
14773
14774 format %{ "orr $dst, $src1, $src2\t# int" %}
14775
14776 ins_cost(INSN_COST);
14777 ins_encode %{
14778 __ orr(as_Register($dst$$reg),
14779 as_Register($src1$$reg),
14780 as_Register($src2$$reg));
14781 %}
14782
14783 ins_pipe(ialu_reg_reg);
14784 %}
14785
14786 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14787 match(Set dst (OrL src1 src2));
14788
14789 format %{ "orr $dst, $src1, $src2\t# int" %}
14790
14791 ins_cost(INSN_COST);
14792 ins_encode %{
14793 __ orr(as_Register($dst$$reg),
14794 as_Register($src1$$reg),
14795 (uint64_t)($src2$$constant));
14796 %}
14797
14798 ins_pipe(ialu_reg_imm);
14799 %}
14800
14801 // Xor Instructions
14802
14803 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14804 match(Set dst (XorL src1 src2));
14805
14806 format %{ "eor $dst, $src1, $src2\t# int" %}
14807
14808 ins_cost(INSN_COST);
14809 ins_encode %{
14810 __ eor(as_Register($dst$$reg),
14811 as_Register($src1$$reg),
14812 as_Register($src2$$reg));
14813 %}
14814
14815 ins_pipe(ialu_reg_reg);
14816 %}
14817
14818 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14819 match(Set dst (XorL src1 src2));
14820
14821 ins_cost(INSN_COST);
14822 format %{ "eor $dst, $src1, $src2\t# int" %}
14823
14824 ins_encode %{
14825 __ eor(as_Register($dst$$reg),
14826 as_Register($src1$$reg),
14827 (uint64_t)($src2$$constant));
14828 %}
14829
14830 ins_pipe(ialu_reg_imm);
14831 %}
14832
14833 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14834 %{
14835 match(Set dst (ConvI2L src));
14836
14837 ins_cost(INSN_COST);
14838 format %{ "sxtw $dst, $src\t# i2l" %}
14839 ins_encode %{
14840 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14841 %}
14842 ins_pipe(ialu_reg_shift);
14843 %}
14844
14845 // this pattern occurs in bigmath arithmetic
14846 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14847 %{
14848 match(Set dst (AndL (ConvI2L src) mask));
14849
14850 ins_cost(INSN_COST);
14851 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14852 ins_encode %{
14853 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14854 %}
14855
14856 ins_pipe(ialu_reg_shift);
14857 %}
14858
14859 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14860 match(Set dst (ConvL2I src));
14861
14862 ins_cost(INSN_COST);
14863 format %{ "movw $dst, $src \t// l2i" %}
14864
14865 ins_encode %{
14866 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14867 %}
14868
14869 ins_pipe(ialu_reg);
14870 %}
14871
14872 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14873 %{
14874 match(Set dst (Conv2B src));
14875 effect(KILL cr);
14876
14877 format %{
14878 "cmpw $src, zr\n\t"
14879 "cset $dst, ne"
14880 %}
14881
14882 ins_encode %{
14883 __ cmpw(as_Register($src$$reg), zr);
14884 __ cset(as_Register($dst$$reg), Assembler::NE);
14885 %}
14886
14887 ins_pipe(ialu_reg);
14888 %}
14889
14890 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14891 %{
14892 match(Set dst (Conv2B src));
14893 effect(KILL cr);
14894
14895 format %{
14896 "cmp $src, zr\n\t"
14897 "cset $dst, ne"
14898 %}
14899
14900 ins_encode %{
14901 __ cmp(as_Register($src$$reg), zr);
14902 __ cset(as_Register($dst$$reg), Assembler::NE);
14903 %}
14904
14905 ins_pipe(ialu_reg);
14906 %}
14907
14908 instruct convD2F_reg(vRegF dst, vRegD src) %{
14909 match(Set dst (ConvD2F src));
14910
14911 ins_cost(INSN_COST * 5);
14912 format %{ "fcvtd $dst, $src \t// d2f" %}
14913
14914 ins_encode %{
14915 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14916 %}
14917
14918 ins_pipe(fp_d2f);
14919 %}
14920
14921 instruct convF2D_reg(vRegD dst, vRegF src) %{
14922 match(Set dst (ConvF2D src));
14923
14924 ins_cost(INSN_COST * 5);
14925 format %{ "fcvts $dst, $src \t// f2d" %}
14926
14927 ins_encode %{
14928 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14929 %}
14930
14931 ins_pipe(fp_f2d);
14932 %}
14933
14934 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14935 match(Set dst (ConvF2I src));
14936
14937 ins_cost(INSN_COST * 5);
14938 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14939
14940 ins_encode %{
14941 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14942 %}
14943
14944 ins_pipe(fp_f2i);
14945 %}
14946
14947 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14948 match(Set dst (ConvF2L src));
14949
14950 ins_cost(INSN_COST * 5);
14951 format %{ "fcvtzs $dst, $src \t// f2l" %}
14952
14953 ins_encode %{
14954 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14955 %}
14956
14957 ins_pipe(fp_f2l);
14958 %}
14959
14960 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14961 match(Set dst (ConvF2HF src));
14962 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14963 "smov $dst, $tmp\t# move result from $tmp to $dst"
14964 %}
14965 effect(TEMP tmp);
14966 ins_encode %{
14967 __ fcvtsh($tmp$$FloatRegister, $src$$FloatRegister);
14968 __ smov($dst$$Register, $tmp$$FloatRegister, __ H, 0);
14969 %}
14970 ins_pipe(pipe_slow);
14971 %}
14972
14973 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14974 match(Set dst (ConvHF2F src));
14975 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14976 "fcvt $dst, $tmp\t# convert half to single precision"
14977 %}
14978 effect(TEMP tmp);
14979 ins_encode %{
14980 __ mov($tmp$$FloatRegister, __ H, 0, $src$$Register);
14981 __ fcvths($dst$$FloatRegister, $tmp$$FloatRegister);
14982 %}
14983 ins_pipe(pipe_slow);
14984 %}
14985
14986 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14987 match(Set dst (ConvI2F src));
14988
14989 ins_cost(INSN_COST * 5);
14990 format %{ "scvtfws $dst, $src \t// i2f" %}
14991
14992 ins_encode %{
14993 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14994 %}
14995
14996 ins_pipe(fp_i2f);
14997 %}
14998
14999 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15000 match(Set dst (ConvL2F src));
15001
15002 ins_cost(INSN_COST * 5);
15003 format %{ "scvtfs $dst, $src \t// l2f" %}
15004
15005 ins_encode %{
15006 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15007 %}
15008
15009 ins_pipe(fp_l2f);
15010 %}
15011
15012 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15013 match(Set dst (ConvD2I src));
15014
15015 ins_cost(INSN_COST * 5);
15016 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15017
15018 ins_encode %{
15019 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15020 %}
15021
15022 ins_pipe(fp_d2i);
15023 %}
15024
15025 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15026 match(Set dst (ConvD2L src));
15027
15028 ins_cost(INSN_COST * 5);
15029 format %{ "fcvtzd $dst, $src \t// d2l" %}
15030
15031 ins_encode %{
15032 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15033 %}
15034
15035 ins_pipe(fp_d2l);
15036 %}
15037
15038 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15039 match(Set dst (ConvI2D src));
15040
15041 ins_cost(INSN_COST * 5);
15042 format %{ "scvtfwd $dst, $src \t// i2d" %}
15043
15044 ins_encode %{
15045 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15046 %}
15047
15048 ins_pipe(fp_i2d);
15049 %}
15050
15051 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15052 match(Set dst (ConvL2D src));
15053
15054 ins_cost(INSN_COST * 5);
15055 format %{ "scvtfd $dst, $src \t// l2d" %}
15056
15057 ins_encode %{
15058 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15059 %}
15060
15061 ins_pipe(fp_l2d);
15062 %}
15063
15064 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15065 %{
15066 match(Set dst (RoundD src));
15067 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15068 format %{ "java_round_double $dst,$src"%}
15069 ins_encode %{
15070 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15071 as_FloatRegister($ftmp$$reg));
15072 %}
15073 ins_pipe(pipe_slow);
15074 %}
15075
15076 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15077 %{
15078 match(Set dst (RoundF src));
15079 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15080 format %{ "java_round_float $dst,$src"%}
15081 ins_encode %{
15082 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15083 as_FloatRegister($ftmp$$reg));
15084 %}
15085 ins_pipe(pipe_slow);
15086 %}
15087
15088 // stack <-> reg and reg <-> reg shuffles with no conversion
15089
15090 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15091
15092 match(Set dst (MoveF2I src));
15093
15094 effect(DEF dst, USE src);
15095
15096 ins_cost(4 * INSN_COST);
15097
15098 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15099
15100 ins_encode %{
15101 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15102 %}
15103
15104 ins_pipe(iload_reg_reg);
15105
15106 %}
15107
15108 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15109
15110 match(Set dst (MoveI2F src));
15111
15112 effect(DEF dst, USE src);
15113
15114 ins_cost(4 * INSN_COST);
15115
15116 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15117
15118 ins_encode %{
15119 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15120 %}
15121
15122 ins_pipe(pipe_class_memory);
15123
15124 %}
15125
15126 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15127
15128 match(Set dst (MoveD2L src));
15129
15130 effect(DEF dst, USE src);
15131
15132 ins_cost(4 * INSN_COST);
15133
15134 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15135
15136 ins_encode %{
15137 __ ldr($dst$$Register, Address(sp, $src$$disp));
15138 %}
15139
15140 ins_pipe(iload_reg_reg);
15141
15142 %}
15143
15144 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15145
15146 match(Set dst (MoveL2D src));
15147
15148 effect(DEF dst, USE src);
15149
15150 ins_cost(4 * INSN_COST);
15151
15152 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15153
15154 ins_encode %{
15155 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15156 %}
15157
15158 ins_pipe(pipe_class_memory);
15159
15160 %}
15161
15162 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15163
15164 match(Set dst (MoveF2I src));
15165
15166 effect(DEF dst, USE src);
15167
15168 ins_cost(INSN_COST);
15169
15170 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15171
15172 ins_encode %{
15173 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15174 %}
15175
15176 ins_pipe(pipe_class_memory);
15177
15178 %}
15179
15180 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15181
15182 match(Set dst (MoveI2F src));
15183
15184 effect(DEF dst, USE src);
15185
15186 ins_cost(INSN_COST);
15187
15188 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15189
15190 ins_encode %{
15191 __ strw($src$$Register, Address(sp, $dst$$disp));
15192 %}
15193
15194 ins_pipe(istore_reg_reg);
15195
15196 %}
15197
15198 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15199
15200 match(Set dst (MoveD2L src));
15201
15202 effect(DEF dst, USE src);
15203
15204 ins_cost(INSN_COST);
15205
15206 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15207
15208 ins_encode %{
15209 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15210 %}
15211
15212 ins_pipe(pipe_class_memory);
15213
15214 %}
15215
15216 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15217
15218 match(Set dst (MoveL2D src));
15219
15220 effect(DEF dst, USE src);
15221
15222 ins_cost(INSN_COST);
15223
15224 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15225
15226 ins_encode %{
15227 __ str($src$$Register, Address(sp, $dst$$disp));
15228 %}
15229
15230 ins_pipe(istore_reg_reg);
15231
15232 %}
15233
15234 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15235
15236 match(Set dst (MoveF2I src));
15237
15238 effect(DEF dst, USE src);
15239
15240 ins_cost(INSN_COST);
15241
15242 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15243
15244 ins_encode %{
15245 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15246 %}
15247
15248 ins_pipe(fp_f2i);
15249
15250 %}
15251
15252 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15253
15254 match(Set dst (MoveI2F src));
15255
15256 effect(DEF dst, USE src);
15257
15258 ins_cost(INSN_COST);
15259
15260 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15261
15262 ins_encode %{
15263 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15264 %}
15265
15266 ins_pipe(fp_i2f);
15267
15268 %}
15269
15270 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15271
15272 match(Set dst (MoveD2L src));
15273
15274 effect(DEF dst, USE src);
15275
15276 ins_cost(INSN_COST);
15277
15278 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15279
15280 ins_encode %{
15281 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15282 %}
15283
15284 ins_pipe(fp_d2l);
15285
15286 %}
15287
15288 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15289
15290 match(Set dst (MoveL2D src));
15291
15292 effect(DEF dst, USE src);
15293
15294 ins_cost(INSN_COST);
15295
15296 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15297
15298 ins_encode %{
15299 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15300 %}
15301
15302 ins_pipe(fp_l2d);
15303
15304 %}
15305
15306 // ============================================================================
15307 // clearing of an array
15308
15309 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15310 %{
15311 match(Set dummy (ClearArray cnt base));
15312 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15313
15314 ins_cost(4 * INSN_COST);
15315 format %{ "ClearArray $cnt, $base" %}
15316
15317 ins_encode %{
15318 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15319 if (tpc == NULL) {
15320 ciEnv::current()->record_failure("CodeCache is full");
15321 return;
15322 }
15323 %}
15324
15325 ins_pipe(pipe_class_memory);
15326 %}
15327
15328 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15329 %{
15330 predicate((uint64_t)n->in(2)->get_long()
15331 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15332 match(Set dummy (ClearArray cnt base));
15333 effect(TEMP temp, USE_KILL base, KILL cr);
15334
15335 ins_cost(4 * INSN_COST);
15336 format %{ "ClearArray $cnt, $base" %}
15337
15338 ins_encode %{
15339 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15340 if (tpc == NULL) {
15341 ciEnv::current()->record_failure("CodeCache is full");
15342 return;
15343 }
15344 %}
15345
15346 ins_pipe(pipe_class_memory);
15347 %}
15348
15349 // ============================================================================
15350 // Overflow Math Instructions
15351
15352 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15353 %{
15354 match(Set cr (OverflowAddI op1 op2));
15355
15356 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15357 ins_cost(INSN_COST);
15358 ins_encode %{
15359 __ cmnw($op1$$Register, $op2$$Register);
15360 %}
15361
15362 ins_pipe(icmp_reg_reg);
15363 %}
15364
15365 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15366 %{
15367 match(Set cr (OverflowAddI op1 op2));
15368
15369 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15370 ins_cost(INSN_COST);
15371 ins_encode %{
15372 __ cmnw($op1$$Register, $op2$$constant);
15373 %}
15374
15375 ins_pipe(icmp_reg_imm);
15376 %}
15377
15378 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15379 %{
15380 match(Set cr (OverflowAddL op1 op2));
15381
15382 format %{ "cmn $op1, $op2\t# overflow check long" %}
15383 ins_cost(INSN_COST);
15384 ins_encode %{
15385 __ cmn($op1$$Register, $op2$$Register);
15386 %}
15387
15388 ins_pipe(icmp_reg_reg);
15389 %}
15390
15391 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15392 %{
15393 match(Set cr (OverflowAddL op1 op2));
15394
15395 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15396 ins_cost(INSN_COST);
15397 ins_encode %{
15398 __ adds(zr, $op1$$Register, $op2$$constant);
15399 %}
15400
15401 ins_pipe(icmp_reg_imm);
15402 %}
15403
15404 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15405 %{
15406 match(Set cr (OverflowSubI op1 op2));
15407
15408 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15409 ins_cost(INSN_COST);
15410 ins_encode %{
15411 __ cmpw($op1$$Register, $op2$$Register);
15412 %}
15413
15414 ins_pipe(icmp_reg_reg);
15415 %}
15416
15417 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15418 %{
15419 match(Set cr (OverflowSubI op1 op2));
15420
15421 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15422 ins_cost(INSN_COST);
15423 ins_encode %{
15424 __ cmpw($op1$$Register, $op2$$constant);
15425 %}
15426
15427 ins_pipe(icmp_reg_imm);
15428 %}
15429
15430 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15431 %{
15432 match(Set cr (OverflowSubL op1 op2));
15433
15434 format %{ "cmp $op1, $op2\t# overflow check long" %}
15435 ins_cost(INSN_COST);
15436 ins_encode %{
15437 __ cmp($op1$$Register, $op2$$Register);
15438 %}
15439
15440 ins_pipe(icmp_reg_reg);
15441 %}
15442
15443 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15444 %{
15445 match(Set cr (OverflowSubL op1 op2));
15446
15447 format %{ "cmp $op1, $op2\t# overflow check long" %}
15448 ins_cost(INSN_COST);
15449 ins_encode %{
15450 __ subs(zr, $op1$$Register, $op2$$constant);
15451 %}
15452
15453 ins_pipe(icmp_reg_imm);
15454 %}
15455
15456 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15457 %{
15458 match(Set cr (OverflowSubI zero op1));
15459
15460 format %{ "cmpw zr, $op1\t# overflow check int" %}
15461 ins_cost(INSN_COST);
15462 ins_encode %{
15463 __ cmpw(zr, $op1$$Register);
15464 %}
15465
15466 ins_pipe(icmp_reg_imm);
15467 %}
15468
15469 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15470 %{
15471 match(Set cr (OverflowSubL zero op1));
15472
15473 format %{ "cmp zr, $op1\t# overflow check long" %}
15474 ins_cost(INSN_COST);
15475 ins_encode %{
15476 __ cmp(zr, $op1$$Register);
15477 %}
15478
15479 ins_pipe(icmp_reg_imm);
15480 %}
15481
15482 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15483 %{
15484 match(Set cr (OverflowMulI op1 op2));
15485
15486 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15487 "cmp rscratch1, rscratch1, sxtw\n\t"
15488 "movw rscratch1, #0x80000000\n\t"
15489 "cselw rscratch1, rscratch1, zr, NE\n\t"
15490 "cmpw rscratch1, #1" %}
15491 ins_cost(5 * INSN_COST);
15492 ins_encode %{
15493 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15494 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15495 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15496 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15497 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15498 %}
15499
15500 ins_pipe(pipe_slow);
15501 %}
15502
15503 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15504 %{
15505 match(If cmp (OverflowMulI op1 op2));
15506 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15507 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15508 effect(USE labl, KILL cr);
15509
15510 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15511 "cmp rscratch1, rscratch1, sxtw\n\t"
15512 "b$cmp $labl" %}
15513 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15514 ins_encode %{
15515 Label* L = $labl$$label;
15516 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15517 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15518 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15519 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15520 %}
15521
15522 ins_pipe(pipe_serial);
15523 %}
15524
15525 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15526 %{
15527 match(Set cr (OverflowMulL op1 op2));
15528
15529 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15530 "smulh rscratch2, $op1, $op2\n\t"
15531 "cmp rscratch2, rscratch1, ASR #63\n\t"
15532 "movw rscratch1, #0x80000000\n\t"
15533 "cselw rscratch1, rscratch1, zr, NE\n\t"
15534 "cmpw rscratch1, #1" %}
15535 ins_cost(6 * INSN_COST);
15536 ins_encode %{
15537 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15538 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15539 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15540 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15541 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15542 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15543 %}
15544
15545 ins_pipe(pipe_slow);
15546 %}
15547
15548 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15549 %{
15550 match(If cmp (OverflowMulL op1 op2));
15551 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15552 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15553 effect(USE labl, KILL cr);
15554
15555 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15556 "smulh rscratch2, $op1, $op2\n\t"
15557 "cmp rscratch2, rscratch1, ASR #63\n\t"
15558 "b$cmp $labl" %}
15559 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15560 ins_encode %{
15561 Label* L = $labl$$label;
15562 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15563 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15564 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15565 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15566 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15567 %}
15568
15569 ins_pipe(pipe_serial);
15570 %}
15571
15572 // ============================================================================
15573 // Compare Instructions
15574
15575 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15576 %{
15577 match(Set cr (CmpI op1 op2));
15578
15579 effect(DEF cr, USE op1, USE op2);
15580
15581 ins_cost(INSN_COST);
15582 format %{ "cmpw $op1, $op2" %}
15583
15584 ins_encode(aarch64_enc_cmpw(op1, op2));
15585
15586 ins_pipe(icmp_reg_reg);
15587 %}
15588
15589 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15590 %{
15591 match(Set cr (CmpI op1 zero));
15592
15593 effect(DEF cr, USE op1);
15594
15595 ins_cost(INSN_COST);
15596 format %{ "cmpw $op1, 0" %}
15597
15598 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15599
15600 ins_pipe(icmp_reg_imm);
15601 %}
15602
15603 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15604 %{
15605 match(Set cr (CmpI op1 op2));
15606
15607 effect(DEF cr, USE op1);
15608
15609 ins_cost(INSN_COST);
15610 format %{ "cmpw $op1, $op2" %}
15611
15612 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15613
15614 ins_pipe(icmp_reg_imm);
15615 %}
15616
15617 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15618 %{
15619 match(Set cr (CmpI op1 op2));
15620
15621 effect(DEF cr, USE op1);
15622
15623 ins_cost(INSN_COST * 2);
15624 format %{ "cmpw $op1, $op2" %}
15625
15626 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15627
15628 ins_pipe(icmp_reg_imm);
15629 %}
15630
15631 // Unsigned compare Instructions; really, same as signed compare
15632 // except it should only be used to feed an If or a CMovI which takes a
15633 // cmpOpU.
15634
15635 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15636 %{
15637 match(Set cr (CmpU op1 op2));
15638
15639 effect(DEF cr, USE op1, USE op2);
15640
15641 ins_cost(INSN_COST);
15642 format %{ "cmpw $op1, $op2\t# unsigned" %}
15643
15644 ins_encode(aarch64_enc_cmpw(op1, op2));
15645
15646 ins_pipe(icmp_reg_reg);
15647 %}
15648
15649 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15650 %{
15651 match(Set cr (CmpU op1 zero));
15652
15653 effect(DEF cr, USE op1);
15654
15655 ins_cost(INSN_COST);
15656 format %{ "cmpw $op1, #0\t# unsigned" %}
15657
15658 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15659
15660 ins_pipe(icmp_reg_imm);
15661 %}
15662
15663 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15664 %{
15665 match(Set cr (CmpU op1 op2));
15666
15667 effect(DEF cr, USE op1);
15668
15669 ins_cost(INSN_COST);
15670 format %{ "cmpw $op1, $op2\t# unsigned" %}
15671
15672 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15673
15674 ins_pipe(icmp_reg_imm);
15675 %}
15676
15677 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15678 %{
15679 match(Set cr (CmpU op1 op2));
15680
15681 effect(DEF cr, USE op1);
15682
15683 ins_cost(INSN_COST * 2);
15684 format %{ "cmpw $op1, $op2\t# unsigned" %}
15685
15686 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15687
15688 ins_pipe(icmp_reg_imm);
15689 %}
15690
15691 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15692 %{
15693 match(Set cr (CmpL op1 op2));
15694
15695 effect(DEF cr, USE op1, USE op2);
15696
15697 ins_cost(INSN_COST);
15698 format %{ "cmp $op1, $op2" %}
15699
15700 ins_encode(aarch64_enc_cmp(op1, op2));
15701
15702 ins_pipe(icmp_reg_reg);
15703 %}
15704
15705 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15706 %{
15707 match(Set cr (CmpL op1 zero));
15708
15709 effect(DEF cr, USE op1);
15710
15711 ins_cost(INSN_COST);
15712 format %{ "tst $op1" %}
15713
15714 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15715
15716 ins_pipe(icmp_reg_imm);
15717 %}
15718
15719 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15720 %{
15721 match(Set cr (CmpL op1 op2));
15722
15723 effect(DEF cr, USE op1);
15724
15725 ins_cost(INSN_COST);
15726 format %{ "cmp $op1, $op2" %}
15727
15728 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15729
15730 ins_pipe(icmp_reg_imm);
15731 %}
15732
15733 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15734 %{
15735 match(Set cr (CmpL op1 op2));
15736
15737 effect(DEF cr, USE op1);
15738
15739 ins_cost(INSN_COST * 2);
15740 format %{ "cmp $op1, $op2" %}
15741
15742 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15743
15744 ins_pipe(icmp_reg_imm);
15745 %}
15746
15747 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15748 %{
15749 match(Set cr (CmpUL op1 op2));
15750
15751 effect(DEF cr, USE op1, USE op2);
15752
15753 ins_cost(INSN_COST);
15754 format %{ "cmp $op1, $op2" %}
15755
15756 ins_encode(aarch64_enc_cmp(op1, op2));
15757
15758 ins_pipe(icmp_reg_reg);
15759 %}
15760
15761 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15762 %{
15763 match(Set cr (CmpUL op1 zero));
15764
15765 effect(DEF cr, USE op1);
15766
15767 ins_cost(INSN_COST);
15768 format %{ "tst $op1" %}
15769
15770 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15771
15772 ins_pipe(icmp_reg_imm);
15773 %}
15774
15775 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15776 %{
15777 match(Set cr (CmpUL op1 op2));
15778
15779 effect(DEF cr, USE op1);
15780
15781 ins_cost(INSN_COST);
15782 format %{ "cmp $op1, $op2" %}
15783
15784 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15785
15786 ins_pipe(icmp_reg_imm);
15787 %}
15788
15789 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15790 %{
15791 match(Set cr (CmpUL op1 op2));
15792
15793 effect(DEF cr, USE op1);
15794
15795 ins_cost(INSN_COST * 2);
15796 format %{ "cmp $op1, $op2" %}
15797
15798 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15799
15800 ins_pipe(icmp_reg_imm);
15801 %}
15802
15803 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15804 %{
15805 match(Set cr (CmpP op1 op2));
15806
15807 effect(DEF cr, USE op1, USE op2);
15808
15809 ins_cost(INSN_COST);
15810 format %{ "cmp $op1, $op2\t // ptr" %}
15811
15812 ins_encode(aarch64_enc_cmpp(op1, op2));
15813
15814 ins_pipe(icmp_reg_reg);
15815 %}
15816
15817 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15818 %{
15819 match(Set cr (CmpN op1 op2));
15820
15821 effect(DEF cr, USE op1, USE op2);
15822
15823 ins_cost(INSN_COST);
15824 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15825
15826 ins_encode(aarch64_enc_cmpn(op1, op2));
15827
15828 ins_pipe(icmp_reg_reg);
15829 %}
15830
15831 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15832 %{
15833 match(Set cr (CmpP op1 zero));
15834
15835 effect(DEF cr, USE op1, USE zero);
15836
15837 ins_cost(INSN_COST);
15838 format %{ "cmp $op1, 0\t // ptr" %}
15839
15840 ins_encode(aarch64_enc_testp(op1));
15841
15842 ins_pipe(icmp_reg_imm);
15843 %}
15844
15845 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15846 %{
15847 match(Set cr (CmpN op1 zero));
15848
15849 effect(DEF cr, USE op1, USE zero);
15850
15851 ins_cost(INSN_COST);
15852 format %{ "cmp $op1, 0\t // compressed ptr" %}
15853
15854 ins_encode(aarch64_enc_testn(op1));
15855
15856 ins_pipe(icmp_reg_imm);
15857 %}
15858
15859 // FP comparisons
15860 //
15861 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15862 // using normal cmpOp. See declaration of rFlagsReg for details.
15863
15864 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15865 %{
15866 match(Set cr (CmpF src1 src2));
15867
15868 ins_cost(3 * INSN_COST);
15869 format %{ "fcmps $src1, $src2" %}
15870
15871 ins_encode %{
15872 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15873 %}
15874
15875 ins_pipe(pipe_class_compare);
15876 %}
15877
15878 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15879 %{
15880 match(Set cr (CmpF src1 src2));
15881
15882 ins_cost(3 * INSN_COST);
15883 format %{ "fcmps $src1, 0.0" %}
15884
15885 ins_encode %{
15886 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15887 %}
15888
15889 ins_pipe(pipe_class_compare);
15890 %}
15891 // FROM HERE
15892
15893 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15894 %{
15895 match(Set cr (CmpD src1 src2));
15896
15897 ins_cost(3 * INSN_COST);
15898 format %{ "fcmpd $src1, $src2" %}
15899
15900 ins_encode %{
15901 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15902 %}
15903
15904 ins_pipe(pipe_class_compare);
15905 %}
15906
15907 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15908 %{
15909 match(Set cr (CmpD src1 src2));
15910
15911 ins_cost(3 * INSN_COST);
15912 format %{ "fcmpd $src1, 0.0" %}
15913
15914 ins_encode %{
15915 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15916 %}
15917
15918 ins_pipe(pipe_class_compare);
15919 %}
15920
15921 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15922 %{
15923 match(Set dst (CmpF3 src1 src2));
15924 effect(KILL cr);
15925
15926 ins_cost(5 * INSN_COST);
15927 format %{ "fcmps $src1, $src2\n\t"
15928 "csinvw($dst, zr, zr, eq\n\t"
15929 "csnegw($dst, $dst, $dst, lt)"
15930 %}
15931
15932 ins_encode %{
15933 Label done;
15934 FloatRegister s1 = as_FloatRegister($src1$$reg);
15935 FloatRegister s2 = as_FloatRegister($src2$$reg);
15936 Register d = as_Register($dst$$reg);
15937 __ fcmps(s1, s2);
15938 // installs 0 if EQ else -1
15939 __ csinvw(d, zr, zr, Assembler::EQ);
15940 // keeps -1 if less or unordered else installs 1
15941 __ csnegw(d, d, d, Assembler::LT);
15942 __ bind(done);
15943 %}
15944
15945 ins_pipe(pipe_class_default);
15946
15947 %}
15948
15949 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15950 %{
15951 match(Set dst (CmpD3 src1 src2));
15952 effect(KILL cr);
15953
15954 ins_cost(5 * INSN_COST);
15955 format %{ "fcmpd $src1, $src2\n\t"
15956 "csinvw($dst, zr, zr, eq\n\t"
15957 "csnegw($dst, $dst, $dst, lt)"
15958 %}
15959
15960 ins_encode %{
15961 Label done;
15962 FloatRegister s1 = as_FloatRegister($src1$$reg);
15963 FloatRegister s2 = as_FloatRegister($src2$$reg);
15964 Register d = as_Register($dst$$reg);
15965 __ fcmpd(s1, s2);
15966 // installs 0 if EQ else -1
15967 __ csinvw(d, zr, zr, Assembler::EQ);
15968 // keeps -1 if less or unordered else installs 1
15969 __ csnegw(d, d, d, Assembler::LT);
15970 __ bind(done);
15971 %}
15972 ins_pipe(pipe_class_default);
15973
15974 %}
15975
15976 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15977 %{
15978 match(Set dst (CmpF3 src1 zero));
15979 effect(KILL cr);
15980
15981 ins_cost(5 * INSN_COST);
15982 format %{ "fcmps $src1, 0.0\n\t"
15983 "csinvw($dst, zr, zr, eq\n\t"
15984 "csnegw($dst, $dst, $dst, lt)"
15985 %}
15986
15987 ins_encode %{
15988 Label done;
15989 FloatRegister s1 = as_FloatRegister($src1$$reg);
15990 Register d = as_Register($dst$$reg);
15991 __ fcmps(s1, 0.0);
15992 // installs 0 if EQ else -1
15993 __ csinvw(d, zr, zr, Assembler::EQ);
15994 // keeps -1 if less or unordered else installs 1
15995 __ csnegw(d, d, d, Assembler::LT);
15996 __ bind(done);
15997 %}
15998
15999 ins_pipe(pipe_class_default);
16000
16001 %}
16002
16003 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16004 %{
16005 match(Set dst (CmpD3 src1 zero));
16006 effect(KILL cr);
16007
16008 ins_cost(5 * INSN_COST);
16009 format %{ "fcmpd $src1, 0.0\n\t"
16010 "csinvw($dst, zr, zr, eq\n\t"
16011 "csnegw($dst, $dst, $dst, lt)"
16012 %}
16013
16014 ins_encode %{
16015 Label done;
16016 FloatRegister s1 = as_FloatRegister($src1$$reg);
16017 Register d = as_Register($dst$$reg);
16018 __ fcmpd(s1, 0.0);
16019 // installs 0 if EQ else -1
16020 __ csinvw(d, zr, zr, Assembler::EQ);
16021 // keeps -1 if less or unordered else installs 1
16022 __ csnegw(d, d, d, Assembler::LT);
16023 __ bind(done);
16024 %}
16025 ins_pipe(pipe_class_default);
16026
16027 %}
16028
16029 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16030 %{
16031 match(Set dst (CmpLTMask p q));
16032 effect(KILL cr);
16033
16034 ins_cost(3 * INSN_COST);
16035
16036 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16037 "csetw $dst, lt\n\t"
16038 "subw $dst, zr, $dst"
16039 %}
16040
16041 ins_encode %{
16042 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16043 __ csetw(as_Register($dst$$reg), Assembler::LT);
16044 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16045 %}
16046
16047 ins_pipe(ialu_reg_reg);
16048 %}
16049
16050 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16051 %{
16052 match(Set dst (CmpLTMask src zero));
16053 effect(KILL cr);
16054
16055 ins_cost(INSN_COST);
16056
16057 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16058
16059 ins_encode %{
16060 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16061 %}
16062
16063 ins_pipe(ialu_reg_shift);
16064 %}
16065
16066 // ============================================================================
16067 // Max and Min
16068
16069 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16070
16071 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16072 %{
16073 effect(DEF cr, USE src);
16074 ins_cost(INSN_COST);
16075 format %{ "cmpw $src, 0" %}
16076
16077 ins_encode %{
16078 __ cmpw($src$$Register, 0);
16079 %}
16080 ins_pipe(icmp_reg_imm);
16081 %}
16082
16083 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16084 %{
16085 match(Set dst (MinI src1 src2));
16086 ins_cost(INSN_COST * 3);
16087
16088 expand %{
16089 rFlagsReg cr;
16090 compI_reg_reg(cr, src1, src2);
16091 cmovI_reg_reg_lt(dst, src1, src2, cr);
16092 %}
16093 %}
16094
16095 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16096 %{
16097 match(Set dst (MaxI src1 src2));
16098 ins_cost(INSN_COST * 3);
16099
16100 expand %{
16101 rFlagsReg cr;
16102 compI_reg_reg(cr, src1, src2);
16103 cmovI_reg_reg_gt(dst, src1, src2, cr);
16104 %}
16105 %}
16106
16107
16108 // ============================================================================
16109 // Branch Instructions
16110
16111 // Direct Branch.
16112 instruct branch(label lbl)
16113 %{
16114 match(Goto);
16115
16116 effect(USE lbl);
16117
16118 ins_cost(BRANCH_COST);
16119 format %{ "b $lbl" %}
16120
16121 ins_encode(aarch64_enc_b(lbl));
16122
16123 ins_pipe(pipe_branch);
16124 %}
16125
16126 // Conditional Near Branch
16127 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16128 %{
16129 // Same match rule as `branchConFar'.
16130 match(If cmp cr);
16131
16132 effect(USE lbl);
16133
16134 ins_cost(BRANCH_COST);
16135 // If set to 1 this indicates that the current instruction is a
16136 // short variant of a long branch. This avoids using this
16137 // instruction in first-pass matching. It will then only be used in
16138 // the `Shorten_branches' pass.
16139 // ins_short_branch(1);
16140 format %{ "b$cmp $lbl" %}
16141
16142 ins_encode(aarch64_enc_br_con(cmp, lbl));
16143
16144 ins_pipe(pipe_branch_cond);
16145 %}
16146
16147 // Conditional Near Branch Unsigned
16148 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16149 %{
16150 // Same match rule as `branchConFar'.
16151 match(If cmp cr);
16152
16153 effect(USE lbl);
16154
16155 ins_cost(BRANCH_COST);
16156 // If set to 1 this indicates that the current instruction is a
16157 // short variant of a long branch. This avoids using this
16158 // instruction in first-pass matching. It will then only be used in
16159 // the `Shorten_branches' pass.
16160 // ins_short_branch(1);
16161 format %{ "b$cmp $lbl\t# unsigned" %}
16162
16163 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16164
16165 ins_pipe(pipe_branch_cond);
16166 %}
16167
16168 // Make use of CBZ and CBNZ. These instructions, as well as being
16169 // shorter than (cmp; branch), have the additional benefit of not
16170 // killing the flags.
16171
16172 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16173 match(If cmp (CmpI op1 op2));
16174 effect(USE labl);
16175
16176 ins_cost(BRANCH_COST);
16177 format %{ "cbw$cmp $op1, $labl" %}
16178 ins_encode %{
16179 Label* L = $labl$$label;
16180 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16181 if (cond == Assembler::EQ)
16182 __ cbzw($op1$$Register, *L);
16183 else
16184 __ cbnzw($op1$$Register, *L);
16185 %}
16186 ins_pipe(pipe_cmp_branch);
16187 %}
16188
16189 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16190 match(If cmp (CmpL op1 op2));
16191 effect(USE labl);
16192
16193 ins_cost(BRANCH_COST);
16194 format %{ "cb$cmp $op1, $labl" %}
16195 ins_encode %{
16196 Label* L = $labl$$label;
16197 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16198 if (cond == Assembler::EQ)
16199 __ cbz($op1$$Register, *L);
16200 else
16201 __ cbnz($op1$$Register, *L);
16202 %}
16203 ins_pipe(pipe_cmp_branch);
16204 %}
16205
16206 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16207 match(If cmp (CmpP op1 op2));
16208 effect(USE labl);
16209
16210 ins_cost(BRANCH_COST);
16211 format %{ "cb$cmp $op1, $labl" %}
16212 ins_encode %{
16213 Label* L = $labl$$label;
16214 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16215 if (cond == Assembler::EQ)
16216 __ cbz($op1$$Register, *L);
16217 else
16218 __ cbnz($op1$$Register, *L);
16219 %}
16220 ins_pipe(pipe_cmp_branch);
16221 %}
16222
16223 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16224 match(If cmp (CmpN op1 op2));
16225 effect(USE labl);
16226
16227 ins_cost(BRANCH_COST);
16228 format %{ "cbw$cmp $op1, $labl" %}
16229 ins_encode %{
16230 Label* L = $labl$$label;
16231 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16232 if (cond == Assembler::EQ)
16233 __ cbzw($op1$$Register, *L);
16234 else
16235 __ cbnzw($op1$$Register, *L);
16236 %}
16237 ins_pipe(pipe_cmp_branch);
16238 %}
16239
16240 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16241 match(If cmp (CmpP (DecodeN oop) zero));
16242 effect(USE labl);
16243
16244 ins_cost(BRANCH_COST);
16245 format %{ "cb$cmp $oop, $labl" %}
16246 ins_encode %{
16247 Label* L = $labl$$label;
16248 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16249 if (cond == Assembler::EQ)
16250 __ cbzw($oop$$Register, *L);
16251 else
16252 __ cbnzw($oop$$Register, *L);
16253 %}
16254 ins_pipe(pipe_cmp_branch);
16255 %}
16256
16257 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16258 match(If cmp (CmpU op1 op2));
16259 effect(USE labl);
16260
16261 ins_cost(BRANCH_COST);
16262 format %{ "cbw$cmp $op1, $labl" %}
16263 ins_encode %{
16264 Label* L = $labl$$label;
16265 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16266 if (cond == Assembler::EQ || cond == Assembler::LS)
16267 __ cbzw($op1$$Register, *L);
16268 else
16269 __ cbnzw($op1$$Register, *L);
16270 %}
16271 ins_pipe(pipe_cmp_branch);
16272 %}
16273
16274 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16275 match(If cmp (CmpUL op1 op2));
16276 effect(USE labl);
16277
16278 ins_cost(BRANCH_COST);
16279 format %{ "cb$cmp $op1, $labl" %}
16280 ins_encode %{
16281 Label* L = $labl$$label;
16282 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16283 if (cond == Assembler::EQ || cond == Assembler::LS)
16284 __ cbz($op1$$Register, *L);
16285 else
16286 __ cbnz($op1$$Register, *L);
16287 %}
16288 ins_pipe(pipe_cmp_branch);
16289 %}
16290
16291 // Test bit and Branch
16292
16293 // Patterns for short (< 32KiB) variants
16294 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16295 match(If cmp (CmpL op1 op2));
16296 effect(USE labl);
16297
16298 ins_cost(BRANCH_COST);
16299 format %{ "cb$cmp $op1, $labl # long" %}
16300 ins_encode %{
16301 Label* L = $labl$$label;
16302 Assembler::Condition cond =
16303 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16304 __ tbr(cond, $op1$$Register, 63, *L);
16305 %}
16306 ins_pipe(pipe_cmp_branch);
16307 ins_short_branch(1);
16308 %}
16309
16310 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16311 match(If cmp (CmpI op1 op2));
16312 effect(USE labl);
16313
16314 ins_cost(BRANCH_COST);
16315 format %{ "cb$cmp $op1, $labl # int" %}
16316 ins_encode %{
16317 Label* L = $labl$$label;
16318 Assembler::Condition cond =
16319 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16320 __ tbr(cond, $op1$$Register, 31, *L);
16321 %}
16322 ins_pipe(pipe_cmp_branch);
16323 ins_short_branch(1);
16324 %}
16325
16326 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16327 match(If cmp (CmpL (AndL op1 op2) op3));
16328 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16329 effect(USE labl);
16330
16331 ins_cost(BRANCH_COST);
16332 format %{ "tb$cmp $op1, $op2, $labl" %}
16333 ins_encode %{
16334 Label* L = $labl$$label;
16335 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16336 int bit = exact_log2_long($op2$$constant);
16337 __ tbr(cond, $op1$$Register, bit, *L);
16338 %}
16339 ins_pipe(pipe_cmp_branch);
16340 ins_short_branch(1);
16341 %}
16342
16343 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16344 match(If cmp (CmpI (AndI op1 op2) op3));
16345 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16346 effect(USE labl);
16347
16348 ins_cost(BRANCH_COST);
16349 format %{ "tb$cmp $op1, $op2, $labl" %}
16350 ins_encode %{
16351 Label* L = $labl$$label;
16352 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16353 int bit = exact_log2((juint)$op2$$constant);
16354 __ tbr(cond, $op1$$Register, bit, *L);
16355 %}
16356 ins_pipe(pipe_cmp_branch);
16357 ins_short_branch(1);
16358 %}
16359
16360 // And far variants
16361 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16362 match(If cmp (CmpL op1 op2));
16363 effect(USE labl);
16364
16365 ins_cost(BRANCH_COST);
16366 format %{ "cb$cmp $op1, $labl # long" %}
16367 ins_encode %{
16368 Label* L = $labl$$label;
16369 Assembler::Condition cond =
16370 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16371 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16372 %}
16373 ins_pipe(pipe_cmp_branch);
16374 %}
16375
16376 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16377 match(If cmp (CmpI op1 op2));
16378 effect(USE labl);
16379
16380 ins_cost(BRANCH_COST);
16381 format %{ "cb$cmp $op1, $labl # int" %}
16382 ins_encode %{
16383 Label* L = $labl$$label;
16384 Assembler::Condition cond =
16385 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16386 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16387 %}
16388 ins_pipe(pipe_cmp_branch);
16389 %}
16390
16391 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16392 match(If cmp (CmpL (AndL op1 op2) op3));
16393 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16394 effect(USE labl);
16395
16396 ins_cost(BRANCH_COST);
16397 format %{ "tb$cmp $op1, $op2, $labl" %}
16398 ins_encode %{
16399 Label* L = $labl$$label;
16400 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16401 int bit = exact_log2_long($op2$$constant);
16402 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16403 %}
16404 ins_pipe(pipe_cmp_branch);
16405 %}
16406
16407 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16408 match(If cmp (CmpI (AndI op1 op2) op3));
16409 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16410 effect(USE labl);
16411
16412 ins_cost(BRANCH_COST);
16413 format %{ "tb$cmp $op1, $op2, $labl" %}
16414 ins_encode %{
16415 Label* L = $labl$$label;
16416 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16417 int bit = exact_log2((juint)$op2$$constant);
16418 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16419 %}
16420 ins_pipe(pipe_cmp_branch);
16421 %}
16422
16423 // Test bits
16424
16425 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16426 match(Set cr (CmpL (AndL op1 op2) op3));
16427 predicate(Assembler::operand_valid_for_logical_immediate
16428 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16429
16430 ins_cost(INSN_COST);
16431 format %{ "tst $op1, $op2 # long" %}
16432 ins_encode %{
16433 __ tst($op1$$Register, $op2$$constant);
16434 %}
16435 ins_pipe(ialu_reg_reg);
16436 %}
16437
16438 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16439 match(Set cr (CmpI (AndI op1 op2) op3));
16440 predicate(Assembler::operand_valid_for_logical_immediate
16441 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16442
16443 ins_cost(INSN_COST);
16444 format %{ "tst $op1, $op2 # int" %}
16445 ins_encode %{
16446 __ tstw($op1$$Register, $op2$$constant);
16447 %}
16448 ins_pipe(ialu_reg_reg);
16449 %}
16450
16451 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16452 match(Set cr (CmpL (AndL op1 op2) op3));
16453
16454 ins_cost(INSN_COST);
16455 format %{ "tst $op1, $op2 # long" %}
16456 ins_encode %{
16457 __ tst($op1$$Register, $op2$$Register);
16458 %}
16459 ins_pipe(ialu_reg_reg);
16460 %}
16461
16462 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16463 match(Set cr (CmpI (AndI op1 op2) op3));
16464
16465 ins_cost(INSN_COST);
16466 format %{ "tstw $op1, $op2 # int" %}
16467 ins_encode %{
16468 __ tstw($op1$$Register, $op2$$Register);
16469 %}
16470 ins_pipe(ialu_reg_reg);
16471 %}
16472
16473
16474 // Conditional Far Branch
16475 // Conditional Far Branch Unsigned
16476 // TODO: fixme
16477
16478 // counted loop end branch near
16479 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16480 %{
16481 match(CountedLoopEnd cmp cr);
16482
16483 effect(USE lbl);
16484
16485 ins_cost(BRANCH_COST);
16486 // short variant.
16487 // ins_short_branch(1);
16488 format %{ "b$cmp $lbl \t// counted loop end" %}
16489
16490 ins_encode(aarch64_enc_br_con(cmp, lbl));
16491
16492 ins_pipe(pipe_branch);
16493 %}
16494
16495 // counted loop end branch near Unsigned
16496 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16497 %{
16498 match(CountedLoopEnd cmp cr);
16499
16500 effect(USE lbl);
16501
16502 ins_cost(BRANCH_COST);
16503 // short variant.
16504 // ins_short_branch(1);
16505 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16506
16507 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16508
16509 ins_pipe(pipe_branch);
16510 %}
16511
16512 // counted loop end branch far
16513 // counted loop end branch far unsigned
16514 // TODO: fixme
16515
16516 // ============================================================================
16517 // inlined locking and unlocking
16518
16519 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16520 %{
16521 match(Set cr (FastLock object box));
16522 effect(TEMP tmp, TEMP tmp2);
16523
16524 // TODO
16525 // identify correct cost
16526 ins_cost(5 * INSN_COST);
16527 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16528
16529 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16530
16531 ins_pipe(pipe_serial);
16532 %}
16533
16534 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16535 %{
16536 match(Set cr (FastUnlock object box));
16537 effect(TEMP tmp, TEMP tmp2);
16538
16539 ins_cost(5 * INSN_COST);
16540 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16541
16542 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16543
16544 ins_pipe(pipe_serial);
16545 %}
16546
16547
16548 // ============================================================================
16549 // Safepoint Instructions
16550
16551 // TODO
16552 // provide a near and far version of this code
16553
16554 instruct safePoint(rFlagsReg cr, iRegP poll)
16555 %{
16556 match(SafePoint poll);
16557 effect(KILL cr);
16558
16559 format %{
16560 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16561 %}
16562 ins_encode %{
16563 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16564 %}
16565 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16566 %}
16567
16568
16569 // ============================================================================
16570 // Procedure Call/Return Instructions
16571
16572 // Call Java Static Instruction
16573
16574 instruct CallStaticJavaDirect(method meth)
16575 %{
16576 match(CallStaticJava);
16577
16578 effect(USE meth);
16579
16580 ins_cost(CALL_COST);
16581
16582 format %{ "call,static $meth \t// ==> " %}
16583
16584 ins_encode(aarch64_enc_java_static_call(meth),
16585 aarch64_enc_call_epilog);
16586
16587 ins_pipe(pipe_class_call);
16588 %}
16589
16590 // TO HERE
16591
16592 // Call Java Dynamic Instruction
16593 instruct CallDynamicJavaDirect(method meth)
16594 %{
16595 match(CallDynamicJava);
16596
16597 effect(USE meth);
16598
16599 ins_cost(CALL_COST);
16600
16601 format %{ "CALL,dynamic $meth \t// ==> " %}
16602
16603 ins_encode(aarch64_enc_java_dynamic_call(meth),
16604 aarch64_enc_call_epilog);
16605
16606 ins_pipe(pipe_class_call);
16607 %}
16608
16609 // Call Runtime Instruction
16610
16611 instruct CallRuntimeDirect(method meth)
16612 %{
16613 match(CallRuntime);
16614
16615 effect(USE meth);
16616
16617 ins_cost(CALL_COST);
16618
16619 format %{ "CALL, runtime $meth" %}
16620
16621 ins_encode( aarch64_enc_java_to_runtime(meth) );
16622
16623 ins_pipe(pipe_class_call);
16624 %}
16625
16626 // Call Runtime Instruction
16627
16628 instruct CallLeafDirect(method meth)
16629 %{
16630 match(CallLeaf);
16631
16632 effect(USE meth);
16633
16634 ins_cost(CALL_COST);
16635
16636 format %{ "CALL, runtime leaf $meth" %}
16637
16638 ins_encode( aarch64_enc_java_to_runtime(meth) );
16639
16640 ins_pipe(pipe_class_call);
16641 %}
16642
16643 // Call Runtime Instruction
16644
16645 instruct CallLeafNoFPDirect(method meth)
16646 %{
16647 match(CallLeafNoFP);
16648
16649 effect(USE meth);
16650
16651 ins_cost(CALL_COST);
16652
16653 format %{ "CALL, runtime leaf nofp $meth" %}
16654
16655 ins_encode( aarch64_enc_java_to_runtime(meth) );
16656
16657 ins_pipe(pipe_class_call);
16658 %}
16659
16660 // Tail Call; Jump from runtime stub to Java code.
16661 // Also known as an 'interprocedural jump'.
16662 // Target of jump will eventually return to caller.
16663 // TailJump below removes the return address.
16664 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16665 %{
16666 match(TailCall jump_target method_ptr);
16667
16668 ins_cost(CALL_COST);
16669
16670 format %{ "br $jump_target\t# $method_ptr holds method" %}
16671
16672 ins_encode(aarch64_enc_tail_call(jump_target));
16673
16674 ins_pipe(pipe_class_call);
16675 %}
16676
16677 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16678 %{
16679 match(TailJump jump_target ex_oop);
16680
16681 ins_cost(CALL_COST);
16682
16683 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16684
16685 ins_encode(aarch64_enc_tail_jmp(jump_target));
16686
16687 ins_pipe(pipe_class_call);
16688 %}
16689
16690 // Create exception oop: created by stack-crawling runtime code.
16691 // Created exception is now available to this handler, and is setup
16692 // just prior to jumping to this handler. No code emitted.
16693 // TODO check
16694 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16695 instruct CreateException(iRegP_R0 ex_oop)
16696 %{
16697 match(Set ex_oop (CreateEx));
16698
16699 format %{ " -- \t// exception oop; no code emitted" %}
16700
16701 size(0);
16702
16703 ins_encode( /*empty*/ );
16704
16705 ins_pipe(pipe_class_empty);
16706 %}
16707
16708 // Rethrow exception: The exception oop will come in the first
16709 // argument position. Then JUMP (not call) to the rethrow stub code.
16710 instruct RethrowException() %{
16711 match(Rethrow);
16712 ins_cost(CALL_COST);
16713
16714 format %{ "b rethrow_stub" %}
16715
16716 ins_encode( aarch64_enc_rethrow() );
16717
16718 ins_pipe(pipe_class_call);
16719 %}
16720
16721
16722 // Return Instruction
16723 // epilog node loads ret address into lr as part of frame pop
16724 instruct Ret()
16725 %{
16726 match(Return);
16727
16728 format %{ "ret\t// return register" %}
16729
16730 ins_encode( aarch64_enc_ret() );
16731
16732 ins_pipe(pipe_branch);
16733 %}
16734
16735 // Die now.
16736 instruct ShouldNotReachHere() %{
16737 match(Halt);
16738
16739 ins_cost(CALL_COST);
16740 format %{ "ShouldNotReachHere" %}
16741
16742 ins_encode %{
16743 if (is_reachable()) {
16744 __ stop(_halt_reason);
16745 }
16746 %}
16747
16748 ins_pipe(pipe_class_default);
16749 %}
16750
16751 // ============================================================================
16752 // Partial Subtype Check
16753 //
16754 // superklass array for an instance of the superklass. Set a hidden
16755 // internal cache on a hit (cache is checked with exposed code in
16756 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16757 // encoding ALSO sets flags.
16758
16759 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16760 %{
16761 match(Set result (PartialSubtypeCheck sub super));
16762 effect(KILL cr, KILL temp);
16763
16764 ins_cost(1100); // slightly larger than the next version
16765 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16766
16767 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16768
16769 opcode(0x1); // Force zero of result reg on hit
16770
16771 ins_pipe(pipe_class_memory);
16772 %}
16773
16774 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16775 %{
16776 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16777 effect(KILL temp, KILL result);
16778
16779 ins_cost(1100); // slightly larger than the next version
16780 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16781
16782 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16783
16784 opcode(0x0); // Don't zero result reg on hit
16785
16786 ins_pipe(pipe_class_memory);
16787 %}
16788
16789 // Intrisics for String.compareTo()
16790
16791 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16792 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16793 %{
16794 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16795 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16796 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16797
16798 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16799 ins_encode %{
16800 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16801 __ string_compare($str1$$Register, $str2$$Register,
16802 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16803 $tmp1$$Register, $tmp2$$Register,
16804 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16805 %}
16806 ins_pipe(pipe_class_memory);
16807 %}
16808
16809 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16810 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16811 %{
16812 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16813 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16814 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16815
16816 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16817 ins_encode %{
16818 __ string_compare($str1$$Register, $str2$$Register,
16819 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16820 $tmp1$$Register, $tmp2$$Register,
16821 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16822 %}
16823 ins_pipe(pipe_class_memory);
16824 %}
16825
16826 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16827 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16828 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16829 %{
16830 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16831 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16832 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16833 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16834
16835 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16836 ins_encode %{
16837 __ string_compare($str1$$Register, $str2$$Register,
16838 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16839 $tmp1$$Register, $tmp2$$Register,
16840 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16841 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16842 %}
16843 ins_pipe(pipe_class_memory);
16844 %}
16845
16846 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16847 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16848 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16849 %{
16850 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16851 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16852 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16853 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16854
16855 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16856 ins_encode %{
16857 __ string_compare($str1$$Register, $str2$$Register,
16858 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16859 $tmp1$$Register, $tmp2$$Register,
16860 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16861 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16862 %}
16863 ins_pipe(pipe_class_memory);
16864 %}
16865
16866 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16867 // these string_compare variants as NEON register type for convenience so that the prototype of
16868 // string_compare can be shared with all variants.
16869
16870 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16871 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16872 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16873 pRegGov_P1 pgtmp2, rFlagsReg cr)
16874 %{
16875 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16876 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16877 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16878 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16879
16880 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16881 ins_encode %{
16882 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16883 __ string_compare($str1$$Register, $str2$$Register,
16884 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16885 $tmp1$$Register, $tmp2$$Register,
16886 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16887 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16888 StrIntrinsicNode::LL);
16889 %}
16890 ins_pipe(pipe_class_memory);
16891 %}
16892
16893 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16894 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16895 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16896 pRegGov_P1 pgtmp2, rFlagsReg cr)
16897 %{
16898 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16899 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16900 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16901 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16902
16903 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16904 ins_encode %{
16905 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16906 __ string_compare($str1$$Register, $str2$$Register,
16907 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16908 $tmp1$$Register, $tmp2$$Register,
16909 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16910 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16911 StrIntrinsicNode::LU);
16912 %}
16913 ins_pipe(pipe_class_memory);
16914 %}
16915
16916 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16917 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16918 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16919 pRegGov_P1 pgtmp2, rFlagsReg cr)
16920 %{
16921 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16922 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16923 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16924 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16925
16926 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16927 ins_encode %{
16928 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16929 __ string_compare($str1$$Register, $str2$$Register,
16930 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16931 $tmp1$$Register, $tmp2$$Register,
16932 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16933 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16934 StrIntrinsicNode::UL);
16935 %}
16936 ins_pipe(pipe_class_memory);
16937 %}
16938
16939 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16940 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16941 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16942 pRegGov_P1 pgtmp2, rFlagsReg cr)
16943 %{
16944 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16945 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16946 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16947 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16948
16949 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16950 ins_encode %{
16951 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16952 __ string_compare($str1$$Register, $str2$$Register,
16953 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16954 $tmp1$$Register, $tmp2$$Register,
16955 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16956 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16957 StrIntrinsicNode::UU);
16958 %}
16959 ins_pipe(pipe_class_memory);
16960 %}
16961
16962 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16963 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16964 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16965 %{
16966 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16967 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16968 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16969 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16970 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16971
16972 ins_encode %{
16973 __ string_indexof($str1$$Register, $str2$$Register,
16974 $cnt1$$Register, $cnt2$$Register,
16975 $tmp1$$Register, $tmp2$$Register,
16976 $tmp3$$Register, $tmp4$$Register,
16977 $tmp5$$Register, $tmp6$$Register,
16978 -1, $result$$Register, StrIntrinsicNode::UU);
16979 %}
16980 ins_pipe(pipe_class_memory);
16981 %}
16982
16983 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16984 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16985 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16986 %{
16987 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16988 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16989 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16990 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16991 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16992
16993 ins_encode %{
16994 __ string_indexof($str1$$Register, $str2$$Register,
16995 $cnt1$$Register, $cnt2$$Register,
16996 $tmp1$$Register, $tmp2$$Register,
16997 $tmp3$$Register, $tmp4$$Register,
16998 $tmp5$$Register, $tmp6$$Register,
16999 -1, $result$$Register, StrIntrinsicNode::LL);
17000 %}
17001 ins_pipe(pipe_class_memory);
17002 %}
17003
17004 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17005 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17006 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17007 %{
17008 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17009 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17010 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17011 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17012 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17013
17014 ins_encode %{
17015 __ string_indexof($str1$$Register, $str2$$Register,
17016 $cnt1$$Register, $cnt2$$Register,
17017 $tmp1$$Register, $tmp2$$Register,
17018 $tmp3$$Register, $tmp4$$Register,
17019 $tmp5$$Register, $tmp6$$Register,
17020 -1, $result$$Register, StrIntrinsicNode::UL);
17021 %}
17022 ins_pipe(pipe_class_memory);
17023 %}
17024
17025 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17026 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17027 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17028 %{
17029 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17030 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17031 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17032 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17033 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17034
17035 ins_encode %{
17036 int icnt2 = (int)$int_cnt2$$constant;
17037 __ string_indexof($str1$$Register, $str2$$Register,
17038 $cnt1$$Register, zr,
17039 $tmp1$$Register, $tmp2$$Register,
17040 $tmp3$$Register, $tmp4$$Register, zr, zr,
17041 icnt2, $result$$Register, StrIntrinsicNode::UU);
17042 %}
17043 ins_pipe(pipe_class_memory);
17044 %}
17045
17046 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17047 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17048 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17049 %{
17050 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17051 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17052 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17053 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17054 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17055
17056 ins_encode %{
17057 int icnt2 = (int)$int_cnt2$$constant;
17058 __ string_indexof($str1$$Register, $str2$$Register,
17059 $cnt1$$Register, zr,
17060 $tmp1$$Register, $tmp2$$Register,
17061 $tmp3$$Register, $tmp4$$Register, zr, zr,
17062 icnt2, $result$$Register, StrIntrinsicNode::LL);
17063 %}
17064 ins_pipe(pipe_class_memory);
17065 %}
17066
17067 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17068 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17069 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17070 %{
17071 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17072 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17073 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17074 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17075 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17076
17077 ins_encode %{
17078 int icnt2 = (int)$int_cnt2$$constant;
17079 __ string_indexof($str1$$Register, $str2$$Register,
17080 $cnt1$$Register, zr,
17081 $tmp1$$Register, $tmp2$$Register,
17082 $tmp3$$Register, $tmp4$$Register, zr, zr,
17083 icnt2, $result$$Register, StrIntrinsicNode::UL);
17084 %}
17085 ins_pipe(pipe_class_memory);
17086 %}
17087
17088 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17089 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17090 iRegINoSp tmp3, rFlagsReg cr)
17091 %{
17092 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17093 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17094 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17095 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17096
17097 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17098
17099 ins_encode %{
17100 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17101 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17102 $tmp3$$Register);
17103 %}
17104 ins_pipe(pipe_class_memory);
17105 %}
17106
17107 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17108 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17109 iRegINoSp tmp3, rFlagsReg cr)
17110 %{
17111 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17112 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17113 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17114 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17115
17116 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17117
17118 ins_encode %{
17119 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17120 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17121 $tmp3$$Register);
17122 %}
17123 ins_pipe(pipe_class_memory);
17124 %}
17125
17126 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17127 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17128 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17129 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17130 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17131 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17132 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17133 ins_encode %{
17134 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17135 $result$$Register, $ztmp1$$FloatRegister,
17136 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17137 $ptmp$$PRegister, true /* isL */);
17138 %}
17139 ins_pipe(pipe_class_memory);
17140 %}
17141
17142 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17143 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17144 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17145 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17146 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17147 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17148 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17149 ins_encode %{
17150 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17151 $result$$Register, $ztmp1$$FloatRegister,
17152 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17153 $ptmp$$PRegister, false /* isL */);
17154 %}
17155 ins_pipe(pipe_class_memory);
17156 %}
17157
17158 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17159 iRegI_R0 result, rFlagsReg cr)
17160 %{
17161 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17162 match(Set result (StrEquals (Binary str1 str2) cnt));
17163 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17164
17165 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17166 ins_encode %{
17167 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17168 __ string_equals($str1$$Register, $str2$$Register,
17169 $result$$Register, $cnt$$Register, 1);
17170 %}
17171 ins_pipe(pipe_class_memory);
17172 %}
17173
17174 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17175 iRegI_R0 result, rFlagsReg cr)
17176 %{
17177 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17178 match(Set result (StrEquals (Binary str1 str2) cnt));
17179 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17180
17181 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17182 ins_encode %{
17183 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17184 __ string_equals($str1$$Register, $str2$$Register,
17185 $result$$Register, $cnt$$Register, 2);
17186 %}
17187 ins_pipe(pipe_class_memory);
17188 %}
17189
17190 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17191 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17192 iRegP_R10 tmp, rFlagsReg cr)
17193 %{
17194 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17195 match(Set result (AryEq ary1 ary2));
17196 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17197
17198 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17199 ins_encode %{
17200 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17201 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17202 $result$$Register, $tmp$$Register, 1);
17203 if (tpc == NULL) {
17204 ciEnv::current()->record_failure("CodeCache is full");
17205 return;
17206 }
17207 %}
17208 ins_pipe(pipe_class_memory);
17209 %}
17210
17211 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17212 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17213 iRegP_R10 tmp, rFlagsReg cr)
17214 %{
17215 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17216 match(Set result (AryEq ary1 ary2));
17217 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17218
17219 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17220 ins_encode %{
17221 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17222 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17223 $result$$Register, $tmp$$Register, 2);
17224 if (tpc == NULL) {
17225 ciEnv::current()->record_failure("CodeCache is full");
17226 return;
17227 }
17228 %}
17229 ins_pipe(pipe_class_memory);
17230 %}
17231
17232 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17233 %{
17234 match(Set result (CountPositives ary1 len));
17235 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17236 format %{ "count positives byte[] $ary1,$len -> $result" %}
17237 ins_encode %{
17238 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17239 if (tpc == NULL) {
17240 ciEnv::current()->record_failure("CodeCache is full");
17241 return;
17242 }
17243 %}
17244 ins_pipe( pipe_slow );
17245 %}
17246
17247 // fast char[] to byte[] compression
17248 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17249 vRegD_V0 tmp1, vRegD_V1 tmp2,
17250 vRegD_V2 tmp3, vRegD_V3 tmp4,
17251 iRegI_R0 result, rFlagsReg cr)
17252 %{
17253 match(Set result (StrCompressedCopy src (Binary dst len)));
17254 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17255 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17256
17257 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
17258 ins_encode %{
17259 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17260 $result$$Register,
17261 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17262 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17263 %}
17264 ins_pipe(pipe_slow);
17265 %}
17266
17267 // fast byte[] to char[] inflation
17268 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17269 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17270 %{
17271 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17272 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17273
17274 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
17275 ins_encode %{
17276 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17277 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17278 $tmp3$$FloatRegister, $tmp4$$Register);
17279 if (tpc == NULL) {
17280 ciEnv::current()->record_failure("CodeCache is full");
17281 return;
17282 }
17283 %}
17284 ins_pipe(pipe_class_memory);
17285 %}
17286
17287 // encode char[] to byte[] in ISO_8859_1
17288 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17289 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17290 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17291 iRegI_R0 result, rFlagsReg cr)
17292 %{
17293 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17294 match(Set result (EncodeISOArray src (Binary dst len)));
17295 effect(USE_KILL src, USE_KILL dst, USE len,
17296 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17297
17298 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17299 ins_encode %{
17300 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17301 $result$$Register, false,
17302 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17303 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17304 %}
17305 ins_pipe(pipe_class_memory);
17306 %}
17307
17308 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17309 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17310 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17311 iRegI_R0 result, rFlagsReg cr)
17312 %{
17313 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17314 match(Set result (EncodeISOArray src (Binary dst len)));
17315 effect(USE_KILL src, USE_KILL dst, USE len,
17316 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17317
17318 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17319 ins_encode %{
17320 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17321 $result$$Register, true,
17322 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17323 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17324 %}
17325 ins_pipe(pipe_class_memory);
17326 %}
17327
17328 // ============================================================================
17329 // This name is KNOWN by the ADLC and cannot be changed.
17330 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17331 // for this guy.
17332 instruct tlsLoadP(thread_RegP dst)
17333 %{
17334 match(Set dst (ThreadLocal));
17335
17336 ins_cost(0);
17337
17338 format %{ " -- \t// $dst=Thread::current(), empty" %}
17339
17340 size(0);
17341
17342 ins_encode( /*empty*/ );
17343
17344 ins_pipe(pipe_class_empty);
17345 %}
17346
17347 //----------PEEPHOLE RULES-----------------------------------------------------
17348 // These must follow all instruction definitions as they use the names
17349 // defined in the instructions definitions.
17350 //
17351 // peepmatch ( root_instr_name [preceding_instruction]* );
17352 //
17353 // peepconstraint %{
17354 // (instruction_number.operand_name relational_op instruction_number.operand_name
17355 // [, ...] );
17356 // // instruction numbers are zero-based using left to right order in peepmatch
17357 //
17358 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17359 // // provide an instruction_number.operand_name for each operand that appears
17360 // // in the replacement instruction's match rule
17361 //
17362 // ---------VM FLAGS---------------------------------------------------------
17363 //
17364 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17365 //
17366 // Each peephole rule is given an identifying number starting with zero and
17367 // increasing by one in the order seen by the parser. An individual peephole
17368 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17369 // on the command-line.
17370 //
17371 // ---------CURRENT LIMITATIONS----------------------------------------------
17372 //
17373 // Only match adjacent instructions in same basic block
17374 // Only equality constraints
17375 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17376 // Only one replacement instruction
17377 //
17378 // ---------EXAMPLE----------------------------------------------------------
17379 //
17380 // // pertinent parts of existing instructions in architecture description
17381 // instruct movI(iRegINoSp dst, iRegI src)
17382 // %{
17383 // match(Set dst (CopyI src));
17384 // %}
17385 //
17386 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17387 // %{
17388 // match(Set dst (AddI dst src));
17389 // effect(KILL cr);
17390 // %}
17391 //
17392 // // Change (inc mov) to lea
17393 // peephole %{
17394 // // increment preceded by register-register move
17395 // peepmatch ( incI_iReg movI );
17396 // // require that the destination register of the increment
17397 // // match the destination register of the move
17398 // peepconstraint ( 0.dst == 1.dst );
17399 // // construct a replacement instruction that sets
17400 // // the destination to ( move's source register + one )
17401 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17402 // %}
17403 //
17404
17405 // Implementation no longer uses movX instructions since
17406 // machine-independent system no longer uses CopyX nodes.
17407 //
17408 // peephole
17409 // %{
17410 // peepmatch (incI_iReg movI);
17411 // peepconstraint (0.dst == 1.dst);
17412 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17413 // %}
17414
17415 // peephole
17416 // %{
17417 // peepmatch (decI_iReg movI);
17418 // peepconstraint (0.dst == 1.dst);
17419 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17420 // %}
17421
17422 // peephole
17423 // %{
17424 // peepmatch (addI_iReg_imm movI);
17425 // peepconstraint (0.dst == 1.dst);
17426 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17427 // %}
17428
17429 // peephole
17430 // %{
17431 // peepmatch (incL_iReg movL);
17432 // peepconstraint (0.dst == 1.dst);
17433 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17434 // %}
17435
17436 // peephole
17437 // %{
17438 // peepmatch (decL_iReg movL);
17439 // peepconstraint (0.dst == 1.dst);
17440 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17441 // %}
17442
17443 // peephole
17444 // %{
17445 // peepmatch (addL_iReg_imm movL);
17446 // peepconstraint (0.dst == 1.dst);
17447 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17448 // %}
17449
17450 // peephole
17451 // %{
17452 // peepmatch (addP_iReg_imm movP);
17453 // peepconstraint (0.dst == 1.dst);
17454 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17455 // %}
17456
17457 // // Change load of spilled value to only a spill
17458 // instruct storeI(memory mem, iRegI src)
17459 // %{
17460 // match(Set mem (StoreI mem src));
17461 // %}
17462 //
17463 // instruct loadI(iRegINoSp dst, memory mem)
17464 // %{
17465 // match(Set dst (LoadI mem));
17466 // %}
17467 //
17468
17469 //----------SMARTSPILL RULES---------------------------------------------------
17470 // These must follow all instruction definitions as they use the names
17471 // defined in the instructions definitions.
17472
17473 // Local Variables:
17474 // mode: c++
17475 // End:
--- EOF ---