1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all float registers
698 reg_class float_reg(
699 V0,
700 V1,
701 V2,
702 V3,
703 V4,
704 V5,
705 V6,
706 V7,
707 V8,
708 V9,
709 V10,
710 V11,
711 V12,
712 V13,
713 V14,
714 V15,
715 V16,
716 V17,
717 V18,
718 V19,
719 V20,
720 V21,
721 V22,
722 V23,
723 V24,
724 V25,
725 V26,
726 V27,
727 V28,
728 V29,
729 V30,
730 V31
731 );
732
733 // Double precision float registers have virtual `high halves' that
734 // are needed by the allocator.
735 // Class for all double registers
736 reg_class double_reg(
737 V0, V0_H,
738 V1, V1_H,
739 V2, V2_H,
740 V3, V3_H,
741 V4, V4_H,
742 V5, V5_H,
743 V6, V6_H,
744 V7, V7_H,
745 V8, V8_H,
746 V9, V9_H,
747 V10, V10_H,
748 V11, V11_H,
749 V12, V12_H,
750 V13, V13_H,
751 V14, V14_H,
752 V15, V15_H,
753 V16, V16_H,
754 V17, V17_H,
755 V18, V18_H,
756 V19, V19_H,
757 V20, V20_H,
758 V21, V21_H,
759 V22, V22_H,
760 V23, V23_H,
761 V24, V24_H,
762 V25, V25_H,
763 V26, V26_H,
764 V27, V27_H,
765 V28, V28_H,
766 V29, V29_H,
767 V30, V30_H,
768 V31, V31_H
769 );
770
771 // Class for all SVE vector registers.
772 reg_class vectora_reg (
773 V0, V0_H, V0_J, V0_K,
774 V1, V1_H, V1_J, V1_K,
775 V2, V2_H, V2_J, V2_K,
776 V3, V3_H, V3_J, V3_K,
777 V4, V4_H, V4_J, V4_K,
778 V5, V5_H, V5_J, V5_K,
779 V6, V6_H, V6_J, V6_K,
780 V7, V7_H, V7_J, V7_K,
781 V8, V8_H, V8_J, V8_K,
782 V9, V9_H, V9_J, V9_K,
783 V10, V10_H, V10_J, V10_K,
784 V11, V11_H, V11_J, V11_K,
785 V12, V12_H, V12_J, V12_K,
786 V13, V13_H, V13_J, V13_K,
787 V14, V14_H, V14_J, V14_K,
788 V15, V15_H, V15_J, V15_K,
789 V16, V16_H, V16_J, V16_K,
790 V17, V17_H, V17_J, V17_K,
791 V18, V18_H, V18_J, V18_K,
792 V19, V19_H, V19_J, V19_K,
793 V20, V20_H, V20_J, V20_K,
794 V21, V21_H, V21_J, V21_K,
795 V22, V22_H, V22_J, V22_K,
796 V23, V23_H, V23_J, V23_K,
797 V24, V24_H, V24_J, V24_K,
798 V25, V25_H, V25_J, V25_K,
799 V26, V26_H, V26_J, V26_K,
800 V27, V27_H, V27_J, V27_K,
801 V28, V28_H, V28_J, V28_K,
802 V29, V29_H, V29_J, V29_K,
803 V30, V30_H, V30_J, V30_K,
804 V31, V31_H, V31_J, V31_K,
805 );
806
807 // Class for all 64bit vector registers
808 reg_class vectord_reg(
809 V0, V0_H,
810 V1, V1_H,
811 V2, V2_H,
812 V3, V3_H,
813 V4, V4_H,
814 V5, V5_H,
815 V6, V6_H,
816 V7, V7_H,
817 V8, V8_H,
818 V9, V9_H,
819 V10, V10_H,
820 V11, V11_H,
821 V12, V12_H,
822 V13, V13_H,
823 V14, V14_H,
824 V15, V15_H,
825 V16, V16_H,
826 V17, V17_H,
827 V18, V18_H,
828 V19, V19_H,
829 V20, V20_H,
830 V21, V21_H,
831 V22, V22_H,
832 V23, V23_H,
833 V24, V24_H,
834 V25, V25_H,
835 V26, V26_H,
836 V27, V27_H,
837 V28, V28_H,
838 V29, V29_H,
839 V30, V30_H,
840 V31, V31_H
841 );
842
843 // Class for all 128bit vector registers
844 reg_class vectorx_reg(
845 V0, V0_H, V0_J, V0_K,
846 V1, V1_H, V1_J, V1_K,
847 V2, V2_H, V2_J, V2_K,
848 V3, V3_H, V3_J, V3_K,
849 V4, V4_H, V4_J, V4_K,
850 V5, V5_H, V5_J, V5_K,
851 V6, V6_H, V6_J, V6_K,
852 V7, V7_H, V7_J, V7_K,
853 V8, V8_H, V8_J, V8_K,
854 V9, V9_H, V9_J, V9_K,
855 V10, V10_H, V10_J, V10_K,
856 V11, V11_H, V11_J, V11_K,
857 V12, V12_H, V12_J, V12_K,
858 V13, V13_H, V13_J, V13_K,
859 V14, V14_H, V14_J, V14_K,
860 V15, V15_H, V15_J, V15_K,
861 V16, V16_H, V16_J, V16_K,
862 V17, V17_H, V17_J, V17_K,
863 V18, V18_H, V18_J, V18_K,
864 V19, V19_H, V19_J, V19_K,
865 V20, V20_H, V20_J, V20_K,
866 V21, V21_H, V21_J, V21_K,
867 V22, V22_H, V22_J, V22_K,
868 V23, V23_H, V23_J, V23_K,
869 V24, V24_H, V24_J, V24_K,
870 V25, V25_H, V25_J, V25_K,
871 V26, V26_H, V26_J, V26_K,
872 V27, V27_H, V27_J, V27_K,
873 V28, V28_H, V28_J, V28_K,
874 V29, V29_H, V29_J, V29_K,
875 V30, V30_H, V30_J, V30_K,
876 V31, V31_H, V31_J, V31_K
877 );
878
879 // Class for 128 bit register v0
880 reg_class v0_reg(
881 V0, V0_H
882 );
883
884 // Class for 128 bit register v1
885 reg_class v1_reg(
886 V1, V1_H
887 );
888
889 // Class for 128 bit register v2
890 reg_class v2_reg(
891 V2, V2_H
892 );
893
894 // Class for 128 bit register v3
895 reg_class v3_reg(
896 V3, V3_H
897 );
898
899 // Class for 128 bit register v4
900 reg_class v4_reg(
901 V4, V4_H
902 );
903
904 // Class for 128 bit register v5
905 reg_class v5_reg(
906 V5, V5_H
907 );
908
909 // Class for 128 bit register v6
910 reg_class v6_reg(
911 V6, V6_H
912 );
913
914 // Class for 128 bit register v7
915 reg_class v7_reg(
916 V7, V7_H
917 );
918
919 // Class for 128 bit register v8
920 reg_class v8_reg(
921 V8, V8_H
922 );
923
924 // Class for 128 bit register v9
925 reg_class v9_reg(
926 V9, V9_H
927 );
928
929 // Class for 128 bit register v10
930 reg_class v10_reg(
931 V10, V10_H
932 );
933
934 // Class for 128 bit register v11
935 reg_class v11_reg(
936 V11, V11_H
937 );
938
939 // Class for 128 bit register v12
940 reg_class v12_reg(
941 V12, V12_H
942 );
943
944 // Class for 128 bit register v13
945 reg_class v13_reg(
946 V13, V13_H
947 );
948
949 // Class for 128 bit register v14
950 reg_class v14_reg(
951 V14, V14_H
952 );
953
954 // Class for 128 bit register v15
955 reg_class v15_reg(
956 V15, V15_H
957 );
958
959 // Class for 128 bit register v16
960 reg_class v16_reg(
961 V16, V16_H
962 );
963
964 // Class for 128 bit register v17
965 reg_class v17_reg(
966 V17, V17_H
967 );
968
969 // Class for 128 bit register v18
970 reg_class v18_reg(
971 V18, V18_H
972 );
973
974 // Class for 128 bit register v19
975 reg_class v19_reg(
976 V19, V19_H
977 );
978
979 // Class for 128 bit register v20
980 reg_class v20_reg(
981 V20, V20_H
982 );
983
984 // Class for 128 bit register v21
985 reg_class v21_reg(
986 V21, V21_H
987 );
988
989 // Class for 128 bit register v22
990 reg_class v22_reg(
991 V22, V22_H
992 );
993
994 // Class for 128 bit register v23
995 reg_class v23_reg(
996 V23, V23_H
997 );
998
999 // Class for 128 bit register v24
1000 reg_class v24_reg(
1001 V24, V24_H
1002 );
1003
1004 // Class for 128 bit register v25
1005 reg_class v25_reg(
1006 V25, V25_H
1007 );
1008
1009 // Class for 128 bit register v26
1010 reg_class v26_reg(
1011 V26, V26_H
1012 );
1013
1014 // Class for 128 bit register v27
1015 reg_class v27_reg(
1016 V27, V27_H
1017 );
1018
1019 // Class for 128 bit register v28
1020 reg_class v28_reg(
1021 V28, V28_H
1022 );
1023
1024 // Class for 128 bit register v29
1025 reg_class v29_reg(
1026 V29, V29_H
1027 );
1028
1029 // Class for 128 bit register v30
1030 reg_class v30_reg(
1031 V30, V30_H
1032 );
1033
1034 // Class for 128 bit register v31
1035 reg_class v31_reg(
1036 V31, V31_H
1037 );
1038
1039 // Class for all SVE predicate registers.
1040 reg_class pr_reg (
1041 P0,
1042 P1,
1043 P2,
1044 P3,
1045 P4,
1046 P5,
1047 P6,
1048 // P7, non-allocatable, preserved with all elements preset to TRUE.
1049 P8,
1050 P9,
1051 P10,
1052 P11,
1053 P12,
1054 P13,
1055 P14,
1056 P15
1057 );
1058
1059 // Class for SVE governing predicate registers, which are used
1060 // to determine the active elements of a predicated instruction.
1061 reg_class gov_pr (
1062 P0,
1063 P1,
1064 P2,
1065 P3,
1066 P4,
1067 P5,
1068 P6,
1069 // P7, non-allocatable, preserved with all elements preset to TRUE.
1070 );
1071
1072 reg_class p0_reg(P0);
1073 reg_class p1_reg(P1);
1074
1075 // Singleton class for condition codes
1076 reg_class int_flags(RFLAGS);
1077
1078 %}
1079
1080 //----------DEFINITION BLOCK---------------------------------------------------
1081 // Define name --> value mappings to inform the ADLC of an integer valued name
1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1083 // Format:
1084 // int_def <name> ( <int_value>, <expression>);
1085 // Generated Code in ad_<arch>.hpp
1086 // #define <name> (<expression>)
1087 // // value == <int_value>
1088 // Generated code in ad_<arch>.cpp adlc_verification()
1089 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1090 //
1091
1092 // we follow the ppc-aix port in using a simple cost model which ranks
1093 // register operations as cheap, memory ops as more expensive and
1094 // branches as most expensive. the first two have a low as well as a
1095 // normal cost. huge cost appears to be a way of saying don't do
1096 // something
1097
1098 definitions %{
1099 // The default cost (of a register move instruction).
1100 int_def INSN_COST ( 100, 100);
1101 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1102 int_def CALL_COST ( 200, 2 * INSN_COST);
1103 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1104 %}
1105
1106
1107 //----------SOURCE BLOCK-------------------------------------------------------
1108 // This is a block of C++ code which provides values, functions, and
1109 // definitions necessary in the rest of the architecture description
1110
1111 source_hpp %{
1112
1113 #include "asm/macroAssembler.hpp"
1114 #include "gc/shared/barrierSetAssembler.hpp"
1115 #include "gc/shared/cardTable.hpp"
1116 #include "gc/shared/cardTableBarrierSet.hpp"
1117 #include "gc/shared/collectedHeap.hpp"
1118 #include "opto/addnode.hpp"
1119 #include "opto/convertnode.hpp"
1120 #include "runtime/objectMonitor.hpp"
1121
1122 extern RegMask _ANY_REG32_mask;
1123 extern RegMask _ANY_REG_mask;
1124 extern RegMask _PTR_REG_mask;
1125 extern RegMask _NO_SPECIAL_REG32_mask;
1126 extern RegMask _NO_SPECIAL_REG_mask;
1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1128
1129 class CallStubImpl {
1130
1131 //--------------------------------------------------------------
1132 //---< Used for optimization in Compile::shorten_branches >---
1133 //--------------------------------------------------------------
1134
1135 public:
1136 // Size of call trampoline stub.
1137 static uint size_call_trampoline() {
1138 return 0; // no call trampolines on this platform
1139 }
1140
1141 // number of relocations needed by a call trampoline stub
1142 static uint reloc_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145 };
1146
1147 class HandlerImpl {
1148
1149 public:
1150
1151 static int emit_exception_handler(CodeBuffer &cbuf);
1152 static int emit_deopt_handler(CodeBuffer& cbuf);
1153
1154 static uint size_exception_handler() {
1155 return MacroAssembler::far_codestub_branch_size();
1156 }
1157
1158 static uint size_deopt_handler() {
1159 // count one adr and one far branch instruction
1160 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1161 }
1162 };
1163
1164 class Node::PD {
1165 public:
1166 enum NodeFlags {
1167 _last_flag = Node::_last_flag
1168 };
1169 };
1170
1171 bool is_CAS(int opcode, bool maybe_volatile);
1172
1173 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1174
1175 bool unnecessary_acquire(const Node *barrier);
1176 bool needs_acquiring_load(const Node *load);
1177
1178 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1179
1180 bool unnecessary_release(const Node *barrier);
1181 bool unnecessary_volatile(const Node *barrier);
1182 bool needs_releasing_store(const Node *store);
1183
1184 // predicate controlling translation of CompareAndSwapX
1185 bool needs_acquiring_load_exclusive(const Node *load);
1186
1187 // predicate controlling addressing modes
1188 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1189
1190 // Convert BootTest condition to Assembler condition.
1191 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1192 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1193 %}
1194
1195 source %{
1196
1197 // Derived RegMask with conditionally allocatable registers
1198
1199 void PhaseOutput::pd_perform_mach_node_analysis() {
1200 }
1201
1202 int MachNode::pd_alignment_required() const {
1203 return 1;
1204 }
1205
1206 int MachNode::compute_padding(int current_offset) const {
1207 return 0;
1208 }
1209
1210 RegMask _ANY_REG32_mask;
1211 RegMask _ANY_REG_mask;
1212 RegMask _PTR_REG_mask;
1213 RegMask _NO_SPECIAL_REG32_mask;
1214 RegMask _NO_SPECIAL_REG_mask;
1215 RegMask _NO_SPECIAL_PTR_REG_mask;
1216
1217 void reg_mask_init() {
1218 // We derive below RegMask(s) from the ones which are auto-generated from
1219 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1220 // registers conditionally reserved.
1221
1222 _ANY_REG32_mask = _ALL_REG32_mask;
1223 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1224
1225 _ANY_REG_mask = _ALL_REG_mask;
1226
1227 _PTR_REG_mask = _ALL_REG_mask;
1228
1229 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1230 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1231
1232 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1236 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1237
1238 // r27 is not allocatable when compressed oops is on and heapbase is not
1239 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1240 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1241 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1242 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1244 }
1245
1246 // r29 is not allocatable when PreserveFramePointer is on
1247 if (PreserveFramePointer) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1251 }
1252 }
1253
1254 // Optimizaton of volatile gets and puts
1255 // -------------------------------------
1256 //
1257 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1258 // use to implement volatile reads and writes. For a volatile read
1259 // we simply need
1260 //
1261 // ldar<x>
1262 //
1263 // and for a volatile write we need
1264 //
1265 // stlr<x>
1266 //
1267 // Alternatively, we can implement them by pairing a normal
1268 // load/store with a memory barrier. For a volatile read we need
1269 //
1270 // ldr<x>
1271 // dmb ishld
1272 //
1273 // for a volatile write
1274 //
1275 // dmb ish
1276 // str<x>
1277 // dmb ish
1278 //
1279 // We can also use ldaxr and stlxr to implement compare and swap CAS
1280 // sequences. These are normally translated to an instruction
1281 // sequence like the following
1282 //
1283 // dmb ish
1284 // retry:
1285 // ldxr<x> rval raddr
1286 // cmp rval rold
1287 // b.ne done
1288 // stlxr<x> rval, rnew, rold
1289 // cbnz rval retry
1290 // done:
1291 // cset r0, eq
1292 // dmb ishld
1293 //
1294 // Note that the exclusive store is already using an stlxr
1295 // instruction. That is required to ensure visibility to other
1296 // threads of the exclusive write (assuming it succeeds) before that
1297 // of any subsequent writes.
1298 //
1299 // The following instruction sequence is an improvement on the above
1300 //
1301 // retry:
1302 // ldaxr<x> rval raddr
1303 // cmp rval rold
1304 // b.ne done
1305 // stlxr<x> rval, rnew, rold
1306 // cbnz rval retry
1307 // done:
1308 // cset r0, eq
1309 //
1310 // We don't need the leading dmb ish since the stlxr guarantees
1311 // visibility of prior writes in the case that the swap is
1312 // successful. Crucially we don't have to worry about the case where
1313 // the swap is not successful since no valid program should be
1314 // relying on visibility of prior changes by the attempting thread
1315 // in the case where the CAS fails.
1316 //
1317 // Similarly, we don't need the trailing dmb ishld if we substitute
1318 // an ldaxr instruction since that will provide all the guarantees we
1319 // require regarding observation of changes made by other threads
1320 // before any change to the CAS address observed by the load.
1321 //
1322 // In order to generate the desired instruction sequence we need to
1323 // be able to identify specific 'signature' ideal graph node
1324 // sequences which i) occur as a translation of a volatile reads or
1325 // writes or CAS operations and ii) do not occur through any other
1326 // translation or graph transformation. We can then provide
1327 // alternative aldc matching rules which translate these node
1328 // sequences to the desired machine code sequences. Selection of the
1329 // alternative rules can be implemented by predicates which identify
1330 // the relevant node sequences.
1331 //
1332 // The ideal graph generator translates a volatile read to the node
1333 // sequence
1334 //
1335 // LoadX[mo_acquire]
1336 // MemBarAcquire
1337 //
1338 // As a special case when using the compressed oops optimization we
1339 // may also see this variant
1340 //
1341 // LoadN[mo_acquire]
1342 // DecodeN
1343 // MemBarAcquire
1344 //
1345 // A volatile write is translated to the node sequence
1346 //
1347 // MemBarRelease
1348 // StoreX[mo_release] {CardMark}-optional
1349 // MemBarVolatile
1350 //
1351 // n.b. the above node patterns are generated with a strict
1352 // 'signature' configuration of input and output dependencies (see
1353 // the predicates below for exact details). The card mark may be as
1354 // simple as a few extra nodes or, in a few GC configurations, may
1355 // include more complex control flow between the leading and
1356 // trailing memory barriers. However, whatever the card mark
1357 // configuration these signatures are unique to translated volatile
1358 // reads/stores -- they will not appear as a result of any other
1359 // bytecode translation or inlining nor as a consequence of
1360 // optimizing transforms.
1361 //
1362 // We also want to catch inlined unsafe volatile gets and puts and
1363 // be able to implement them using either ldar<x>/stlr<x> or some
1364 // combination of ldr<x>/stlr<x> and dmb instructions.
1365 //
1366 // Inlined unsafe volatiles puts manifest as a minor variant of the
1367 // normal volatile put node sequence containing an extra cpuorder
1368 // membar
1369 //
1370 // MemBarRelease
1371 // MemBarCPUOrder
1372 // StoreX[mo_release] {CardMark}-optional
1373 // MemBarCPUOrder
1374 // MemBarVolatile
1375 //
1376 // n.b. as an aside, a cpuorder membar is not itself subject to
1377 // matching and translation by adlc rules. However, the rule
1378 // predicates need to detect its presence in order to correctly
1379 // select the desired adlc rules.
1380 //
1381 // Inlined unsafe volatile gets manifest as a slightly different
1382 // node sequence to a normal volatile get because of the
1383 // introduction of some CPUOrder memory barriers to bracket the
1384 // Load. However, but the same basic skeleton of a LoadX feeding a
1385 // MemBarAcquire, possibly through an optional DecodeN, is still
1386 // present
1387 //
1388 // MemBarCPUOrder
1389 // || \\
1390 // MemBarCPUOrder LoadX[mo_acquire]
1391 // || |
1392 // || {DecodeN} optional
1393 // || /
1394 // MemBarAcquire
1395 //
1396 // In this case the acquire membar does not directly depend on the
1397 // load. However, we can be sure that the load is generated from an
1398 // inlined unsafe volatile get if we see it dependent on this unique
1399 // sequence of membar nodes. Similarly, given an acquire membar we
1400 // can know that it was added because of an inlined unsafe volatile
1401 // get if it is fed and feeds a cpuorder membar and if its feed
1402 // membar also feeds an acquiring load.
1403 //
1404 // Finally an inlined (Unsafe) CAS operation is translated to the
1405 // following ideal graph
1406 //
1407 // MemBarRelease
1408 // MemBarCPUOrder
1409 // CompareAndSwapX {CardMark}-optional
1410 // MemBarCPUOrder
1411 // MemBarAcquire
1412 //
1413 // So, where we can identify these volatile read and write
1414 // signatures we can choose to plant either of the above two code
1415 // sequences. For a volatile read we can simply plant a normal
1416 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1417 // also choose to inhibit translation of the MemBarAcquire and
1418 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1419 //
1420 // When we recognise a volatile store signature we can choose to
1421 // plant at a dmb ish as a translation for the MemBarRelease, a
1422 // normal str<x> and then a dmb ish for the MemBarVolatile.
1423 // Alternatively, we can inhibit translation of the MemBarRelease
1424 // and MemBarVolatile and instead plant a simple stlr<x>
1425 // instruction.
1426 //
1427 // when we recognise a CAS signature we can choose to plant a dmb
1428 // ish as a translation for the MemBarRelease, the conventional
1429 // macro-instruction sequence for the CompareAndSwap node (which
1430 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1431 // Alternatively, we can elide generation of the dmb instructions
1432 // and plant the alternative CompareAndSwap macro-instruction
1433 // sequence (which uses ldaxr<x>).
1434 //
1435 // Of course, the above only applies when we see these signature
1436 // configurations. We still want to plant dmb instructions in any
1437 // other cases where we may see a MemBarAcquire, MemBarRelease or
1438 // MemBarVolatile. For example, at the end of a constructor which
1439 // writes final/volatile fields we will see a MemBarRelease
1440 // instruction and this needs a 'dmb ish' lest we risk the
1441 // constructed object being visible without making the
1442 // final/volatile field writes visible.
1443 //
1444 // n.b. the translation rules below which rely on detection of the
1445 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1446 // If we see anything other than the signature configurations we
1447 // always just translate the loads and stores to ldr<x> and str<x>
1448 // and translate acquire, release and volatile membars to the
1449 // relevant dmb instructions.
1450 //
1451
1452 // is_CAS(int opcode, bool maybe_volatile)
1453 //
1454 // return true if opcode is one of the possible CompareAndSwapX
1455 // values otherwise false.
1456
1457 bool is_CAS(int opcode, bool maybe_volatile)
1458 {
1459 switch(opcode) {
1460 // We handle these
1461 case Op_CompareAndSwapI:
1462 case Op_CompareAndSwapL:
1463 case Op_CompareAndSwapP:
1464 case Op_CompareAndSwapN:
1465 case Op_ShenandoahCompareAndSwapP:
1466 case Op_ShenandoahCompareAndSwapN:
1467 case Op_CompareAndSwapB:
1468 case Op_CompareAndSwapS:
1469 case Op_GetAndSetI:
1470 case Op_GetAndSetL:
1471 case Op_GetAndSetP:
1472 case Op_GetAndSetN:
1473 case Op_GetAndAddI:
1474 case Op_GetAndAddL:
1475 return true;
1476 case Op_CompareAndExchangeI:
1477 case Op_CompareAndExchangeN:
1478 case Op_CompareAndExchangeB:
1479 case Op_CompareAndExchangeS:
1480 case Op_CompareAndExchangeL:
1481 case Op_CompareAndExchangeP:
1482 case Op_WeakCompareAndSwapB:
1483 case Op_WeakCompareAndSwapS:
1484 case Op_WeakCompareAndSwapI:
1485 case Op_WeakCompareAndSwapL:
1486 case Op_WeakCompareAndSwapP:
1487 case Op_WeakCompareAndSwapN:
1488 case Op_ShenandoahWeakCompareAndSwapP:
1489 case Op_ShenandoahWeakCompareAndSwapN:
1490 case Op_ShenandoahCompareAndExchangeP:
1491 case Op_ShenandoahCompareAndExchangeN:
1492 return maybe_volatile;
1493 default:
1494 return false;
1495 }
1496 }
1497
1498 // helper to determine the maximum number of Phi nodes we may need to
1499 // traverse when searching from a card mark membar for the merge mem
1500 // feeding a trailing membar or vice versa
1501
1502 // predicates controlling emit of ldr<x>/ldar<x>
1503
1504 bool unnecessary_acquire(const Node *barrier)
1505 {
1506 assert(barrier->is_MemBar(), "expecting a membar");
1507
1508 MemBarNode* mb = barrier->as_MemBar();
1509
1510 if (mb->trailing_load()) {
1511 return true;
1512 }
1513
1514 if (mb->trailing_load_store()) {
1515 Node* load_store = mb->in(MemBarNode::Precedent);
1516 assert(load_store->is_LoadStore(), "unexpected graph shape");
1517 return is_CAS(load_store->Opcode(), true);
1518 }
1519
1520 return false;
1521 }
1522
1523 bool needs_acquiring_load(const Node *n)
1524 {
1525 assert(n->is_Load(), "expecting a load");
1526 LoadNode *ld = n->as_Load();
1527 return ld->is_acquire();
1528 }
1529
1530 bool unnecessary_release(const Node *n)
1531 {
1532 assert((n->is_MemBar() &&
1533 n->Opcode() == Op_MemBarRelease),
1534 "expecting a release membar");
1535
1536 MemBarNode *barrier = n->as_MemBar();
1537 if (!barrier->leading()) {
1538 return false;
1539 } else {
1540 Node* trailing = barrier->trailing_membar();
1541 MemBarNode* trailing_mb = trailing->as_MemBar();
1542 assert(trailing_mb->trailing(), "Not a trailing membar?");
1543 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1544
1545 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1546 if (mem->is_Store()) {
1547 assert(mem->as_Store()->is_release(), "");
1548 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1549 return true;
1550 } else {
1551 assert(mem->is_LoadStore(), "");
1552 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1553 return is_CAS(mem->Opcode(), true);
1554 }
1555 }
1556 return false;
1557 }
1558
1559 bool unnecessary_volatile(const Node *n)
1560 {
1561 // assert n->is_MemBar();
1562 MemBarNode *mbvol = n->as_MemBar();
1563
1564 bool release = mbvol->trailing_store();
1565 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1566 #ifdef ASSERT
1567 if (release) {
1568 Node* leading = mbvol->leading_membar();
1569 assert(leading->Opcode() == Op_MemBarRelease, "");
1570 assert(leading->as_MemBar()->leading_store(), "");
1571 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1572 }
1573 #endif
1574
1575 return release;
1576 }
1577
1578 // predicates controlling emit of str<x>/stlr<x>
1579
1580 bool needs_releasing_store(const Node *n)
1581 {
1582 // assert n->is_Store();
1583 StoreNode *st = n->as_Store();
1584 return st->trailing_membar() != NULL;
1585 }
1586
1587 // predicate controlling translation of CAS
1588 //
1589 // returns true if CAS needs to use an acquiring load otherwise false
1590
1591 bool needs_acquiring_load_exclusive(const Node *n)
1592 {
1593 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1594 LoadStoreNode* ldst = n->as_LoadStore();
1595 if (is_CAS(n->Opcode(), false)) {
1596 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1597 } else {
1598 return ldst->trailing_membar() != NULL;
1599 }
1600
1601 // so we can just return true here
1602 return true;
1603 }
1604
1605 #define __ _masm.
1606
1607 // advance declarations for helper functions to convert register
1608 // indices to register objects
1609
1610 // the ad file has to provide implementations of certain methods
1611 // expected by the generic code
1612 //
1613 // REQUIRED FUNCTIONALITY
1614
1615 //=============================================================================
1616
1617 // !!!!! Special hack to get all types of calls to specify the byte offset
1618 // from the start of the call to the point where the return address
1619 // will point.
1620
1621 int MachCallStaticJavaNode::ret_addr_offset()
1622 {
1623 // call should be a simple bl
1624 int off = 4;
1625 return off;
1626 }
1627
1628 int MachCallDynamicJavaNode::ret_addr_offset()
1629 {
1630 return 16; // movz, movk, movk, bl
1631 }
1632
1633 int MachCallRuntimeNode::ret_addr_offset() {
1634 // for generated stubs the call will be
1635 // bl(addr)
1636 // or with far branches
1637 // bl(trampoline_stub)
1638 // for real runtime callouts it will be six instructions
1639 // see aarch64_enc_java_to_runtime
1640 // adr(rscratch2, retaddr)
1641 // lea(rscratch1, RuntimeAddress(addr)
1642 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1643 // blr(rscratch1)
1644 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1645 if (cb) {
1646 return 1 * NativeInstruction::instruction_size;
1647 } else {
1648 return 6 * NativeInstruction::instruction_size;
1649 }
1650 }
1651
1652 //=============================================================================
1653
1654 #ifndef PRODUCT
1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1656 st->print("BREAKPOINT");
1657 }
1658 #endif
1659
1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1661 C2_MacroAssembler _masm(&cbuf);
1662 __ brk(0);
1663 }
1664
1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1666 return MachNode::size(ra_);
1667 }
1668
1669 //=============================================================================
1670
1671 #ifndef PRODUCT
1672 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1673 st->print("nop \t# %d bytes pad for loops and calls", _count);
1674 }
1675 #endif
1676
1677 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1678 C2_MacroAssembler _masm(&cbuf);
1679 for (int i = 0; i < _count; i++) {
1680 __ nop();
1681 }
1682 }
1683
1684 uint MachNopNode::size(PhaseRegAlloc*) const {
1685 return _count * NativeInstruction::instruction_size;
1686 }
1687
1688 //=============================================================================
1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1690
1691 int ConstantTable::calculate_table_base_offset() const {
1692 return 0; // absolute addressing, no offset
1693 }
1694
1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1697 ShouldNotReachHere();
1698 }
1699
1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1701 // Empty encoding
1702 }
1703
1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1705 return 0;
1706 }
1707
1708 #ifndef PRODUCT
1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1710 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1711 }
1712 #endif
1713
1714 #ifndef PRODUCT
1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1716 Compile* C = ra_->C;
1717
1718 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1719
1720 if (C->output()->need_stack_bang(framesize))
1721 st->print("# stack bang size=%d\n\t", framesize);
1722
1723 if (VM_Version::use_rop_protection()) {
1724 st->print("ldr zr, [lr]\n\t");
1725 st->print("pacia lr, rfp\n\t");
1726 }
1727 if (framesize < ((1 << 9) + 2 * wordSize)) {
1728 st->print("sub sp, sp, #%d\n\t", framesize);
1729 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1730 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1731 } else {
1732 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1733 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1734 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1735 st->print("sub sp, sp, rscratch1");
1736 }
1737 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1738 st->print("\n\t");
1739 st->print("ldr rscratch1, [guard]\n\t");
1740 st->print("dmb ishld\n\t");
1741 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1742 st->print("cmp rscratch1, rscratch2\n\t");
1743 st->print("b.eq skip");
1744 st->print("\n\t");
1745 st->print("blr #nmethod_entry_barrier_stub\n\t");
1746 st->print("b skip\n\t");
1747 st->print("guard: int\n\t");
1748 st->print("\n\t");
1749 st->print("skip:\n\t");
1750 }
1751 }
1752 #endif
1753
1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1755 Compile* C = ra_->C;
1756 C2_MacroAssembler _masm(&cbuf);
1757
1758 // n.b. frame size includes space for return pc and rfp
1759 const int framesize = C->output()->frame_size_in_bytes();
1760
1761 // insert a nop at the start of the prolog so we can patch in a
1762 // branch if we need to invalidate the method later
1763 __ nop();
1764
1765 if (C->clinit_barrier_on_entry()) {
1766 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1767
1768 Label L_skip_barrier;
1769
1770 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1771 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1772 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1773 __ bind(L_skip_barrier);
1774 }
1775
1776 if (C->max_vector_size() > 0) {
1777 __ reinitialize_ptrue();
1778 }
1779
1780 int bangsize = C->output()->bang_size_in_bytes();
1781 if (C->output()->need_stack_bang(bangsize))
1782 __ generate_stack_overflow_check(bangsize);
1783
1784 __ build_frame(framesize);
1785
1786 if (C->stub_function() == NULL) {
1787 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1788 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1789 // Dummy labels for just measuring the code size
1790 Label dummy_slow_path;
1791 Label dummy_continuation;
1792 Label dummy_guard;
1793 Label* slow_path = &dummy_slow_path;
1794 Label* continuation = &dummy_continuation;
1795 Label* guard = &dummy_guard;
1796 if (!Compile::current()->output()->in_scratch_emit_size()) {
1797 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1798 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1799 Compile::current()->output()->add_stub(stub);
1800 slow_path = &stub->entry();
1801 continuation = &stub->continuation();
1802 guard = &stub->guard();
1803 }
1804 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1805 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1806 }
1807 }
1808
1809 if (VerifyStackAtCalls) {
1810 Unimplemented();
1811 }
1812
1813 C->output()->set_frame_complete(cbuf.insts_size());
1814
1815 if (C->has_mach_constant_base_node()) {
1816 // NOTE: We set the table base offset here because users might be
1817 // emitted before MachConstantBaseNode.
1818 ConstantTable& constant_table = C->output()->constant_table();
1819 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1820 }
1821 }
1822
1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1824 {
1825 return MachNode::size(ra_); // too many variables; just compute it
1826 // the hard way
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autia lr, rfp\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 C2_MacroAssembler _masm(&cbuf);
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 __ remove_frame(framesize);
1873
1874 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1875 __ reserved_stack_check();
1876 }
1877
1878 if (do_polling() && C->is_method_compilation()) {
1879 Label dummy_label;
1880 Label* code_stub = &dummy_label;
1881 if (!C->output()->in_scratch_emit_size()) {
1882 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1883 C->output()->add_stub(stub);
1884 code_stub = &stub->entry();
1885 }
1886 __ relocate(relocInfo::poll_return_type);
1887 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1888 }
1889 }
1890
1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1892 // Variable size. Determine dynamically.
1893 return MachNode::size(ra_);
1894 }
1895
1896 int MachEpilogNode::reloc() const {
1897 // Return number of relocatable values contained in this instruction.
1898 return 1; // 1 for polling page.
1899 }
1900
1901 const Pipeline * MachEpilogNode::pipeline() const {
1902 return MachNode::pipeline_class();
1903 }
1904
1905 //=============================================================================
1906
1907 // Figure out which register class each belongs in: rc_int, rc_float or
1908 // rc_stack.
1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && cbuf) {
1975 C2_MacroAssembler _masm(cbuf);
1976 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1977 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1978 // stack->stack
1979 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1980 sve_vector_reg_size_in_bytes);
1981 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1982 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1983 sve_vector_reg_size_in_bytes);
1984 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1985 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1986 sve_vector_reg_size_in_bytes);
1987 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1988 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]));
1991 } else {
1992 ShouldNotReachHere();
1993 }
1994 } else if (cbuf) {
1995 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1996 C2_MacroAssembler _masm(cbuf);
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (cbuf) {
2024 C2_MacroAssembler _masm(cbuf);
2025 switch (src_lo_rc) {
2026 case rc_int:
2027 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2028 if (is64) {
2029 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 } else {
2032 C2_MacroAssembler _masm(cbuf);
2033 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2034 as_Register(Matcher::_regEncode[src_lo]));
2035 }
2036 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2037 if (is64) {
2038 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 as_Register(Matcher::_regEncode[src_lo]));
2040 } else {
2041 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 as_Register(Matcher::_regEncode[src_lo]));
2043 }
2044 } else { // gpr --> stack spill
2045 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2046 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2047 }
2048 break;
2049 case rc_float:
2050 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2051 if (is64) {
2052 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 } else {
2055 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_FloatRegister(Matcher::_regEncode[src_lo]));
2057 }
2058 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2059 if (is64) {
2060 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2061 as_FloatRegister(Matcher::_regEncode[src_lo]));
2062 } else {
2063 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_FloatRegister(Matcher::_regEncode[src_lo]));
2065 }
2066 } else { // fpr --> stack spill
2067 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2068 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2069 is64 ? __ D : __ S, dst_offset);
2070 }
2071 break;
2072 case rc_stack:
2073 if (dst_lo_rc == rc_int) { // stack --> gpr load
2074 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2075 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2076 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2077 is64 ? __ D : __ S, src_offset);
2078 } else if (dst_lo_rc == rc_predicate) {
2079 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2080 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2081 } else { // stack --> stack copy
2082 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2083 if (ideal_reg() == Op_RegVectMask) {
2084 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2085 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2086 } else {
2087 __ unspill(rscratch1, is64, src_offset);
2088 __ spill(rscratch1, is64, dst_offset);
2089 }
2090 }
2091 break;
2092 case rc_predicate:
2093 if (dst_lo_rc == rc_predicate) {
2094 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2095 } else if (dst_lo_rc == rc_stack) {
2096 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2097 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2098 } else {
2099 assert(false, "bad src and dst rc_class combination.");
2100 ShouldNotReachHere();
2101 }
2102 break;
2103 default:
2104 assert(false, "bad rc_class for spill");
2105 ShouldNotReachHere();
2106 }
2107 }
2108
2109 if (st) {
2110 st->print("spill ");
2111 if (src_lo_rc == rc_stack) {
2112 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2113 } else {
2114 st->print("%s -> ", Matcher::regName[src_lo]);
2115 }
2116 if (dst_lo_rc == rc_stack) {
2117 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2118 } else {
2119 st->print("%s", Matcher::regName[dst_lo]);
2120 }
2121 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2122 int vsize = 0;
2123 switch (ideal_reg()) {
2124 case Op_VecD:
2125 vsize = 64;
2126 break;
2127 case Op_VecX:
2128 vsize = 128;
2129 break;
2130 case Op_VecA:
2131 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2132 break;
2133 default:
2134 assert(false, "bad register type for spill");
2135 ShouldNotReachHere();
2136 }
2137 st->print("\t# vector spill size = %d", vsize);
2138 } else if (ideal_reg() == Op_RegVectMask) {
2139 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2140 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2141 st->print("\t# predicate spill size = %d", vsize);
2142 } else {
2143 st->print("\t# spill size = %d", is64 ? 64 : 32);
2144 }
2145 }
2146
2147 return 0;
2148
2149 }
2150
2151 #ifndef PRODUCT
2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2153 if (!ra_)
2154 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2155 else
2156 implementation(NULL, ra_, false, st);
2157 }
2158 #endif
2159
2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2161 implementation(&cbuf, ra_, false, NULL);
2162 }
2163
2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2165 return MachNode::size(ra_);
2166 }
2167
2168 //=============================================================================
2169
2170 #ifndef PRODUCT
2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2172 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2173 int reg = ra_->get_reg_first(this);
2174 st->print("add %s, rsp, #%d]\t# box lock",
2175 Matcher::regName[reg], offset);
2176 }
2177 #endif
2178
2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2180 C2_MacroAssembler _masm(&cbuf);
2181
2182 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2183 int reg = ra_->get_encode(this);
2184
2185 // This add will handle any 24-bit signed offset. 24 bits allows an
2186 // 8 megabyte stack frame.
2187 __ add(as_Register(reg), sp, offset);
2188 }
2189
2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2191 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2192 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2193
2194 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2195 return NativeInstruction::instruction_size;
2196 } else {
2197 return 2 * NativeInstruction::instruction_size;
2198 }
2199 }
2200
2201 //=============================================================================
2202
2203 #ifndef PRODUCT
2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2205 {
2206 st->print_cr("# MachUEPNode");
2207 if (UseCompressedClassPointers) {
2208 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 if (CompressedKlassPointers::shift() != 0) {
2210 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2211 }
2212 } else {
2213 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2214 }
2215 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2216 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2217 }
2218 #endif
2219
2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2221 {
2222 // This is the unverified entry point.
2223 C2_MacroAssembler _masm(&cbuf);
2224
2225 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2226 Label skip;
2227 // TODO
2228 // can we avoid this skip and still use a reloc?
2229 __ br(Assembler::EQ, skip);
2230 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2231 __ bind(skip);
2232 }
2233
2234 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2235 {
2236 return MachNode::size(ra_);
2237 }
2238
2239 // REQUIRED EMIT CODE
2240
2241 //=============================================================================
2242
2243 // Emit exception handler code.
2244 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2245 {
2246 // mov rscratch1 #exception_blob_entry_point
2247 // br rscratch1
2248 // Note that the code buffer's insts_mark is always relative to insts.
2249 // That's why we must use the macroassembler to generate a handler.
2250 C2_MacroAssembler _masm(&cbuf);
2251 address base = __ start_a_stub(size_exception_handler());
2252 if (base == NULL) {
2253 ciEnv::current()->record_failure("CodeCache is full");
2254 return 0; // CodeBuffer::expand failed
2255 }
2256 int offset = __ offset();
2257 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2258 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2259 __ end_a_stub();
2260 return offset;
2261 }
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 C2_MacroAssembler _masm(&cbuf);
2269 address base = __ start_a_stub(size_deopt_handler());
2270 if (base == NULL) {
2271 ciEnv::current()->record_failure("CodeCache is full");
2272 return 0; // CodeBuffer::expand failed
2273 }
2274 int offset = __ offset();
2275
2276 __ adr(lr, __ pc());
2277 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 __ end_a_stub();
2281 return offset;
2282 }
2283
2284 // REQUIRED MATCHER CODE
2285
2286 //=============================================================================
2287
2288 const bool Matcher::match_rule_supported(int opcode) {
2289 if (!has_match_rule(opcode))
2290 return false;
2291
2292 bool ret_value = true;
2293 switch (opcode) {
2294 case Op_OnSpinWait:
2295 return VM_Version::supports_on_spin_wait();
2296 case Op_CacheWB:
2297 case Op_CacheWBPreSync:
2298 case Op_CacheWBPostSync:
2299 if (!VM_Version::supports_data_cache_line_flush()) {
2300 ret_value = false;
2301 }
2302 break;
2303 case Op_ExpandBits:
2304 case Op_CompressBits:
2305 if (!(UseSVE > 1 && VM_Version::supports_svebitperm())) {
2306 ret_value = false;
2307 }
2308 break;
2309 }
2310
2311 return ret_value; // Per default match rules are supported.
2312 }
2313
2314 const RegMask* Matcher::predicate_reg_mask(void) {
2315 return &_PR_REG_mask;
2316 }
2317
2318 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2319 return new TypeVectMask(elemTy, length);
2320 }
2321
2322 // Vector calling convention not yet implemented.
2323 const bool Matcher::supports_vector_calling_convention(void) {
2324 return false;
2325 }
2326
2327 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2328 Unimplemented();
2329 return OptoRegPair(0, 0);
2330 }
2331
2332 // Is this branch offset short enough that a short branch can be used?
2333 //
2334 // NOTE: If the platform does not provide any short branch variants, then
2335 // this method should return false for offset 0.
2336 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2337 // The passed offset is relative to address of the branch.
2338
2339 return (-32768 <= offset && offset < 32768);
2340 }
2341
2342 // Vector width in bytes.
2343 const int Matcher::vector_width_in_bytes(BasicType bt) {
2344 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2345 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2346 // Minimum 2 values in vector
2347 if (size < 2*type2aelembytes(bt)) size = 0;
2348 // But never < 4
2349 if (size < 4) size = 0;
2350 return size;
2351 }
2352
2353 // Limits on vector size (number of elements) loaded into vector.
2354 const int Matcher::max_vector_size(const BasicType bt) {
2355 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2356 }
2357
2358 const int Matcher::min_vector_size(const BasicType bt) {
2359 int max_size = max_vector_size(bt);
2360 // Limit the min vector size to 8 bytes.
2361 int size = 8 / type2aelembytes(bt);
2362 if (bt == T_BYTE) {
2363 // To support vector api shuffle/rearrange.
2364 size = 4;
2365 } else if (bt == T_BOOLEAN) {
2366 // To support vector api load/store mask.
2367 size = 2;
2368 }
2369 if (size < 2) size = 2;
2370 return MIN2(size, max_size);
2371 }
2372
2373 const int Matcher::superword_max_vector_size(const BasicType bt) {
2374 return Matcher::max_vector_size(bt);
2375 }
2376
2377 // Actual max scalable vector register length.
2378 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2379 return Matcher::max_vector_size(bt);
2380 }
2381
2382 // Vector ideal reg.
2383 const uint Matcher::vector_ideal_reg(int len) {
2384 if (UseSVE > 0 && 16 < len && len <= 256) {
2385 return Op_VecA;
2386 }
2387 switch(len) {
2388 // For 16-bit/32-bit mask vector, reuse VecD.
2389 case 2:
2390 case 4:
2391 case 8: return Op_VecD;
2392 case 16: return Op_VecX;
2393 }
2394 ShouldNotReachHere();
2395 return 0;
2396 }
2397
2398 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2399 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2400 switch (ideal_reg) {
2401 case Op_VecA: return new vecAOper();
2402 case Op_VecD: return new vecDOper();
2403 case Op_VecX: return new vecXOper();
2404 }
2405 ShouldNotReachHere();
2406 return NULL;
2407 }
2408
2409 bool Matcher::is_reg2reg_move(MachNode* m) {
2410 return false;
2411 }
2412
2413 bool Matcher::is_generic_vector(MachOper* opnd) {
2414 return opnd->opcode() == VREG;
2415 }
2416
2417 // Return whether or not this register is ever used as an argument.
2418 // This function is used on startup to build the trampoline stubs in
2419 // generateOptoStub. Registers not mentioned will be killed by the VM
2420 // call in the trampoline, and arguments in those registers not be
2421 // available to the callee.
2422 bool Matcher::can_be_java_arg(int reg)
2423 {
2424 return
2425 reg == R0_num || reg == R0_H_num ||
2426 reg == R1_num || reg == R1_H_num ||
2427 reg == R2_num || reg == R2_H_num ||
2428 reg == R3_num || reg == R3_H_num ||
2429 reg == R4_num || reg == R4_H_num ||
2430 reg == R5_num || reg == R5_H_num ||
2431 reg == R6_num || reg == R6_H_num ||
2432 reg == R7_num || reg == R7_H_num ||
2433 reg == V0_num || reg == V0_H_num ||
2434 reg == V1_num || reg == V1_H_num ||
2435 reg == V2_num || reg == V2_H_num ||
2436 reg == V3_num || reg == V3_H_num ||
2437 reg == V4_num || reg == V4_H_num ||
2438 reg == V5_num || reg == V5_H_num ||
2439 reg == V6_num || reg == V6_H_num ||
2440 reg == V7_num || reg == V7_H_num;
2441 }
2442
2443 bool Matcher::is_spillable_arg(int reg)
2444 {
2445 return can_be_java_arg(reg);
2446 }
2447
2448 uint Matcher::int_pressure_limit()
2449 {
2450 // JDK-8183543: When taking the number of available registers as int
2451 // register pressure threshold, the jtreg test:
2452 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2453 // failed due to C2 compilation failure with
2454 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2455 //
2456 // A derived pointer is live at CallNode and then is flagged by RA
2457 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2458 // derived pointers and lastly fail to spill after reaching maximum
2459 // number of iterations. Lowering the default pressure threshold to
2460 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2461 // a high register pressure area of the code so that split_DEF can
2462 // generate DefinitionSpillCopy for the derived pointer.
2463 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2464 if (!PreserveFramePointer) {
2465 // When PreserveFramePointer is off, frame pointer is allocatable,
2466 // but different from other SOC registers, it is excluded from
2467 // fatproj's mask because its save type is No-Save. Decrease 1 to
2468 // ensure high pressure at fatproj when PreserveFramePointer is off.
2469 // See check_pressure_at_fatproj().
2470 default_int_pressure_threshold--;
2471 }
2472 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2473 }
2474
2475 uint Matcher::float_pressure_limit()
2476 {
2477 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2478 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2479 }
2480
2481 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2482 return false;
2483 }
2484
2485 RegMask Matcher::divI_proj_mask() {
2486 ShouldNotReachHere();
2487 return RegMask();
2488 }
2489
2490 // Register for MODI projection of divmodI.
2491 RegMask Matcher::modI_proj_mask() {
2492 ShouldNotReachHere();
2493 return RegMask();
2494 }
2495
2496 // Register for DIVL projection of divmodL.
2497 RegMask Matcher::divL_proj_mask() {
2498 ShouldNotReachHere();
2499 return RegMask();
2500 }
2501
2502 // Register for MODL projection of divmodL.
2503 RegMask Matcher::modL_proj_mask() {
2504 ShouldNotReachHere();
2505 return RegMask();
2506 }
2507
2508 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2509 return FP_REG_mask();
2510 }
2511
2512 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2513 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2514 Node* u = addp->fast_out(i);
2515 if (u->is_LoadStore()) {
2516 // On AArch64, LoadStoreNodes (i.e. compare and swap
2517 // instructions) only take register indirect as an operand, so
2518 // any attempt to use an AddPNode as an input to a LoadStoreNode
2519 // must fail.
2520 return false;
2521 }
2522 if (u->is_Mem()) {
2523 int opsize = u->as_Mem()->memory_size();
2524 assert(opsize > 0, "unexpected memory operand size");
2525 if (u->as_Mem()->memory_size() != (1<<shift)) {
2526 return false;
2527 }
2528 }
2529 }
2530 return true;
2531 }
2532
2533 // Convert BootTest condition to Assembler condition.
2534 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2535 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2536 Assembler::Condition result;
2537 switch(cond) {
2538 case BoolTest::eq:
2539 result = Assembler::EQ; break;
2540 case BoolTest::ne:
2541 result = Assembler::NE; break;
2542 case BoolTest::le:
2543 result = Assembler::LE; break;
2544 case BoolTest::ge:
2545 result = Assembler::GE; break;
2546 case BoolTest::lt:
2547 result = Assembler::LT; break;
2548 case BoolTest::gt:
2549 result = Assembler::GT; break;
2550 case BoolTest::ule:
2551 result = Assembler::LS; break;
2552 case BoolTest::uge:
2553 result = Assembler::HS; break;
2554 case BoolTest::ult:
2555 result = Assembler::LO; break;
2556 case BoolTest::ugt:
2557 result = Assembler::HI; break;
2558 case BoolTest::overflow:
2559 result = Assembler::VS; break;
2560 case BoolTest::no_overflow:
2561 result = Assembler::VC; break;
2562 default:
2563 ShouldNotReachHere();
2564 return Assembler::Condition(-1);
2565 }
2566
2567 // Check conversion
2568 if (cond & BoolTest::unsigned_compare) {
2569 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2570 } else {
2571 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2572 }
2573
2574 return result;
2575 }
2576
2577 // Binary src (Replicate con)
2578 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2579 if (n == NULL || m == NULL) {
2580 return false;
2581 }
2582
2583 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2584 return false;
2585 }
2586
2587 Node* imm_node = m->in(1);
2588 if (!imm_node->is_Con()) {
2589 return false;
2590 }
2591
2592 const Type* t = imm_node->bottom_type();
2593 if (!(t->isa_int() || t->isa_long())) {
2594 return false;
2595 }
2596
2597 switch (n->Opcode()) {
2598 case Op_AndV:
2599 case Op_OrV:
2600 case Op_XorV: {
2601 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2602 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2603 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2604 }
2605 case Op_AddVB:
2606 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2607 case Op_AddVS:
2608 case Op_AddVI:
2609 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2610 case Op_AddVL:
2611 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2612 default:
2613 return false;
2614 }
2615 }
2616
2617 // (XorV src (Replicate m1))
2618 // (XorVMask src (MaskAll m1))
2619 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2620 if (n != NULL && m != NULL) {
2621 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2622 VectorNode::is_all_ones_vector(m);
2623 }
2624 return false;
2625 }
2626
2627 // Should the matcher clone input 'm' of node 'n'?
2628 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2629 if (is_vshift_con_pattern(n, m) ||
2630 is_vector_bitwise_not_pattern(n, m) ||
2631 is_valid_sve_arith_imm_pattern(n, m)) {
2632 mstack.push(m, Visit);
2633 return true;
2634 }
2635 return false;
2636 }
2637
2638 // Should the Matcher clone shifts on addressing modes, expecting them
2639 // to be subsumed into complex addressing expressions or compute them
2640 // into registers?
2641 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2642 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2643 return true;
2644 }
2645
2646 Node *off = m->in(AddPNode::Offset);
2647 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2648 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2649 // Are there other uses besides address expressions?
2650 !is_visited(off)) {
2651 address_visited.set(off->_idx); // Flag as address_visited
2652 mstack.push(off->in(2), Visit);
2653 Node *conv = off->in(1);
2654 if (conv->Opcode() == Op_ConvI2L &&
2655 // Are there other uses besides address expressions?
2656 !is_visited(conv)) {
2657 address_visited.set(conv->_idx); // Flag as address_visited
2658 mstack.push(conv->in(1), Pre_Visit);
2659 } else {
2660 mstack.push(conv, Pre_Visit);
2661 }
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2664 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2665 return true;
2666 } else if (off->Opcode() == Op_ConvI2L &&
2667 // Are there other uses besides address expressions?
2668 !is_visited(off)) {
2669 address_visited.test_set(m->_idx); // Flag as address_visited
2670 address_visited.set(off->_idx); // Flag as address_visited
2671 mstack.push(off->in(1), Pre_Visit);
2672 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2673 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2674 return true;
2675 }
2676 return false;
2677 }
2678
2679 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2680 C2_MacroAssembler _masm(&cbuf); \
2681 { \
2682 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2683 guarantee(DISP == 0, "mode not permitted for volatile"); \
2684 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2685 __ INSN(REG, as_Register(BASE)); \
2686 }
2687
2688
2689 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2690 {
2691 Address::extend scale;
2692
2693 // Hooboy, this is fugly. We need a way to communicate to the
2694 // encoder that the index needs to be sign extended, so we have to
2695 // enumerate all the cases.
2696 switch (opcode) {
2697 case INDINDEXSCALEDI2L:
2698 case INDINDEXSCALEDI2LN:
2699 case INDINDEXI2L:
2700 case INDINDEXI2LN:
2701 scale = Address::sxtw(size);
2702 break;
2703 default:
2704 scale = Address::lsl(size);
2705 }
2706
2707 if (index == -1) {
2708 return Address(base, disp);
2709 } else {
2710 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2711 return Address(base, as_Register(index), scale);
2712 }
2713 }
2714
2715
2716 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2717 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2718 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2719 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2720 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2721
2722 // Used for all non-volatile memory accesses. The use of
2723 // $mem->opcode() to discover whether this pattern uses sign-extended
2724 // offsets is something of a kludge.
2725 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2726 Register reg, int opcode,
2727 Register base, int index, int scale, int disp,
2728 int size_in_memory)
2729 {
2730 Address addr = mem2address(opcode, base, index, scale, disp);
2731 if (addr.getMode() == Address::base_plus_offset) {
2732 /* If we get an out-of-range offset it is a bug in the compiler,
2733 so we assert here. */
2734 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2735 "c2 compiler bug");
2736 /* Fix up any out-of-range offsets. */
2737 assert_different_registers(rscratch1, base);
2738 assert_different_registers(rscratch1, reg);
2739 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2740 }
2741 (masm.*insn)(reg, addr);
2742 }
2743
2744 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2745 FloatRegister reg, int opcode,
2746 Register base, int index, int size, int disp,
2747 int size_in_memory)
2748 {
2749 Address::extend scale;
2750
2751 switch (opcode) {
2752 case INDINDEXSCALEDI2L:
2753 case INDINDEXSCALEDI2LN:
2754 scale = Address::sxtw(size);
2755 break;
2756 default:
2757 scale = Address::lsl(size);
2758 }
2759
2760 if (index == -1) {
2761 /* If we get an out-of-range offset it is a bug in the compiler,
2762 so we assert here. */
2763 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2764 /* Fix up any out-of-range offsets. */
2765 assert_different_registers(rscratch1, base);
2766 Address addr = Address(base, disp);
2767 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2768 (masm.*insn)(reg, addr);
2769 } else {
2770 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2771 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2772 }
2773 }
2774
2775 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2776 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2777 int opcode, Register base, int index, int size, int disp)
2778 {
2779 if (index == -1) {
2780 (masm.*insn)(reg, T, Address(base, disp));
2781 } else {
2782 assert(disp == 0, "unsupported address mode");
2783 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2784 }
2785 }
2786
2787 %}
2788
2789
2790
2791 //----------ENCODING BLOCK-----------------------------------------------------
2792 // This block specifies the encoding classes used by the compiler to
2793 // output byte streams. Encoding classes are parameterized macros
2794 // used by Machine Instruction Nodes in order to generate the bit
2795 // encoding of the instruction. Operands specify their base encoding
2796 // interface with the interface keyword. There are currently
2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2798 // COND_INTER. REG_INTER causes an operand to generate a function
2799 // which returns its register number when queried. CONST_INTER causes
2800 // an operand to generate a function which returns the value of the
2801 // constant when queried. MEMORY_INTER causes an operand to generate
2802 // four functions which return the Base Register, the Index Register,
2803 // the Scale Value, and the Offset Value of the operand when queried.
2804 // COND_INTER causes an operand to generate six functions which return
2805 // the encoding code (ie - encoding bits for the instruction)
2806 // associated with each basic boolean condition for a conditional
2807 // instruction.
2808 //
2809 // Instructions specify two basic values for encoding. Again, a
2810 // function is available to check if the constant displacement is an
2811 // oop. They use the ins_encode keyword to specify their encoding
2812 // classes (which must be a sequence of enc_class names, and their
2813 // parameters, specified in the encoding block), and they use the
2814 // opcode keyword to specify, in order, their primary, secondary, and
2815 // tertiary opcode. Only the opcode sections which a particular
2816 // instruction needs for encoding need to be specified.
2817 encode %{
2818 // Build emit functions for each basic byte or larger field in the
2819 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2820 // from C++ code in the enc_class source block. Emit functions will
2821 // live in the main source block for now. In future, we can
2822 // generalize this by adding a syntax that specifies the sizes of
2823 // fields in an order, so that the adlc can build the emit functions
2824 // automagically
2825
2826 // catch all for unimplemented encodings
2827 enc_class enc_unimplemented %{
2828 C2_MacroAssembler _masm(&cbuf);
2829 __ unimplemented("C2 catch all");
2830 %}
2831
2832 // BEGIN Non-volatile memory access
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2933 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2934 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2941 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2942 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2949 Register src_reg = as_Register($src$$reg);
2950 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_strb0(memory1 mem) %{
2957 C2_MacroAssembler _masm(&cbuf);
2958 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strh0(memory2 mem) %{
2973 C2_MacroAssembler _masm(&cbuf);
2974 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2981 Register src_reg = as_Register($src$$reg);
2982 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_strw0(memory4 mem) %{
2989 C2_MacroAssembler _masm(&cbuf);
2990 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2997 Register src_reg = as_Register($src$$reg);
2998 // we sometimes get asked to store the stack pointer into the
2999 // current thread -- we cannot do that directly on AArch64
3000 if (src_reg == r31_sp) {
3001 C2_MacroAssembler _masm(&cbuf);
3002 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3003 __ mov(rscratch2, sp);
3004 src_reg = rscratch2;
3005 }
3006 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str0(memory8 mem) %{
3013 C2_MacroAssembler _masm(&cbuf);
3014 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3021 FloatRegister src_reg = as_FloatRegister($src$$reg);
3022 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3029 FloatRegister src_reg = as_FloatRegister($src$$reg);
3030 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3037 C2_MacroAssembler _masm(&cbuf);
3038 __ membar(Assembler::StoreStore);
3039 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3041 %}
3042
3043 // END Non-volatile memory access
3044
3045 // Vector loads and stores
3046 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3047 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3048 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3049 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3050 %}
3051
3052 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3053 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3054 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3055 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3056 %}
3057
3058 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3059 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3060 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3061 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3062 %}
3063
3064 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3065 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3066 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3067 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3068 %}
3069
3070 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3071 FloatRegister src_reg = as_FloatRegister($src$$reg);
3072 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3073 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3074 %}
3075
3076 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3079 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3080 %}
3081
3082 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3083 FloatRegister src_reg = as_FloatRegister($src$$reg);
3084 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3085 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3086 %}
3087
3088 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3089 FloatRegister src_reg = as_FloatRegister($src$$reg);
3090 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3091 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3092 %}
3093
3094 // volatile loads and stores
3095
3096 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3097 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrb0(memory mem) %{
3102 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrb);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3107 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrh0(memory mem) %{
3112 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrh);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3117 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_stlrw0(memory mem) %{
3122 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, stlrw);
3124 %}
3125
3126 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3127 Register dst_reg = as_Register($dst$$reg);
3128 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 __ sxtbw(dst_reg, dst_reg);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtb(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3146 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarb);
3148 %}
3149
3150 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3151 Register dst_reg = as_Register($dst$$reg);
3152 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 __ sxthw(dst_reg, dst_reg);
3155 %}
3156
3157 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxth(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarh);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarw);
3182 %}
3183
3184 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3185 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldar);
3187 %}
3188
3189 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3190 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarw);
3192 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3193 %}
3194
3195 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3196 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldar);
3198 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3199 %}
3200
3201 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3202 Register src_reg = as_Register($src$$reg);
3203 // we sometimes get asked to store the stack pointer into the
3204 // current thread -- we cannot do that directly on AArch64
3205 if (src_reg == r31_sp) {
3206 C2_MacroAssembler _masm(&cbuf);
3207 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3208 __ mov(rscratch2, sp);
3209 src_reg = rscratch2;
3210 }
3211 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 enc_class aarch64_enc_stlr0(memory mem) %{
3216 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, stlr);
3218 %}
3219
3220 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3221 {
3222 C2_MacroAssembler _masm(&cbuf);
3223 FloatRegister src_reg = as_FloatRegister($src$$reg);
3224 __ fmovs(rscratch2, src_reg);
3225 }
3226 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, stlrw);
3228 %}
3229
3230 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3231 {
3232 C2_MacroAssembler _masm(&cbuf);
3233 FloatRegister src_reg = as_FloatRegister($src$$reg);
3234 __ fmovd(rscratch2, src_reg);
3235 }
3236 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, stlr);
3238 %}
3239
3240 // synchronized read/update encodings
3241
3242 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3243 C2_MacroAssembler _masm(&cbuf);
3244 Register dst_reg = as_Register($dst$$reg);
3245 Register base = as_Register($mem$$base);
3246 int index = $mem$$index;
3247 int scale = $mem$$scale;
3248 int disp = $mem$$disp;
3249 if (index == -1) {
3250 if (disp != 0) {
3251 __ lea(rscratch1, Address(base, disp));
3252 __ ldaxr(dst_reg, rscratch1);
3253 } else {
3254 // TODO
3255 // should we ever get anything other than this case?
3256 __ ldaxr(dst_reg, base);
3257 }
3258 } else {
3259 Register index_reg = as_Register(index);
3260 if (disp == 0) {
3261 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3262 __ ldaxr(dst_reg, rscratch1);
3263 } else {
3264 __ lea(rscratch1, Address(base, disp));
3265 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3266 __ ldaxr(dst_reg, rscratch1);
3267 }
3268 }
3269 %}
3270
3271 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3272 C2_MacroAssembler _masm(&cbuf);
3273 Register src_reg = as_Register($src$$reg);
3274 Register base = as_Register($mem$$base);
3275 int index = $mem$$index;
3276 int scale = $mem$$scale;
3277 int disp = $mem$$disp;
3278 if (index == -1) {
3279 if (disp != 0) {
3280 __ lea(rscratch2, Address(base, disp));
3281 __ stlxr(rscratch1, src_reg, rscratch2);
3282 } else {
3283 // TODO
3284 // should we ever get anything other than this case?
3285 __ stlxr(rscratch1, src_reg, base);
3286 }
3287 } else {
3288 Register index_reg = as_Register(index);
3289 if (disp == 0) {
3290 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3291 __ stlxr(rscratch1, src_reg, rscratch2);
3292 } else {
3293 __ lea(rscratch2, Address(base, disp));
3294 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3295 __ stlxr(rscratch1, src_reg, rscratch2);
3296 }
3297 }
3298 __ cmpw(rscratch1, zr);
3299 %}
3300
3301 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3302 C2_MacroAssembler _masm(&cbuf);
3303 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3304 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3305 Assembler::xword, /*acquire*/ false, /*release*/ true,
3306 /*weak*/ false, noreg);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3310 C2_MacroAssembler _masm(&cbuf);
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::word, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 C2_MacroAssembler _masm(&cbuf);
3319 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3320 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3321 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3322 /*weak*/ false, noreg);
3323 %}
3324
3325 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3326 C2_MacroAssembler _masm(&cbuf);
3327 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3328 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3329 Assembler::byte, /*acquire*/ false, /*release*/ true,
3330 /*weak*/ false, noreg);
3331 %}
3332
3333
3334 // The only difference between aarch64_enc_cmpxchg and
3335 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3336 // CompareAndSwap sequence to serve as a barrier on acquiring a
3337 // lock.
3338 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3339 C2_MacroAssembler _masm(&cbuf);
3340 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3341 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3342 Assembler::xword, /*acquire*/ true, /*release*/ true,
3343 /*weak*/ false, noreg);
3344 %}
3345
3346 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3347 C2_MacroAssembler _masm(&cbuf);
3348 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3349 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3350 Assembler::word, /*acquire*/ true, /*release*/ true,
3351 /*weak*/ false, noreg);
3352 %}
3353
3354 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3355 C2_MacroAssembler _masm(&cbuf);
3356 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3357 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3358 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3359 /*weak*/ false, noreg);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3363 C2_MacroAssembler _masm(&cbuf);
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 C2_MacroAssembler _masm(&cbuf);
3373 Register res_reg = as_Register($res$$reg);
3374 __ cset(res_reg, Assembler::EQ);
3375 %}
3376
3377 // prefetch encodings
3378
3379 enc_class aarch64_enc_prefetchw(memory mem) %{
3380 C2_MacroAssembler _masm(&cbuf);
3381 Register base = as_Register($mem$$base);
3382 int index = $mem$$index;
3383 int scale = $mem$$scale;
3384 int disp = $mem$$disp;
3385 if (index == -1) {
3386 __ prfm(Address(base, disp), PSTL1KEEP);
3387 } else {
3388 Register index_reg = as_Register(index);
3389 if (disp == 0) {
3390 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3391 } else {
3392 __ lea(rscratch1, Address(base, disp));
3393 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3394 }
3395 }
3396 %}
3397
3398 /// mov envcodings
3399
3400 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3401 C2_MacroAssembler _masm(&cbuf);
3402 uint32_t con = (uint32_t)$src$$constant;
3403 Register dst_reg = as_Register($dst$$reg);
3404 if (con == 0) {
3405 __ movw(dst_reg, zr);
3406 } else {
3407 __ movw(dst_reg, con);
3408 }
3409 %}
3410
3411 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3412 C2_MacroAssembler _masm(&cbuf);
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 C2_MacroAssembler _masm(&cbuf);
3424 Register dst_reg = as_Register($dst$$reg);
3425 address con = (address)$src$$constant;
3426 if (con == NULL || con == (address)1) {
3427 ShouldNotReachHere();
3428 } else {
3429 relocInfo::relocType rtype = $src->constant_reloc();
3430 if (rtype == relocInfo::oop_type) {
3431 __ movoop(dst_reg, (jobject)con);
3432 } else if (rtype == relocInfo::metadata_type) {
3433 __ mov_metadata(dst_reg, (Metadata*)con);
3434 } else {
3435 assert(rtype == relocInfo::none, "unexpected reloc type");
3436 if (! __ is_valid_AArch64_address(con) ||
3437 con < (address)(uintptr_t)os::vm_page_size()) {
3438 __ mov(dst_reg, con);
3439 } else {
3440 uint64_t offset;
3441 __ adrp(dst_reg, con, offset);
3442 __ add(dst_reg, dst_reg, offset);
3443 }
3444 }
3445 }
3446 %}
3447
3448 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3449 C2_MacroAssembler _masm(&cbuf);
3450 Register dst_reg = as_Register($dst$$reg);
3451 __ mov(dst_reg, zr);
3452 %}
3453
3454 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3455 C2_MacroAssembler _masm(&cbuf);
3456 Register dst_reg = as_Register($dst$$reg);
3457 __ mov(dst_reg, (uint64_t)1);
3458 %}
3459
3460 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3461 C2_MacroAssembler _masm(&cbuf);
3462 __ load_byte_map_base($dst$$Register);
3463 %}
3464
3465 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3466 C2_MacroAssembler _masm(&cbuf);
3467 Register dst_reg = as_Register($dst$$reg);
3468 address con = (address)$src$$constant;
3469 if (con == NULL) {
3470 ShouldNotReachHere();
3471 } else {
3472 relocInfo::relocType rtype = $src->constant_reloc();
3473 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3474 __ set_narrow_oop(dst_reg, (jobject)con);
3475 }
3476 %}
3477
3478 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3479 C2_MacroAssembler _masm(&cbuf);
3480 Register dst_reg = as_Register($dst$$reg);
3481 __ mov(dst_reg, zr);
3482 %}
3483
3484 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3485 C2_MacroAssembler _masm(&cbuf);
3486 Register dst_reg = as_Register($dst$$reg);
3487 address con = (address)$src$$constant;
3488 if (con == NULL) {
3489 ShouldNotReachHere();
3490 } else {
3491 relocInfo::relocType rtype = $src->constant_reloc();
3492 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3493 __ set_narrow_klass(dst_reg, (Klass *)con);
3494 }
3495 %}
3496
3497 // arithmetic encodings
3498
3499 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3500 C2_MacroAssembler _masm(&cbuf);
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src_reg = as_Register($src1$$reg);
3503 int32_t con = (int32_t)$src2$$constant;
3504 // add has primary == 0, subtract has primary == 1
3505 if ($primary) { con = -con; }
3506 if (con < 0) {
3507 __ subw(dst_reg, src_reg, -con);
3508 } else {
3509 __ addw(dst_reg, src_reg, con);
3510 }
3511 %}
3512
3513 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3514 C2_MacroAssembler _masm(&cbuf);
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src_reg = as_Register($src1$$reg);
3517 int32_t con = (int32_t)$src2$$constant;
3518 // add has primary == 0, subtract has primary == 1
3519 if ($primary) { con = -con; }
3520 if (con < 0) {
3521 __ sub(dst_reg, src_reg, -con);
3522 } else {
3523 __ add(dst_reg, src_reg, con);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3528 C2_MacroAssembler _masm(&cbuf);
3529 Register dst_reg = as_Register($dst$$reg);
3530 Register src1_reg = as_Register($src1$$reg);
3531 Register src2_reg = as_Register($src2$$reg);
3532 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3533 %}
3534
3535 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3536 C2_MacroAssembler _masm(&cbuf);
3537 Register dst_reg = as_Register($dst$$reg);
3538 Register src1_reg = as_Register($src1$$reg);
3539 Register src2_reg = as_Register($src2$$reg);
3540 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3541 %}
3542
3543 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3544 C2_MacroAssembler _masm(&cbuf);
3545 Register dst_reg = as_Register($dst$$reg);
3546 Register src1_reg = as_Register($src1$$reg);
3547 Register src2_reg = as_Register($src2$$reg);
3548 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3552 C2_MacroAssembler _masm(&cbuf);
3553 Register dst_reg = as_Register($dst$$reg);
3554 Register src1_reg = as_Register($src1$$reg);
3555 Register src2_reg = as_Register($src2$$reg);
3556 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3557 %}
3558
3559 // compare instruction encodings
3560
3561 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3562 C2_MacroAssembler _masm(&cbuf);
3563 Register reg1 = as_Register($src1$$reg);
3564 Register reg2 = as_Register($src2$$reg);
3565 __ cmpw(reg1, reg2);
3566 %}
3567
3568 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3569 C2_MacroAssembler _masm(&cbuf);
3570 Register reg = as_Register($src1$$reg);
3571 int32_t val = $src2$$constant;
3572 if (val >= 0) {
3573 __ subsw(zr, reg, val);
3574 } else {
3575 __ addsw(zr, reg, -val);
3576 }
3577 %}
3578
3579 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3580 C2_MacroAssembler _masm(&cbuf);
3581 Register reg1 = as_Register($src1$$reg);
3582 uint32_t val = (uint32_t)$src2$$constant;
3583 __ movw(rscratch1, val);
3584 __ cmpw(reg1, rscratch1);
3585 %}
3586
3587 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3588 C2_MacroAssembler _masm(&cbuf);
3589 Register reg1 = as_Register($src1$$reg);
3590 Register reg2 = as_Register($src2$$reg);
3591 __ cmp(reg1, reg2);
3592 %}
3593
3594 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3595 C2_MacroAssembler _masm(&cbuf);
3596 Register reg = as_Register($src1$$reg);
3597 int64_t val = $src2$$constant;
3598 if (val >= 0) {
3599 __ subs(zr, reg, val);
3600 } else if (val != -val) {
3601 __ adds(zr, reg, -val);
3602 } else {
3603 // aargh, Long.MIN_VALUE is a special case
3604 __ orr(rscratch1, zr, (uint64_t)val);
3605 __ subs(zr, reg, rscratch1);
3606 }
3607 %}
3608
3609 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3610 C2_MacroAssembler _masm(&cbuf);
3611 Register reg1 = as_Register($src1$$reg);
3612 uint64_t val = (uint64_t)$src2$$constant;
3613 __ mov(rscratch1, val);
3614 __ cmp(reg1, rscratch1);
3615 %}
3616
3617 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3618 C2_MacroAssembler _masm(&cbuf);
3619 Register reg1 = as_Register($src1$$reg);
3620 Register reg2 = as_Register($src2$$reg);
3621 __ cmp(reg1, reg2);
3622 %}
3623
3624 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3625 C2_MacroAssembler _masm(&cbuf);
3626 Register reg1 = as_Register($src1$$reg);
3627 Register reg2 = as_Register($src2$$reg);
3628 __ cmpw(reg1, reg2);
3629 %}
3630
3631 enc_class aarch64_enc_testp(iRegP src) %{
3632 C2_MacroAssembler _masm(&cbuf);
3633 Register reg = as_Register($src$$reg);
3634 __ cmp(reg, zr);
3635 %}
3636
3637 enc_class aarch64_enc_testn(iRegN src) %{
3638 C2_MacroAssembler _masm(&cbuf);
3639 Register reg = as_Register($src$$reg);
3640 __ cmpw(reg, zr);
3641 %}
3642
3643 enc_class aarch64_enc_b(label lbl) %{
3644 C2_MacroAssembler _masm(&cbuf);
3645 Label *L = $lbl$$label;
3646 __ b(*L);
3647 %}
3648
3649 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3650 C2_MacroAssembler _masm(&cbuf);
3651 Label *L = $lbl$$label;
3652 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3653 %}
3654
3655 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3656 C2_MacroAssembler _masm(&cbuf);
3657 Label *L = $lbl$$label;
3658 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3659 %}
3660
3661 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3662 %{
3663 Register sub_reg = as_Register($sub$$reg);
3664 Register super_reg = as_Register($super$$reg);
3665 Register temp_reg = as_Register($temp$$reg);
3666 Register result_reg = as_Register($result$$reg);
3667
3668 Label miss;
3669 C2_MacroAssembler _masm(&cbuf);
3670 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3671 NULL, &miss,
3672 /*set_cond_codes:*/ true);
3673 if ($primary) {
3674 __ mov(result_reg, zr);
3675 }
3676 __ bind(miss);
3677 %}
3678
3679 enc_class aarch64_enc_java_static_call(method meth) %{
3680 C2_MacroAssembler _masm(&cbuf);
3681
3682 address addr = (address)$meth$$method;
3683 address call;
3684 if (!_method) {
3685 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3686 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3687 if (call == NULL) {
3688 ciEnv::current()->record_failure("CodeCache is full");
3689 return;
3690 }
3691 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3692 // The NOP here is purely to ensure that eliding a call to
3693 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3694 __ nop();
3695 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3696 } else {
3697 int method_index = resolved_method_index(cbuf);
3698 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3699 : static_call_Relocation::spec(method_index);
3700 call = __ trampoline_call(Address(addr, rspec));
3701 if (call == NULL) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3706 // Calls of the same statically bound method can share
3707 // a stub to the interpreter.
3708 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3709 } else {
3710 // Emit stub for static call
3711 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3712 if (stub == NULL) {
3713 ciEnv::current()->record_failure("CodeCache is full");
3714 return;
3715 }
3716 }
3717 }
3718
3719 __ post_call_nop();
3720
3721 // Only non uncommon_trap calls need to reinitialize ptrue.
3722 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3723 __ reinitialize_ptrue();
3724 }
3725 %}
3726
3727 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3728 C2_MacroAssembler _masm(&cbuf);
3729 int method_index = resolved_method_index(cbuf);
3730 address call = __ ic_call((address)$meth$$method, method_index);
3731 if (call == NULL) {
3732 ciEnv::current()->record_failure("CodeCache is full");
3733 return;
3734 }
3735 __ post_call_nop();
3736 if (Compile::current()->max_vector_size() > 0) {
3737 __ reinitialize_ptrue();
3738 }
3739 %}
3740
3741 enc_class aarch64_enc_call_epilog() %{
3742 C2_MacroAssembler _masm(&cbuf);
3743 if (VerifyStackAtCalls) {
3744 // Check that stack depth is unchanged: find majik cookie on stack
3745 __ call_Unimplemented();
3746 }
3747 %}
3748
3749 enc_class aarch64_enc_java_to_runtime(method meth) %{
3750 C2_MacroAssembler _masm(&cbuf);
3751
3752 // some calls to generated routines (arraycopy code) are scheduled
3753 // by C2 as runtime calls. if so we can call them using a br (they
3754 // will be in a reachable segment) otherwise we have to use a blr
3755 // which loads the absolute address into a register.
3756 address entry = (address)$meth$$method;
3757 CodeBlob *cb = CodeCache::find_blob(entry);
3758 if (cb) {
3759 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3760 if (call == NULL) {
3761 ciEnv::current()->record_failure("CodeCache is full");
3762 return;
3763 }
3764 __ post_call_nop();
3765 } else {
3766 Label retaddr;
3767 __ adr(rscratch2, retaddr);
3768 __ lea(rscratch1, RuntimeAddress(entry));
3769 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3770 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3771 __ blr(rscratch1);
3772 __ bind(retaddr);
3773 __ post_call_nop();
3774 __ add(sp, sp, 2 * wordSize);
3775 }
3776 if (Compile::current()->max_vector_size() > 0) {
3777 __ reinitialize_ptrue();
3778 }
3779 %}
3780
3781 enc_class aarch64_enc_rethrow() %{
3782 C2_MacroAssembler _masm(&cbuf);
3783 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3784 %}
3785
3786 enc_class aarch64_enc_ret() %{
3787 C2_MacroAssembler _masm(&cbuf);
3788 #ifdef ASSERT
3789 if (Compile::current()->max_vector_size() > 0) {
3790 __ verify_ptrue();
3791 }
3792 #endif
3793 __ ret(lr);
3794 %}
3795
3796 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3797 C2_MacroAssembler _masm(&cbuf);
3798 Register target_reg = as_Register($jump_target$$reg);
3799 __ br(target_reg);
3800 %}
3801
3802 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3803 C2_MacroAssembler _masm(&cbuf);
3804 Register target_reg = as_Register($jump_target$$reg);
3805 // exception oop should be in r0
3806 // ret addr has been popped into lr
3807 // callee expects it in r3
3808 __ mov(r3, lr);
3809 __ br(target_reg);
3810 %}
3811
3812 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3813 C2_MacroAssembler _masm(&cbuf);
3814 Register oop = as_Register($object$$reg);
3815 Register box = as_Register($box$$reg);
3816 Register disp_hdr = as_Register($tmp$$reg);
3817 Register tmp = as_Register($tmp2$$reg);
3818 Label cont;
3819 Label object_has_monitor;
3820 Label count, no_count;
3821
3822 assert_different_registers(oop, box, tmp, disp_hdr);
3823
3824 // Load markWord from object into displaced_header.
3825 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3826
3827 if (DiagnoseSyncOnValueBasedClasses != 0) {
3828 __ load_klass(tmp, oop);
3829 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3830 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3831 __ br(Assembler::NE, cont);
3832 }
3833
3834 // Check for existing monitor
3835 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3836
3837 if (LockingMode == LM_MONITOR) {
3838 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3839 __ b(cont);
3840 } else if (LockingMode == LM_LEGACY) {
3841 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3842 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3843
3844 // Initialize the box. (Must happen before we update the object mark!)
3845 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3846
3847 // Compare object markWord with an unlocked value (tmp) and if
3848 // equal exchange the stack address of our box with object markWord.
3849 // On failure disp_hdr contains the possibly locked markWord.
3850 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3851 /*release*/ true, /*weak*/ false, disp_hdr);
3852 __ br(Assembler::EQ, cont);
3853
3854 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3855
3856 // If the compare-and-exchange succeeded, then we found an unlocked
3857 // object, will have now locked it will continue at label cont
3858
3859 // Check if the owner is self by comparing the value in the
3860 // markWord of object (disp_hdr) with the stack pointer.
3861 __ mov(rscratch1, sp);
3862 __ sub(disp_hdr, disp_hdr, rscratch1);
3863 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3864 // If condition is true we are cont and hence we can store 0 as the
3865 // displaced header in the box, which indicates that it is a recursive lock.
3866 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3867 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3868 __ b(cont);
3869 } else {
3870 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3871 __ fast_lock(oop, disp_hdr, tmp, rscratch1, no_count);
3872 __ b(count);
3873 }
3874
3875 // Handle existing monitor.
3876 __ bind(object_has_monitor);
3877
3878 // The object's monitor m is unlocked iff m->owner == NULL,
3879 // otherwise m->owner may contain a thread or a stack address.
3880 //
3881 // Try to CAS m->owner from NULL to current thread.
3882 __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value));
3883 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3884 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3885
3886 if (LockingMode != LM_LIGHTWEIGHT) {
3887 // Store a non-null value into the box to avoid looking like a re-entrant
3888 // lock. The fast-path monitor unlock code checks for
3889 // markWord::monitor_value so use markWord::unused_mark which has the
3890 // relevant bit set, and also matches ObjectSynchronizer::enter.
3891 __ mov(tmp, (address)markWord::unused_mark().value());
3892 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3893 }
3894 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3895
3896 __ cmp(rscratch1, rthread);
3897 __ br(Assembler::NE, cont); // Check for recursive locking
3898
3899 // Recursive lock case
3900 __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1);
3901 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3902
3903 __ bind(cont);
3904 // flag == EQ indicates success
3905 // flag == NE indicates failure
3906 __ br(Assembler::NE, no_count);
3907
3908 __ bind(count);
3909 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3910
3911 __ bind(no_count);
3912 %}
3913
3914 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3915 C2_MacroAssembler _masm(&cbuf);
3916 Register oop = as_Register($object$$reg);
3917 Register box = as_Register($box$$reg);
3918 Register disp_hdr = as_Register($tmp$$reg);
3919 Register tmp = as_Register($tmp2$$reg);
3920 Label cont;
3921 Label object_has_monitor;
3922 Label count, no_count;
3923
3924 assert_different_registers(oop, box, tmp, disp_hdr);
3925
3926 if (LockingMode == LM_LEGACY) {
3927 // Find the lock address and load the displaced header from the stack.
3928 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3929
3930 // If the displaced header is 0, we have a recursive unlock.
3931 __ cmp(disp_hdr, zr);
3932 __ br(Assembler::EQ, cont);
3933 }
3934
3935 // Handle existing monitor.
3936 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3937 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3938
3939 if (LockingMode == LM_MONITOR) {
3940 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3941 __ b(cont);
3942 } else if (LockingMode == LM_LEGACY) {
3943 // Check if it is still a light weight lock, this is is true if we
3944 // see the stack address of the basicLock in the markWord of the
3945 // object.
3946
3947 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3948 /*release*/ true, /*weak*/ false, tmp);
3949 __ b(cont);
3950 } else {
3951 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3952 __ fast_unlock(oop, tmp, box, disp_hdr, no_count);
3953 __ b(count);
3954 }
3955
3956 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3957
3958 // Handle existing monitor.
3959 __ bind(object_has_monitor);
3960 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3961 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3962
3963 if (LockingMode == LM_LIGHTWEIGHT) {
3964 // If the owner is anonymous, we need to fix it -- in an outline stub.
3965 Register tmp2 = disp_hdr;
3966 __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
3967 // We cannot use tbnz here, the target might be too far away and cannot
3968 // be encoded.
3969 __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
3970 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
3971 Compile::current()->output()->add_stub(stub);
3972 __ br(Assembler::NE, stub->entry());
3973 __ bind(stub->continuation());
3974 }
3975
3976 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
3977
3978 Label notRecursive;
3979 __ cbz(disp_hdr, notRecursive);
3980
3981 // Recursive lock
3982 __ sub(disp_hdr, disp_hdr, 1u);
3983 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
3984 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3985 __ b(cont);
3986
3987 __ bind(notRecursive);
3988 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
3989 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
3990 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3991 __ cmp(rscratch1, zr); // Sets flags for result
3992 __ cbnz(rscratch1, cont);
3993 // need a release store here
3994 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
3995 __ stlr(zr, tmp); // set unowned
3996
3997 __ bind(cont);
3998 // flag == EQ indicates success
3999 // flag == NE indicates failure
4000 __ br(Assembler::NE, no_count);
4001
4002 __ bind(count);
4003 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
4004
4005 __ bind(no_count);
4006 %}
4007
4008 %}
4009
4010 //----------FRAME--------------------------------------------------------------
4011 // Definition of frame structure and management information.
4012 //
4013 // S T A C K L A Y O U T Allocators stack-slot number
4014 // | (to get allocators register number
4015 // G Owned by | | v add OptoReg::stack0())
4016 // r CALLER | |
4017 // o | +--------+ pad to even-align allocators stack-slot
4018 // w V | pad0 | numbers; owned by CALLER
4019 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4020 // h ^ | in | 5
4021 // | | args | 4 Holes in incoming args owned by SELF
4022 // | | | | 3
4023 // | | +--------+
4024 // V | | old out| Empty on Intel, window on Sparc
4025 // | old |preserve| Must be even aligned.
4026 // | SP-+--------+----> Matcher::_old_SP, even aligned
4027 // | | in | 3 area for Intel ret address
4028 // Owned by |preserve| Empty on Sparc.
4029 // SELF +--------+
4030 // | | pad2 | 2 pad to align old SP
4031 // | +--------+ 1
4032 // | | locks | 0
4033 // | +--------+----> OptoReg::stack0(), even aligned
4034 // | | pad1 | 11 pad to align new SP
4035 // | +--------+
4036 // | | | 10
4037 // | | spills | 9 spills
4038 // V | | 8 (pad0 slot for callee)
4039 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4040 // ^ | out | 7
4041 // | | args | 6 Holes in outgoing args owned by CALLEE
4042 // Owned by +--------+
4043 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4044 // | new |preserve| Must be even-aligned.
4045 // | SP-+--------+----> Matcher::_new_SP, even aligned
4046 // | | |
4047 //
4048 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4049 // known from SELF's arguments and the Java calling convention.
4050 // Region 6-7 is determined per call site.
4051 // Note 2: If the calling convention leaves holes in the incoming argument
4052 // area, those holes are owned by SELF. Holes in the outgoing area
4053 // are owned by the CALLEE. Holes should not be necessary in the
4054 // incoming area, as the Java calling convention is completely under
4055 // the control of the AD file. Doubles can be sorted and packed to
4056 // avoid holes. Holes in the outgoing arguments may be necessary for
4057 // varargs C calling conventions.
4058 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4059 // even aligned with pad0 as needed.
4060 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4061 // (the latter is true on Intel but is it false on AArch64?)
4062 // region 6-11 is even aligned; it may be padded out more so that
4063 // the region from SP to FP meets the minimum stack alignment.
4064 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4065 // alignment. Region 11, pad1, may be dynamically extended so that
4066 // SP meets the minimum alignment.
4067
4068 frame %{
4069 // These three registers define part of the calling convention
4070 // between compiled code and the interpreter.
4071
4072 // Inline Cache Register or Method for I2C.
4073 inline_cache_reg(R12);
4074
4075 // Number of stack slots consumed by locking an object
4076 sync_stack_slots(2);
4077
4078 // Compiled code's Frame Pointer
4079 frame_pointer(R31);
4080
4081 // Interpreter stores its frame pointer in a register which is
4082 // stored to the stack by I2CAdaptors.
4083 // I2CAdaptors convert from interpreted java to compiled java.
4084 interpreter_frame_pointer(R29);
4085
4086 // Stack alignment requirement
4087 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4088
4089 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4090 // for calls to C. Supports the var-args backing area for register parms.
4091 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4092
4093 // The after-PROLOG location of the return address. Location of
4094 // return address specifies a type (REG or STACK) and a number
4095 // representing the register number (i.e. - use a register name) or
4096 // stack slot.
4097 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4098 // Otherwise, it is above the locks and verification slot and alignment word
4099 // TODO this may well be correct but need to check why that - 2 is there
4100 // ppc port uses 0 but we definitely need to allow for fixed_slots
4101 // which folds in the space used for monitors
4102 return_addr(STACK - 2 +
4103 align_up((Compile::current()->in_preserve_stack_slots() +
4104 Compile::current()->fixed_slots()),
4105 stack_alignment_in_slots()));
4106
4107 // Location of compiled Java return values. Same as C for now.
4108 return_value
4109 %{
4110 // TODO do we allow ideal_reg == Op_RegN???
4111 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4112 "only return normal values");
4113
4114 static const int lo[Op_RegL + 1] = { // enum name
4115 0, // Op_Node
4116 0, // Op_Set
4117 R0_num, // Op_RegN
4118 R0_num, // Op_RegI
4119 R0_num, // Op_RegP
4120 V0_num, // Op_RegF
4121 V0_num, // Op_RegD
4122 R0_num // Op_RegL
4123 };
4124
4125 static const int hi[Op_RegL + 1] = { // enum name
4126 0, // Op_Node
4127 0, // Op_Set
4128 OptoReg::Bad, // Op_RegN
4129 OptoReg::Bad, // Op_RegI
4130 R0_H_num, // Op_RegP
4131 OptoReg::Bad, // Op_RegF
4132 V0_H_num, // Op_RegD
4133 R0_H_num // Op_RegL
4134 };
4135
4136 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4137 %}
4138 %}
4139
4140 //----------ATTRIBUTES---------------------------------------------------------
4141 //----------Operand Attributes-------------------------------------------------
4142 op_attrib op_cost(1); // Required cost attribute
4143
4144 //----------Instruction Attributes---------------------------------------------
4145 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4146 ins_attrib ins_size(32); // Required size attribute (in bits)
4147 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4148 // a non-matching short branch variant
4149 // of some long branch?
4150 ins_attrib ins_alignment(4); // Required alignment attribute (must
4151 // be a power of 2) specifies the
4152 // alignment that some part of the
4153 // instruction (not necessarily the
4154 // start) requires. If > 1, a
4155 // compute_padding() function must be
4156 // provided for the instruction
4157
4158 //----------OPERANDS-----------------------------------------------------------
4159 // Operand definitions must precede instruction definitions for correct parsing
4160 // in the ADLC because operands constitute user defined types which are used in
4161 // instruction definitions.
4162
4163 //----------Simple Operands----------------------------------------------------
4164
4165 // Integer operands 32 bit
4166 // 32 bit immediate
4167 operand immI()
4168 %{
4169 match(ConI);
4170
4171 op_cost(0);
4172 format %{ %}
4173 interface(CONST_INTER);
4174 %}
4175
4176 // 32 bit zero
4177 operand immI0()
4178 %{
4179 predicate(n->get_int() == 0);
4180 match(ConI);
4181
4182 op_cost(0);
4183 format %{ %}
4184 interface(CONST_INTER);
4185 %}
4186
4187 // 32 bit unit increment
4188 operand immI_1()
4189 %{
4190 predicate(n->get_int() == 1);
4191 match(ConI);
4192
4193 op_cost(0);
4194 format %{ %}
4195 interface(CONST_INTER);
4196 %}
4197
4198 // 32 bit unit decrement
4199 operand immI_M1()
4200 %{
4201 predicate(n->get_int() == -1);
4202 match(ConI);
4203
4204 op_cost(0);
4205 format %{ %}
4206 interface(CONST_INTER);
4207 %}
4208
4209 // Shift values for add/sub extension shift
4210 operand immIExt()
4211 %{
4212 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immI_gt_1()
4221 %{
4222 predicate(n->get_int() > 1);
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immI_le_4()
4231 %{
4232 predicate(n->get_int() <= 4);
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immI_16()
4241 %{
4242 predicate(n->get_int() == 16);
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immI_24()
4251 %{
4252 predicate(n->get_int() == 24);
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immI_32()
4261 %{
4262 predicate(n->get_int() == 32);
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immI_48()
4271 %{
4272 predicate(n->get_int() == 48);
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immI_56()
4281 %{
4282 predicate(n->get_int() == 56);
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immI_63()
4291 %{
4292 predicate(n->get_int() == 63);
4293 match(ConI);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immI_64()
4301 %{
4302 predicate(n->get_int() == 64);
4303 match(ConI);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immI_255()
4311 %{
4312 predicate(n->get_int() == 255);
4313 match(ConI);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immI_65535()
4321 %{
4322 predicate(n->get_int() == 65535);
4323 match(ConI);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immI_positive()
4331 %{
4332 predicate(n->get_int() > 0);
4333 match(ConI);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 // BoolTest condition for signed compare
4341 operand immI_cmp_cond()
4342 %{
4343 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // BoolTest condition for unsigned compare
4352 operand immI_cmpU_cond()
4353 %{
4354 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4355 match(ConI);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 operand immL_255()
4363 %{
4364 predicate(n->get_long() == 255L);
4365 match(ConL);
4366
4367 op_cost(0);
4368 format %{ %}
4369 interface(CONST_INTER);
4370 %}
4371
4372 operand immL_65535()
4373 %{
4374 predicate(n->get_long() == 65535L);
4375 match(ConL);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 operand immL_4294967295()
4383 %{
4384 predicate(n->get_long() == 4294967295L);
4385 match(ConL);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 operand immL_bitmask()
4393 %{
4394 predicate((n->get_long() != 0)
4395 && ((n->get_long() & 0xc000000000000000l) == 0)
4396 && is_power_of_2(n->get_long() + 1));
4397 match(ConL);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 operand immI_bitmask()
4405 %{
4406 predicate((n->get_int() != 0)
4407 && ((n->get_int() & 0xc0000000) == 0)
4408 && is_power_of_2(n->get_int() + 1));
4409 match(ConI);
4410
4411 op_cost(0);
4412 format %{ %}
4413 interface(CONST_INTER);
4414 %}
4415
4416 operand immL_positive_bitmaskI()
4417 %{
4418 predicate((n->get_long() != 0)
4419 && ((julong)n->get_long() < 0x80000000ULL)
4420 && is_power_of_2(n->get_long() + 1));
4421 match(ConL);
4422
4423 op_cost(0);
4424 format %{ %}
4425 interface(CONST_INTER);
4426 %}
4427
4428 // Scale values for scaled offset addressing modes (up to long but not quad)
4429 operand immIScale()
4430 %{
4431 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 // 26 bit signed offset -- for pc-relative branches
4440 operand immI26()
4441 %{
4442 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4443 match(ConI);
4444
4445 op_cost(0);
4446 format %{ %}
4447 interface(CONST_INTER);
4448 %}
4449
4450 // 19 bit signed offset -- for pc-relative loads
4451 operand immI19()
4452 %{
4453 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4454 match(ConI);
4455
4456 op_cost(0);
4457 format %{ %}
4458 interface(CONST_INTER);
4459 %}
4460
4461 // 5 bit signed integer
4462 operand immI5()
4463 %{
4464 predicate(Assembler::is_simm(n->get_int(), 5));
4465 match(ConI);
4466
4467 op_cost(0);
4468 format %{ %}
4469 interface(CONST_INTER);
4470 %}
4471
4472 // 7 bit unsigned integer
4473 operand immIU7()
4474 %{
4475 predicate(Assembler::is_uimm(n->get_int(), 7));
4476 match(ConI);
4477
4478 op_cost(0);
4479 format %{ %}
4480 interface(CONST_INTER);
4481 %}
4482
4483 // 12 bit unsigned offset -- for base plus immediate loads
4484 operand immIU12()
4485 %{
4486 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4487 match(ConI);
4488
4489 op_cost(0);
4490 format %{ %}
4491 interface(CONST_INTER);
4492 %}
4493
4494 operand immLU12()
4495 %{
4496 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4497 match(ConL);
4498
4499 op_cost(0);
4500 format %{ %}
4501 interface(CONST_INTER);
4502 %}
4503
4504 // Offset for scaled or unscaled immediate loads and stores
4505 operand immIOffset()
4506 %{
4507 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4508 match(ConI);
4509
4510 op_cost(0);
4511 format %{ %}
4512 interface(CONST_INTER);
4513 %}
4514
4515 operand immIOffset1()
4516 %{
4517 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4518 match(ConI);
4519
4520 op_cost(0);
4521 format %{ %}
4522 interface(CONST_INTER);
4523 %}
4524
4525 operand immIOffset2()
4526 %{
4527 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4528 match(ConI);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 operand immIOffset4()
4536 %{
4537 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4538 match(ConI);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 operand immIOffset8()
4546 %{
4547 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4548 match(ConI);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 operand immIOffset16()
4556 %{
4557 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4558 match(ConI);
4559
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 operand immLoffset()
4566 %{
4567 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4568 match(ConL);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 operand immLoffset1()
4576 %{
4577 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4578 match(ConL);
4579
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 operand immLoffset2()
4586 %{
4587 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4588 match(ConL);
4589
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 operand immLoffset4()
4596 %{
4597 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4598 match(ConL);
4599
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 operand immLoffset8()
4606 %{
4607 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4608 match(ConL);
4609
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 operand immLoffset16()
4616 %{
4617 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4618 match(ConL);
4619
4620 op_cost(0);
4621 format %{ %}
4622 interface(CONST_INTER);
4623 %}
4624
4625 // 5 bit signed long integer
4626 operand immL5()
4627 %{
4628 predicate(Assembler::is_simm(n->get_long(), 5));
4629 match(ConL);
4630
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // 7 bit unsigned long integer
4637 operand immLU7()
4638 %{
4639 predicate(Assembler::is_uimm(n->get_long(), 7));
4640 match(ConL);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // 8 bit signed value.
4648 operand immI8()
4649 %{
4650 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4651 match(ConI);
4652
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // 8 bit signed value (simm8), or #simm8 LSL 8.
4659 operand immI8_shift8()
4660 %{
4661 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4662 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4663 match(ConI);
4664
4665 op_cost(0);
4666 format %{ %}
4667 interface(CONST_INTER);
4668 %}
4669
4670 // 8 bit signed value (simm8), or #simm8 LSL 8.
4671 operand immL8_shift8()
4672 %{
4673 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4674 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4675 match(ConL);
4676
4677 op_cost(0);
4678 format %{ %}
4679 interface(CONST_INTER);
4680 %}
4681
4682 // 8 bit integer valid for vector add sub immediate
4683 operand immBAddSubV()
4684 %{
4685 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4686 match(ConI);
4687
4688 op_cost(0);
4689 format %{ %}
4690 interface(CONST_INTER);
4691 %}
4692
4693 // 32 bit integer valid for add sub immediate
4694 operand immIAddSub()
4695 %{
4696 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4697 match(ConI);
4698 op_cost(0);
4699 format %{ %}
4700 interface(CONST_INTER);
4701 %}
4702
4703 // 32 bit integer valid for vector add sub immediate
4704 operand immIAddSubV()
4705 %{
4706 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4707 match(ConI);
4708
4709 op_cost(0);
4710 format %{ %}
4711 interface(CONST_INTER);
4712 %}
4713
4714 // 32 bit unsigned integer valid for logical immediate
4715
4716 operand immBLog()
4717 %{
4718 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4719 match(ConI);
4720
4721 op_cost(0);
4722 format %{ %}
4723 interface(CONST_INTER);
4724 %}
4725
4726 operand immSLog()
4727 %{
4728 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4729 match(ConI);
4730
4731 op_cost(0);
4732 format %{ %}
4733 interface(CONST_INTER);
4734 %}
4735
4736 operand immILog()
4737 %{
4738 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4739 match(ConI);
4740
4741 op_cost(0);
4742 format %{ %}
4743 interface(CONST_INTER);
4744 %}
4745
4746 // Integer operands 64 bit
4747 // 64 bit immediate
4748 operand immL()
4749 %{
4750 match(ConL);
4751
4752 op_cost(0);
4753 format %{ %}
4754 interface(CONST_INTER);
4755 %}
4756
4757 // 64 bit zero
4758 operand immL0()
4759 %{
4760 predicate(n->get_long() == 0);
4761 match(ConL);
4762
4763 op_cost(0);
4764 format %{ %}
4765 interface(CONST_INTER);
4766 %}
4767
4768 // 64 bit unit increment
4769 operand immL_1()
4770 %{
4771 predicate(n->get_long() == 1);
4772 match(ConL);
4773
4774 op_cost(0);
4775 format %{ %}
4776 interface(CONST_INTER);
4777 %}
4778
4779 // 64 bit unit decrement
4780 operand immL_M1()
4781 %{
4782 predicate(n->get_long() == -1);
4783 match(ConL);
4784
4785 op_cost(0);
4786 format %{ %}
4787 interface(CONST_INTER);
4788 %}
4789
4790 // 32 bit offset of pc in thread anchor
4791
4792 operand immL_pc_off()
4793 %{
4794 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4795 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4796 match(ConL);
4797
4798 op_cost(0);
4799 format %{ %}
4800 interface(CONST_INTER);
4801 %}
4802
4803 // 64 bit integer valid for add sub immediate
4804 operand immLAddSub()
4805 %{
4806 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4807 match(ConL);
4808 op_cost(0);
4809 format %{ %}
4810 interface(CONST_INTER);
4811 %}
4812
4813 // 64 bit integer valid for addv subv immediate
4814 operand immLAddSubV()
4815 %{
4816 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4817 match(ConL);
4818
4819 op_cost(0);
4820 format %{ %}
4821 interface(CONST_INTER);
4822 %}
4823
4824 // 64 bit integer valid for logical immediate
4825 operand immLLog()
4826 %{
4827 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4828 match(ConL);
4829 op_cost(0);
4830 format %{ %}
4831 interface(CONST_INTER);
4832 %}
4833
4834 // Long Immediate: low 32-bit mask
4835 operand immL_32bits()
4836 %{
4837 predicate(n->get_long() == 0xFFFFFFFFL);
4838 match(ConL);
4839 op_cost(0);
4840 format %{ %}
4841 interface(CONST_INTER);
4842 %}
4843
4844 // Pointer operands
4845 // Pointer Immediate
4846 operand immP()
4847 %{
4848 match(ConP);
4849
4850 op_cost(0);
4851 format %{ %}
4852 interface(CONST_INTER);
4853 %}
4854
4855 // NULL Pointer Immediate
4856 operand immP0()
4857 %{
4858 predicate(n->get_ptr() == 0);
4859 match(ConP);
4860
4861 op_cost(0);
4862 format %{ %}
4863 interface(CONST_INTER);
4864 %}
4865
4866 // Pointer Immediate One
4867 // this is used in object initialization (initial object header)
4868 operand immP_1()
4869 %{
4870 predicate(n->get_ptr() == 1);
4871 match(ConP);
4872
4873 op_cost(0);
4874 format %{ %}
4875 interface(CONST_INTER);
4876 %}
4877
4878 // Card Table Byte Map Base
4879 operand immByteMapBase()
4880 %{
4881 // Get base of card map
4882 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4883 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4884 match(ConP);
4885
4886 op_cost(0);
4887 format %{ %}
4888 interface(CONST_INTER);
4889 %}
4890
4891 // Pointer Immediate Minus One
4892 // this is used when we want to write the current PC to the thread anchor
4893 operand immP_M1()
4894 %{
4895 predicate(n->get_ptr() == -1);
4896 match(ConP);
4897
4898 op_cost(0);
4899 format %{ %}
4900 interface(CONST_INTER);
4901 %}
4902
4903 // Pointer Immediate Minus Two
4904 // this is used when we want to write the current PC to the thread anchor
4905 operand immP_M2()
4906 %{
4907 predicate(n->get_ptr() == -2);
4908 match(ConP);
4909
4910 op_cost(0);
4911 format %{ %}
4912 interface(CONST_INTER);
4913 %}
4914
4915 // Float and Double operands
4916 // Double Immediate
4917 operand immD()
4918 %{
4919 match(ConD);
4920 op_cost(0);
4921 format %{ %}
4922 interface(CONST_INTER);
4923 %}
4924
4925 // Double Immediate: +0.0d
4926 operand immD0()
4927 %{
4928 predicate(jlong_cast(n->getd()) == 0);
4929 match(ConD);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(CONST_INTER);
4934 %}
4935
4936 // constant 'double +0.0'.
4937 operand immDPacked()
4938 %{
4939 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4940 match(ConD);
4941 op_cost(0);
4942 format %{ %}
4943 interface(CONST_INTER);
4944 %}
4945
4946 // Float Immediate
4947 operand immF()
4948 %{
4949 match(ConF);
4950 op_cost(0);
4951 format %{ %}
4952 interface(CONST_INTER);
4953 %}
4954
4955 // Float Immediate: +0.0f.
4956 operand immF0()
4957 %{
4958 predicate(jint_cast(n->getf()) == 0);
4959 match(ConF);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(CONST_INTER);
4964 %}
4965
4966 //
4967 operand immFPacked()
4968 %{
4969 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4970 match(ConF);
4971 op_cost(0);
4972 format %{ %}
4973 interface(CONST_INTER);
4974 %}
4975
4976 // Narrow pointer operands
4977 // Narrow Pointer Immediate
4978 operand immN()
4979 %{
4980 match(ConN);
4981
4982 op_cost(0);
4983 format %{ %}
4984 interface(CONST_INTER);
4985 %}
4986
4987 // Narrow NULL Pointer Immediate
4988 operand immN0()
4989 %{
4990 predicate(n->get_narrowcon() == 0);
4991 match(ConN);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(CONST_INTER);
4996 %}
4997
4998 operand immNKlass()
4999 %{
5000 match(ConNKlass);
5001
5002 op_cost(0);
5003 format %{ %}
5004 interface(CONST_INTER);
5005 %}
5006
5007 // Integer 32 bit Register Operands
5008 // Integer 32 bitRegister (excludes SP)
5009 operand iRegI()
5010 %{
5011 constraint(ALLOC_IN_RC(any_reg32));
5012 match(RegI);
5013 match(iRegINoSp);
5014 op_cost(0);
5015 format %{ %}
5016 interface(REG_INTER);
5017 %}
5018
5019 // Integer 32 bit Register not Special
5020 operand iRegINoSp()
5021 %{
5022 constraint(ALLOC_IN_RC(no_special_reg32));
5023 match(RegI);
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 // Integer 64 bit Register Operands
5030 // Integer 64 bit Register (includes SP)
5031 operand iRegL()
5032 %{
5033 constraint(ALLOC_IN_RC(any_reg));
5034 match(RegL);
5035 match(iRegLNoSp);
5036 op_cost(0);
5037 format %{ %}
5038 interface(REG_INTER);
5039 %}
5040
5041 // Integer 64 bit Register not Special
5042 operand iRegLNoSp()
5043 %{
5044 constraint(ALLOC_IN_RC(no_special_reg));
5045 match(RegL);
5046 match(iRegL_R0);
5047 format %{ %}
5048 interface(REG_INTER);
5049 %}
5050
5051 // Pointer Register Operands
5052 // Pointer Register
5053 operand iRegP()
5054 %{
5055 constraint(ALLOC_IN_RC(ptr_reg));
5056 match(RegP);
5057 match(iRegPNoSp);
5058 match(iRegP_R0);
5059 //match(iRegP_R2);
5060 //match(iRegP_R4);
5061 match(iRegP_R5);
5062 match(thread_RegP);
5063 op_cost(0);
5064 format %{ %}
5065 interface(REG_INTER);
5066 %}
5067
5068 // Pointer 64 bit Register not Special
5069 operand iRegPNoSp()
5070 %{
5071 constraint(ALLOC_IN_RC(no_special_ptr_reg));
5072 match(RegP);
5073 // match(iRegP);
5074 // match(iRegP_R0);
5075 // match(iRegP_R2);
5076 // match(iRegP_R4);
5077 // match(iRegP_R5);
5078 // match(thread_RegP);
5079 op_cost(0);
5080 format %{ %}
5081 interface(REG_INTER);
5082 %}
5083
5084 // Pointer 64 bit Register R0 only
5085 operand iRegP_R0()
5086 %{
5087 constraint(ALLOC_IN_RC(r0_reg));
5088 match(RegP);
5089 // match(iRegP);
5090 match(iRegPNoSp);
5091 op_cost(0);
5092 format %{ %}
5093 interface(REG_INTER);
5094 %}
5095
5096 // Pointer 64 bit Register R1 only
5097 operand iRegP_R1()
5098 %{
5099 constraint(ALLOC_IN_RC(r1_reg));
5100 match(RegP);
5101 // match(iRegP);
5102 match(iRegPNoSp);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 // Pointer 64 bit Register R2 only
5109 operand iRegP_R2()
5110 %{
5111 constraint(ALLOC_IN_RC(r2_reg));
5112 match(RegP);
5113 // match(iRegP);
5114 match(iRegPNoSp);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 // Pointer 64 bit Register R3 only
5121 operand iRegP_R3()
5122 %{
5123 constraint(ALLOC_IN_RC(r3_reg));
5124 match(RegP);
5125 // match(iRegP);
5126 match(iRegPNoSp);
5127 op_cost(0);
5128 format %{ %}
5129 interface(REG_INTER);
5130 %}
5131
5132 // Pointer 64 bit Register R4 only
5133 operand iRegP_R4()
5134 %{
5135 constraint(ALLOC_IN_RC(r4_reg));
5136 match(RegP);
5137 // match(iRegP);
5138 match(iRegPNoSp);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 // Pointer 64 bit Register R5 only
5145 operand iRegP_R5()
5146 %{
5147 constraint(ALLOC_IN_RC(r5_reg));
5148 match(RegP);
5149 // match(iRegP);
5150 match(iRegPNoSp);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 // Pointer 64 bit Register R10 only
5157 operand iRegP_R10()
5158 %{
5159 constraint(ALLOC_IN_RC(r10_reg));
5160 match(RegP);
5161 // match(iRegP);
5162 match(iRegPNoSp);
5163 op_cost(0);
5164 format %{ %}
5165 interface(REG_INTER);
5166 %}
5167
5168 // Long 64 bit Register R0 only
5169 operand iRegL_R0()
5170 %{
5171 constraint(ALLOC_IN_RC(r0_reg));
5172 match(RegL);
5173 match(iRegLNoSp);
5174 op_cost(0);
5175 format %{ %}
5176 interface(REG_INTER);
5177 %}
5178
5179 // Long 64 bit Register R2 only
5180 operand iRegL_R2()
5181 %{
5182 constraint(ALLOC_IN_RC(r2_reg));
5183 match(RegL);
5184 match(iRegLNoSp);
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 // Long 64 bit Register R3 only
5191 operand iRegL_R3()
5192 %{
5193 constraint(ALLOC_IN_RC(r3_reg));
5194 match(RegL);
5195 match(iRegLNoSp);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 // Long 64 bit Register R11 only
5202 operand iRegL_R11()
5203 %{
5204 constraint(ALLOC_IN_RC(r11_reg));
5205 match(RegL);
5206 match(iRegLNoSp);
5207 op_cost(0);
5208 format %{ %}
5209 interface(REG_INTER);
5210 %}
5211
5212 // Pointer 64 bit Register FP only
5213 operand iRegP_FP()
5214 %{
5215 constraint(ALLOC_IN_RC(fp_reg));
5216 match(RegP);
5217 // match(iRegP);
5218 op_cost(0);
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Register R0 only
5224 operand iRegI_R0()
5225 %{
5226 constraint(ALLOC_IN_RC(int_r0_reg));
5227 match(RegI);
5228 match(iRegINoSp);
5229 op_cost(0);
5230 format %{ %}
5231 interface(REG_INTER);
5232 %}
5233
5234 // Register R2 only
5235 operand iRegI_R2()
5236 %{
5237 constraint(ALLOC_IN_RC(int_r2_reg));
5238 match(RegI);
5239 match(iRegINoSp);
5240 op_cost(0);
5241 format %{ %}
5242 interface(REG_INTER);
5243 %}
5244
5245 // Register R3 only
5246 operand iRegI_R3()
5247 %{
5248 constraint(ALLOC_IN_RC(int_r3_reg));
5249 match(RegI);
5250 match(iRegINoSp);
5251 op_cost(0);
5252 format %{ %}
5253 interface(REG_INTER);
5254 %}
5255
5256
5257 // Register R4 only
5258 operand iRegI_R4()
5259 %{
5260 constraint(ALLOC_IN_RC(int_r4_reg));
5261 match(RegI);
5262 match(iRegINoSp);
5263 op_cost(0);
5264 format %{ %}
5265 interface(REG_INTER);
5266 %}
5267
5268
5269 // Pointer Register Operands
5270 // Narrow Pointer Register
5271 operand iRegN()
5272 %{
5273 constraint(ALLOC_IN_RC(any_reg32));
5274 match(RegN);
5275 match(iRegNNoSp);
5276 op_cost(0);
5277 format %{ %}
5278 interface(REG_INTER);
5279 %}
5280
5281 operand iRegN_R0()
5282 %{
5283 constraint(ALLOC_IN_RC(r0_reg));
5284 match(iRegN);
5285 op_cost(0);
5286 format %{ %}
5287 interface(REG_INTER);
5288 %}
5289
5290 operand iRegN_R2()
5291 %{
5292 constraint(ALLOC_IN_RC(r2_reg));
5293 match(iRegN);
5294 op_cost(0);
5295 format %{ %}
5296 interface(REG_INTER);
5297 %}
5298
5299 operand iRegN_R3()
5300 %{
5301 constraint(ALLOC_IN_RC(r3_reg));
5302 match(iRegN);
5303 op_cost(0);
5304 format %{ %}
5305 interface(REG_INTER);
5306 %}
5307
5308 // Integer 64 bit Register not Special
5309 operand iRegNNoSp()
5310 %{
5311 constraint(ALLOC_IN_RC(no_special_reg32));
5312 match(RegN);
5313 op_cost(0);
5314 format %{ %}
5315 interface(REG_INTER);
5316 %}
5317
5318 // Float Register
5319 // Float register operands
5320 operand vRegF()
5321 %{
5322 constraint(ALLOC_IN_RC(float_reg));
5323 match(RegF);
5324
5325 op_cost(0);
5326 format %{ %}
5327 interface(REG_INTER);
5328 %}
5329
5330 // Double Register
5331 // Double register operands
5332 operand vRegD()
5333 %{
5334 constraint(ALLOC_IN_RC(double_reg));
5335 match(RegD);
5336
5337 op_cost(0);
5338 format %{ %}
5339 interface(REG_INTER);
5340 %}
5341
5342 // Generic vector class. This will be used for
5343 // all vector operands, including NEON and SVE.
5344 operand vReg()
5345 %{
5346 constraint(ALLOC_IN_RC(dynamic));
5347 match(VecA);
5348 match(VecD);
5349 match(VecX);
5350
5351 op_cost(0);
5352 format %{ %}
5353 interface(REG_INTER);
5354 %}
5355
5356 operand vecA()
5357 %{
5358 constraint(ALLOC_IN_RC(vectora_reg));
5359 match(VecA);
5360
5361 op_cost(0);
5362 format %{ %}
5363 interface(REG_INTER);
5364 %}
5365
5366 operand vecD()
5367 %{
5368 constraint(ALLOC_IN_RC(vectord_reg));
5369 match(VecD);
5370
5371 op_cost(0);
5372 format %{ %}
5373 interface(REG_INTER);
5374 %}
5375
5376 operand vecX()
5377 %{
5378 constraint(ALLOC_IN_RC(vectorx_reg));
5379 match(VecX);
5380
5381 op_cost(0);
5382 format %{ %}
5383 interface(REG_INTER);
5384 %}
5385
5386 operand vRegD_V0()
5387 %{
5388 constraint(ALLOC_IN_RC(v0_reg));
5389 match(RegD);
5390 op_cost(0);
5391 format %{ %}
5392 interface(REG_INTER);
5393 %}
5394
5395 operand vRegD_V1()
5396 %{
5397 constraint(ALLOC_IN_RC(v1_reg));
5398 match(RegD);
5399 op_cost(0);
5400 format %{ %}
5401 interface(REG_INTER);
5402 %}
5403
5404 operand vRegD_V2()
5405 %{
5406 constraint(ALLOC_IN_RC(v2_reg));
5407 match(RegD);
5408 op_cost(0);
5409 format %{ %}
5410 interface(REG_INTER);
5411 %}
5412
5413 operand vRegD_V3()
5414 %{
5415 constraint(ALLOC_IN_RC(v3_reg));
5416 match(RegD);
5417 op_cost(0);
5418 format %{ %}
5419 interface(REG_INTER);
5420 %}
5421
5422 operand vRegD_V4()
5423 %{
5424 constraint(ALLOC_IN_RC(v4_reg));
5425 match(RegD);
5426 op_cost(0);
5427 format %{ %}
5428 interface(REG_INTER);
5429 %}
5430
5431 operand vRegD_V5()
5432 %{
5433 constraint(ALLOC_IN_RC(v5_reg));
5434 match(RegD);
5435 op_cost(0);
5436 format %{ %}
5437 interface(REG_INTER);
5438 %}
5439
5440 operand vRegD_V6()
5441 %{
5442 constraint(ALLOC_IN_RC(v6_reg));
5443 match(RegD);
5444 op_cost(0);
5445 format %{ %}
5446 interface(REG_INTER);
5447 %}
5448
5449 operand vRegD_V7()
5450 %{
5451 constraint(ALLOC_IN_RC(v7_reg));
5452 match(RegD);
5453 op_cost(0);
5454 format %{ %}
5455 interface(REG_INTER);
5456 %}
5457
5458 operand vRegD_V8()
5459 %{
5460 constraint(ALLOC_IN_RC(v8_reg));
5461 match(RegD);
5462 op_cost(0);
5463 format %{ %}
5464 interface(REG_INTER);
5465 %}
5466
5467 operand vRegD_V9()
5468 %{
5469 constraint(ALLOC_IN_RC(v9_reg));
5470 match(RegD);
5471 op_cost(0);
5472 format %{ %}
5473 interface(REG_INTER);
5474 %}
5475
5476 operand vRegD_V10()
5477 %{
5478 constraint(ALLOC_IN_RC(v10_reg));
5479 match(RegD);
5480 op_cost(0);
5481 format %{ %}
5482 interface(REG_INTER);
5483 %}
5484
5485 operand vRegD_V11()
5486 %{
5487 constraint(ALLOC_IN_RC(v11_reg));
5488 match(RegD);
5489 op_cost(0);
5490 format %{ %}
5491 interface(REG_INTER);
5492 %}
5493
5494 operand vRegD_V12()
5495 %{
5496 constraint(ALLOC_IN_RC(v12_reg));
5497 match(RegD);
5498 op_cost(0);
5499 format %{ %}
5500 interface(REG_INTER);
5501 %}
5502
5503 operand vRegD_V13()
5504 %{
5505 constraint(ALLOC_IN_RC(v13_reg));
5506 match(RegD);
5507 op_cost(0);
5508 format %{ %}
5509 interface(REG_INTER);
5510 %}
5511
5512 operand vRegD_V14()
5513 %{
5514 constraint(ALLOC_IN_RC(v14_reg));
5515 match(RegD);
5516 op_cost(0);
5517 format %{ %}
5518 interface(REG_INTER);
5519 %}
5520
5521 operand vRegD_V15()
5522 %{
5523 constraint(ALLOC_IN_RC(v15_reg));
5524 match(RegD);
5525 op_cost(0);
5526 format %{ %}
5527 interface(REG_INTER);
5528 %}
5529
5530 operand vRegD_V16()
5531 %{
5532 constraint(ALLOC_IN_RC(v16_reg));
5533 match(RegD);
5534 op_cost(0);
5535 format %{ %}
5536 interface(REG_INTER);
5537 %}
5538
5539 operand vRegD_V17()
5540 %{
5541 constraint(ALLOC_IN_RC(v17_reg));
5542 match(RegD);
5543 op_cost(0);
5544 format %{ %}
5545 interface(REG_INTER);
5546 %}
5547
5548 operand vRegD_V18()
5549 %{
5550 constraint(ALLOC_IN_RC(v18_reg));
5551 match(RegD);
5552 op_cost(0);
5553 format %{ %}
5554 interface(REG_INTER);
5555 %}
5556
5557 operand vRegD_V19()
5558 %{
5559 constraint(ALLOC_IN_RC(v19_reg));
5560 match(RegD);
5561 op_cost(0);
5562 format %{ %}
5563 interface(REG_INTER);
5564 %}
5565
5566 operand vRegD_V20()
5567 %{
5568 constraint(ALLOC_IN_RC(v20_reg));
5569 match(RegD);
5570 op_cost(0);
5571 format %{ %}
5572 interface(REG_INTER);
5573 %}
5574
5575 operand vRegD_V21()
5576 %{
5577 constraint(ALLOC_IN_RC(v21_reg));
5578 match(RegD);
5579 op_cost(0);
5580 format %{ %}
5581 interface(REG_INTER);
5582 %}
5583
5584 operand vRegD_V22()
5585 %{
5586 constraint(ALLOC_IN_RC(v22_reg));
5587 match(RegD);
5588 op_cost(0);
5589 format %{ %}
5590 interface(REG_INTER);
5591 %}
5592
5593 operand vRegD_V23()
5594 %{
5595 constraint(ALLOC_IN_RC(v23_reg));
5596 match(RegD);
5597 op_cost(0);
5598 format %{ %}
5599 interface(REG_INTER);
5600 %}
5601
5602 operand vRegD_V24()
5603 %{
5604 constraint(ALLOC_IN_RC(v24_reg));
5605 match(RegD);
5606 op_cost(0);
5607 format %{ %}
5608 interface(REG_INTER);
5609 %}
5610
5611 operand vRegD_V25()
5612 %{
5613 constraint(ALLOC_IN_RC(v25_reg));
5614 match(RegD);
5615 op_cost(0);
5616 format %{ %}
5617 interface(REG_INTER);
5618 %}
5619
5620 operand vRegD_V26()
5621 %{
5622 constraint(ALLOC_IN_RC(v26_reg));
5623 match(RegD);
5624 op_cost(0);
5625 format %{ %}
5626 interface(REG_INTER);
5627 %}
5628
5629 operand vRegD_V27()
5630 %{
5631 constraint(ALLOC_IN_RC(v27_reg));
5632 match(RegD);
5633 op_cost(0);
5634 format %{ %}
5635 interface(REG_INTER);
5636 %}
5637
5638 operand vRegD_V28()
5639 %{
5640 constraint(ALLOC_IN_RC(v28_reg));
5641 match(RegD);
5642 op_cost(0);
5643 format %{ %}
5644 interface(REG_INTER);
5645 %}
5646
5647 operand vRegD_V29()
5648 %{
5649 constraint(ALLOC_IN_RC(v29_reg));
5650 match(RegD);
5651 op_cost(0);
5652 format %{ %}
5653 interface(REG_INTER);
5654 %}
5655
5656 operand vRegD_V30()
5657 %{
5658 constraint(ALLOC_IN_RC(v30_reg));
5659 match(RegD);
5660 op_cost(0);
5661 format %{ %}
5662 interface(REG_INTER);
5663 %}
5664
5665 operand vRegD_V31()
5666 %{
5667 constraint(ALLOC_IN_RC(v31_reg));
5668 match(RegD);
5669 op_cost(0);
5670 format %{ %}
5671 interface(REG_INTER);
5672 %}
5673
5674 operand pReg()
5675 %{
5676 constraint(ALLOC_IN_RC(pr_reg));
5677 match(RegVectMask);
5678 match(pRegGov);
5679 op_cost(0);
5680 format %{ %}
5681 interface(REG_INTER);
5682 %}
5683
5684 operand pRegGov()
5685 %{
5686 constraint(ALLOC_IN_RC(gov_pr));
5687 match(RegVectMask);
5688 match(pReg);
5689 op_cost(0);
5690 format %{ %}
5691 interface(REG_INTER);
5692 %}
5693
5694 operand pRegGov_P0()
5695 %{
5696 constraint(ALLOC_IN_RC(p0_reg));
5697 match(RegVectMask);
5698 op_cost(0);
5699 format %{ %}
5700 interface(REG_INTER);
5701 %}
5702
5703 operand pRegGov_P1()
5704 %{
5705 constraint(ALLOC_IN_RC(p1_reg));
5706 match(RegVectMask);
5707 op_cost(0);
5708 format %{ %}
5709 interface(REG_INTER);
5710 %}
5711
5712 // Flags register, used as output of signed compare instructions
5713
5714 // note that on AArch64 we also use this register as the output for
5715 // for floating point compare instructions (CmpF CmpD). this ensures
5716 // that ordered inequality tests use GT, GE, LT or LE none of which
5717 // pass through cases where the result is unordered i.e. one or both
5718 // inputs to the compare is a NaN. this means that the ideal code can
5719 // replace e.g. a GT with an LE and not end up capturing the NaN case
5720 // (where the comparison should always fail). EQ and NE tests are
5721 // always generated in ideal code so that unordered folds into the NE
5722 // case, matching the behaviour of AArch64 NE.
5723 //
5724 // This differs from x86 where the outputs of FP compares use a
5725 // special FP flags registers and where compares based on this
5726 // register are distinguished into ordered inequalities (cmpOpUCF) and
5727 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5728 // to explicitly handle the unordered case in branches. x86 also has
5729 // to include extra CMoveX rules to accept a cmpOpUCF input.
5730
5731 operand rFlagsReg()
5732 %{
5733 constraint(ALLOC_IN_RC(int_flags));
5734 match(RegFlags);
5735
5736 op_cost(0);
5737 format %{ "RFLAGS" %}
5738 interface(REG_INTER);
5739 %}
5740
5741 // Flags register, used as output of unsigned compare instructions
5742 operand rFlagsRegU()
5743 %{
5744 constraint(ALLOC_IN_RC(int_flags));
5745 match(RegFlags);
5746
5747 op_cost(0);
5748 format %{ "RFLAGSU" %}
5749 interface(REG_INTER);
5750 %}
5751
5752 // Special Registers
5753
5754 // Method Register
5755 operand inline_cache_RegP(iRegP reg)
5756 %{
5757 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5758 match(reg);
5759 match(iRegPNoSp);
5760 op_cost(0);
5761 format %{ %}
5762 interface(REG_INTER);
5763 %}
5764
5765 // Thread Register
5766 operand thread_RegP(iRegP reg)
5767 %{
5768 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5769 match(reg);
5770 op_cost(0);
5771 format %{ %}
5772 interface(REG_INTER);
5773 %}
5774
5775 operand lr_RegP(iRegP reg)
5776 %{
5777 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5778 match(reg);
5779 op_cost(0);
5780 format %{ %}
5781 interface(REG_INTER);
5782 %}
5783
5784 //----------Memory Operands----------------------------------------------------
5785
5786 operand indirect(iRegP reg)
5787 %{
5788 constraint(ALLOC_IN_RC(ptr_reg));
5789 match(reg);
5790 op_cost(0);
5791 format %{ "[$reg]" %}
5792 interface(MEMORY_INTER) %{
5793 base($reg);
5794 index(0xffffffff);
5795 scale(0x0);
5796 disp(0x0);
5797 %}
5798 %}
5799
5800 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5801 %{
5802 constraint(ALLOC_IN_RC(ptr_reg));
5803 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5804 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5805 op_cost(0);
5806 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5807 interface(MEMORY_INTER) %{
5808 base($reg);
5809 index($ireg);
5810 scale($scale);
5811 disp(0x0);
5812 %}
5813 %}
5814
5815 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5816 %{
5817 constraint(ALLOC_IN_RC(ptr_reg));
5818 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5819 match(AddP reg (LShiftL lreg scale));
5820 op_cost(0);
5821 format %{ "$reg, $lreg lsl($scale)" %}
5822 interface(MEMORY_INTER) %{
5823 base($reg);
5824 index($lreg);
5825 scale($scale);
5826 disp(0x0);
5827 %}
5828 %}
5829
5830 operand indIndexI2L(iRegP reg, iRegI ireg)
5831 %{
5832 constraint(ALLOC_IN_RC(ptr_reg));
5833 match(AddP reg (ConvI2L ireg));
5834 op_cost(0);
5835 format %{ "$reg, $ireg, 0, I2L" %}
5836 interface(MEMORY_INTER) %{
5837 base($reg);
5838 index($ireg);
5839 scale(0x0);
5840 disp(0x0);
5841 %}
5842 %}
5843
5844 operand indIndex(iRegP reg, iRegL lreg)
5845 %{
5846 constraint(ALLOC_IN_RC(ptr_reg));
5847 match(AddP reg lreg);
5848 op_cost(0);
5849 format %{ "$reg, $lreg" %}
5850 interface(MEMORY_INTER) %{
5851 base($reg);
5852 index($lreg);
5853 scale(0x0);
5854 disp(0x0);
5855 %}
5856 %}
5857
5858 operand indOffI(iRegP reg, immIOffset off)
5859 %{
5860 constraint(ALLOC_IN_RC(ptr_reg));
5861 match(AddP reg off);
5862 op_cost(0);
5863 format %{ "[$reg, $off]" %}
5864 interface(MEMORY_INTER) %{
5865 base($reg);
5866 index(0xffffffff);
5867 scale(0x0);
5868 disp($off);
5869 %}
5870 %}
5871
5872 operand indOffI1(iRegP reg, immIOffset1 off)
5873 %{
5874 constraint(ALLOC_IN_RC(ptr_reg));
5875 match(AddP reg off);
5876 op_cost(0);
5877 format %{ "[$reg, $off]" %}
5878 interface(MEMORY_INTER) %{
5879 base($reg);
5880 index(0xffffffff);
5881 scale(0x0);
5882 disp($off);
5883 %}
5884 %}
5885
5886 operand indOffI2(iRegP reg, immIOffset2 off)
5887 %{
5888 constraint(ALLOC_IN_RC(ptr_reg));
5889 match(AddP reg off);
5890 op_cost(0);
5891 format %{ "[$reg, $off]" %}
5892 interface(MEMORY_INTER) %{
5893 base($reg);
5894 index(0xffffffff);
5895 scale(0x0);
5896 disp($off);
5897 %}
5898 %}
5899
5900 operand indOffI4(iRegP reg, immIOffset4 off)
5901 %{
5902 constraint(ALLOC_IN_RC(ptr_reg));
5903 match(AddP reg off);
5904 op_cost(0);
5905 format %{ "[$reg, $off]" %}
5906 interface(MEMORY_INTER) %{
5907 base($reg);
5908 index(0xffffffff);
5909 scale(0x0);
5910 disp($off);
5911 %}
5912 %}
5913
5914 operand indOffI8(iRegP reg, immIOffset8 off)
5915 %{
5916 constraint(ALLOC_IN_RC(ptr_reg));
5917 match(AddP reg off);
5918 op_cost(0);
5919 format %{ "[$reg, $off]" %}
5920 interface(MEMORY_INTER) %{
5921 base($reg);
5922 index(0xffffffff);
5923 scale(0x0);
5924 disp($off);
5925 %}
5926 %}
5927
5928 operand indOffI16(iRegP reg, immIOffset16 off)
5929 %{
5930 constraint(ALLOC_IN_RC(ptr_reg));
5931 match(AddP reg off);
5932 op_cost(0);
5933 format %{ "[$reg, $off]" %}
5934 interface(MEMORY_INTER) %{
5935 base($reg);
5936 index(0xffffffff);
5937 scale(0x0);
5938 disp($off);
5939 %}
5940 %}
5941
5942 operand indOffL(iRegP reg, immLoffset off)
5943 %{
5944 constraint(ALLOC_IN_RC(ptr_reg));
5945 match(AddP reg off);
5946 op_cost(0);
5947 format %{ "[$reg, $off]" %}
5948 interface(MEMORY_INTER) %{
5949 base($reg);
5950 index(0xffffffff);
5951 scale(0x0);
5952 disp($off);
5953 %}
5954 %}
5955
5956 operand indOffL1(iRegP reg, immLoffset1 off)
5957 %{
5958 constraint(ALLOC_IN_RC(ptr_reg));
5959 match(AddP reg off);
5960 op_cost(0);
5961 format %{ "[$reg, $off]" %}
5962 interface(MEMORY_INTER) %{
5963 base($reg);
5964 index(0xffffffff);
5965 scale(0x0);
5966 disp($off);
5967 %}
5968 %}
5969
5970 operand indOffL2(iRegP reg, immLoffset2 off)
5971 %{
5972 constraint(ALLOC_IN_RC(ptr_reg));
5973 match(AddP reg off);
5974 op_cost(0);
5975 format %{ "[$reg, $off]" %}
5976 interface(MEMORY_INTER) %{
5977 base($reg);
5978 index(0xffffffff);
5979 scale(0x0);
5980 disp($off);
5981 %}
5982 %}
5983
5984 operand indOffL4(iRegP reg, immLoffset4 off)
5985 %{
5986 constraint(ALLOC_IN_RC(ptr_reg));
5987 match(AddP reg off);
5988 op_cost(0);
5989 format %{ "[$reg, $off]" %}
5990 interface(MEMORY_INTER) %{
5991 base($reg);
5992 index(0xffffffff);
5993 scale(0x0);
5994 disp($off);
5995 %}
5996 %}
5997
5998 operand indOffL8(iRegP reg, immLoffset8 off)
5999 %{
6000 constraint(ALLOC_IN_RC(ptr_reg));
6001 match(AddP reg off);
6002 op_cost(0);
6003 format %{ "[$reg, $off]" %}
6004 interface(MEMORY_INTER) %{
6005 base($reg);
6006 index(0xffffffff);
6007 scale(0x0);
6008 disp($off);
6009 %}
6010 %}
6011
6012 operand indOffL16(iRegP reg, immLoffset16 off)
6013 %{
6014 constraint(ALLOC_IN_RC(ptr_reg));
6015 match(AddP reg off);
6016 op_cost(0);
6017 format %{ "[$reg, $off]" %}
6018 interface(MEMORY_INTER) %{
6019 base($reg);
6020 index(0xffffffff);
6021 scale(0x0);
6022 disp($off);
6023 %}
6024 %}
6025
6026 operand indirectN(iRegN reg)
6027 %{
6028 predicate(CompressedOops::shift() == 0);
6029 constraint(ALLOC_IN_RC(ptr_reg));
6030 match(DecodeN reg);
6031 op_cost(0);
6032 format %{ "[$reg]\t# narrow" %}
6033 interface(MEMORY_INTER) %{
6034 base($reg);
6035 index(0xffffffff);
6036 scale(0x0);
6037 disp(0x0);
6038 %}
6039 %}
6040
6041 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6042 %{
6043 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6044 constraint(ALLOC_IN_RC(ptr_reg));
6045 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6046 op_cost(0);
6047 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6048 interface(MEMORY_INTER) %{
6049 base($reg);
6050 index($ireg);
6051 scale($scale);
6052 disp(0x0);
6053 %}
6054 %}
6055
6056 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6057 %{
6058 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6059 constraint(ALLOC_IN_RC(ptr_reg));
6060 match(AddP (DecodeN reg) (LShiftL lreg scale));
6061 op_cost(0);
6062 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6063 interface(MEMORY_INTER) %{
6064 base($reg);
6065 index($lreg);
6066 scale($scale);
6067 disp(0x0);
6068 %}
6069 %}
6070
6071 operand indIndexI2LN(iRegN reg, iRegI ireg)
6072 %{
6073 predicate(CompressedOops::shift() == 0);
6074 constraint(ALLOC_IN_RC(ptr_reg));
6075 match(AddP (DecodeN reg) (ConvI2L ireg));
6076 op_cost(0);
6077 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6078 interface(MEMORY_INTER) %{
6079 base($reg);
6080 index($ireg);
6081 scale(0x0);
6082 disp(0x0);
6083 %}
6084 %}
6085
6086 operand indIndexN(iRegN reg, iRegL lreg)
6087 %{
6088 predicate(CompressedOops::shift() == 0);
6089 constraint(ALLOC_IN_RC(ptr_reg));
6090 match(AddP (DecodeN reg) lreg);
6091 op_cost(0);
6092 format %{ "$reg, $lreg\t# narrow" %}
6093 interface(MEMORY_INTER) %{
6094 base($reg);
6095 index($lreg);
6096 scale(0x0);
6097 disp(0x0);
6098 %}
6099 %}
6100
6101 operand indOffIN(iRegN reg, immIOffset off)
6102 %{
6103 predicate(CompressedOops::shift() == 0);
6104 constraint(ALLOC_IN_RC(ptr_reg));
6105 match(AddP (DecodeN reg) off);
6106 op_cost(0);
6107 format %{ "[$reg, $off]\t# narrow" %}
6108 interface(MEMORY_INTER) %{
6109 base($reg);
6110 index(0xffffffff);
6111 scale(0x0);
6112 disp($off);
6113 %}
6114 %}
6115
6116 operand indOffLN(iRegN reg, immLoffset off)
6117 %{
6118 predicate(CompressedOops::shift() == 0);
6119 constraint(ALLOC_IN_RC(ptr_reg));
6120 match(AddP (DecodeN reg) off);
6121 op_cost(0);
6122 format %{ "[$reg, $off]\t# narrow" %}
6123 interface(MEMORY_INTER) %{
6124 base($reg);
6125 index(0xffffffff);
6126 scale(0x0);
6127 disp($off);
6128 %}
6129 %}
6130
6131
6132
6133 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6134 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6135 %{
6136 constraint(ALLOC_IN_RC(ptr_reg));
6137 match(AddP reg off);
6138 op_cost(0);
6139 format %{ "[$reg, $off]" %}
6140 interface(MEMORY_INTER) %{
6141 base($reg);
6142 index(0xffffffff);
6143 scale(0x0);
6144 disp($off);
6145 %}
6146 %}
6147
6148 //----------Special Memory Operands--------------------------------------------
6149 // Stack Slot Operand - This operand is used for loading and storing temporary
6150 // values on the stack where a match requires a value to
6151 // flow through memory.
6152 operand stackSlotP(sRegP reg)
6153 %{
6154 constraint(ALLOC_IN_RC(stack_slots));
6155 op_cost(100);
6156 // No match rule because this operand is only generated in matching
6157 // match(RegP);
6158 format %{ "[$reg]" %}
6159 interface(MEMORY_INTER) %{
6160 base(0x1e); // RSP
6161 index(0x0); // No Index
6162 scale(0x0); // No Scale
6163 disp($reg); // Stack Offset
6164 %}
6165 %}
6166
6167 operand stackSlotI(sRegI reg)
6168 %{
6169 constraint(ALLOC_IN_RC(stack_slots));
6170 // No match rule because this operand is only generated in matching
6171 // match(RegI);
6172 format %{ "[$reg]" %}
6173 interface(MEMORY_INTER) %{
6174 base(0x1e); // RSP
6175 index(0x0); // No Index
6176 scale(0x0); // No Scale
6177 disp($reg); // Stack Offset
6178 %}
6179 %}
6180
6181 operand stackSlotF(sRegF reg)
6182 %{
6183 constraint(ALLOC_IN_RC(stack_slots));
6184 // No match rule because this operand is only generated in matching
6185 // match(RegF);
6186 format %{ "[$reg]" %}
6187 interface(MEMORY_INTER) %{
6188 base(0x1e); // RSP
6189 index(0x0); // No Index
6190 scale(0x0); // No Scale
6191 disp($reg); // Stack Offset
6192 %}
6193 %}
6194
6195 operand stackSlotD(sRegD reg)
6196 %{
6197 constraint(ALLOC_IN_RC(stack_slots));
6198 // No match rule because this operand is only generated in matching
6199 // match(RegD);
6200 format %{ "[$reg]" %}
6201 interface(MEMORY_INTER) %{
6202 base(0x1e); // RSP
6203 index(0x0); // No Index
6204 scale(0x0); // No Scale
6205 disp($reg); // Stack Offset
6206 %}
6207 %}
6208
6209 operand stackSlotL(sRegL reg)
6210 %{
6211 constraint(ALLOC_IN_RC(stack_slots));
6212 // No match rule because this operand is only generated in matching
6213 // match(RegL);
6214 format %{ "[$reg]" %}
6215 interface(MEMORY_INTER) %{
6216 base(0x1e); // RSP
6217 index(0x0); // No Index
6218 scale(0x0); // No Scale
6219 disp($reg); // Stack Offset
6220 %}
6221 %}
6222
6223 // Operands for expressing Control Flow
6224 // NOTE: Label is a predefined operand which should not be redefined in
6225 // the AD file. It is generically handled within the ADLC.
6226
6227 //----------Conditional Branch Operands----------------------------------------
6228 // Comparison Op - This is the operation of the comparison, and is limited to
6229 // the following set of codes:
6230 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6231 //
6232 // Other attributes of the comparison, such as unsignedness, are specified
6233 // by the comparison instruction that sets a condition code flags register.
6234 // That result is represented by a flags operand whose subtype is appropriate
6235 // to the unsignedness (etc.) of the comparison.
6236 //
6237 // Later, the instruction which matches both the Comparison Op (a Bool) and
6238 // the flags (produced by the Cmp) specifies the coding of the comparison op
6239 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6240
6241 // used for signed integral comparisons and fp comparisons
6242
6243 operand cmpOp()
6244 %{
6245 match(Bool);
6246
6247 format %{ "" %}
6248 interface(COND_INTER) %{
6249 equal(0x0, "eq");
6250 not_equal(0x1, "ne");
6251 less(0xb, "lt");
6252 greater_equal(0xa, "ge");
6253 less_equal(0xd, "le");
6254 greater(0xc, "gt");
6255 overflow(0x6, "vs");
6256 no_overflow(0x7, "vc");
6257 %}
6258 %}
6259
6260 // used for unsigned integral comparisons
6261
6262 operand cmpOpU()
6263 %{
6264 match(Bool);
6265
6266 format %{ "" %}
6267 interface(COND_INTER) %{
6268 equal(0x0, "eq");
6269 not_equal(0x1, "ne");
6270 less(0x3, "lo");
6271 greater_equal(0x2, "hs");
6272 less_equal(0x9, "ls");
6273 greater(0x8, "hi");
6274 overflow(0x6, "vs");
6275 no_overflow(0x7, "vc");
6276 %}
6277 %}
6278
6279 // used for certain integral comparisons which can be
6280 // converted to cbxx or tbxx instructions
6281
6282 operand cmpOpEqNe()
6283 %{
6284 match(Bool);
6285 op_cost(0);
6286 predicate(n->as_Bool()->_test._test == BoolTest::ne
6287 || n->as_Bool()->_test._test == BoolTest::eq);
6288
6289 format %{ "" %}
6290 interface(COND_INTER) %{
6291 equal(0x0, "eq");
6292 not_equal(0x1, "ne");
6293 less(0xb, "lt");
6294 greater_equal(0xa, "ge");
6295 less_equal(0xd, "le");
6296 greater(0xc, "gt");
6297 overflow(0x6, "vs");
6298 no_overflow(0x7, "vc");
6299 %}
6300 %}
6301
6302 // used for certain integral comparisons which can be
6303 // converted to cbxx or tbxx instructions
6304
6305 operand cmpOpLtGe()
6306 %{
6307 match(Bool);
6308 op_cost(0);
6309
6310 predicate(n->as_Bool()->_test._test == BoolTest::lt
6311 || n->as_Bool()->_test._test == BoolTest::ge);
6312
6313 format %{ "" %}
6314 interface(COND_INTER) %{
6315 equal(0x0, "eq");
6316 not_equal(0x1, "ne");
6317 less(0xb, "lt");
6318 greater_equal(0xa, "ge");
6319 less_equal(0xd, "le");
6320 greater(0xc, "gt");
6321 overflow(0x6, "vs");
6322 no_overflow(0x7, "vc");
6323 %}
6324 %}
6325
6326 // used for certain unsigned integral comparisons which can be
6327 // converted to cbxx or tbxx instructions
6328
6329 operand cmpOpUEqNeLtGe()
6330 %{
6331 match(Bool);
6332 op_cost(0);
6333
6334 predicate(n->as_Bool()->_test._test == BoolTest::eq
6335 || n->as_Bool()->_test._test == BoolTest::ne
6336 || n->as_Bool()->_test._test == BoolTest::lt
6337 || n->as_Bool()->_test._test == BoolTest::ge);
6338
6339 format %{ "" %}
6340 interface(COND_INTER) %{
6341 equal(0x0, "eq");
6342 not_equal(0x1, "ne");
6343 less(0xb, "lt");
6344 greater_equal(0xa, "ge");
6345 less_equal(0xd, "le");
6346 greater(0xc, "gt");
6347 overflow(0x6, "vs");
6348 no_overflow(0x7, "vc");
6349 %}
6350 %}
6351
6352 // Special operand allowing long args to int ops to be truncated for free
6353
6354 operand iRegL2I(iRegL reg) %{
6355
6356 op_cost(0);
6357
6358 match(ConvL2I reg);
6359
6360 format %{ "l2i($reg)" %}
6361
6362 interface(REG_INTER)
6363 %}
6364
6365 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6366 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6367 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6368 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6369
6370 //----------OPERAND CLASSES----------------------------------------------------
6371 // Operand Classes are groups of operands that are used as to simplify
6372 // instruction definitions by not requiring the AD writer to specify
6373 // separate instructions for every form of operand when the
6374 // instruction accepts multiple operand types with the same basic
6375 // encoding and format. The classic case of this is memory operands.
6376
6377 // memory is used to define read/write location for load/store
6378 // instruction defs. we can turn a memory op into an Address
6379
6380 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6381 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6382
6383 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6384 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6385
6386 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6387 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6388
6389 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6390 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6391
6392 // All of the memory operands. For the pipeline description.
6393 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6394 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6395 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6396
6397
6398 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6399 // operations. it allows the src to be either an iRegI or a (ConvL2I
6400 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6401 // can be elided because the 32-bit instruction will just employ the
6402 // lower 32 bits anyway.
6403 //
6404 // n.b. this does not elide all L2I conversions. if the truncated
6405 // value is consumed by more than one operation then the ConvL2I
6406 // cannot be bundled into the consuming nodes so an l2i gets planted
6407 // (actually a movw $dst $src) and the downstream instructions consume
6408 // the result of the l2i as an iRegI input. That's a shame since the
6409 // movw is actually redundant but its not too costly.
6410
6411 opclass iRegIorL2I(iRegI, iRegL2I);
6412
6413 //----------PIPELINE-----------------------------------------------------------
6414 // Rules which define the behavior of the target architectures pipeline.
6415
6416 // For specific pipelines, eg A53, define the stages of that pipeline
6417 //pipe_desc(ISS, EX1, EX2, WR);
6418 #define ISS S0
6419 #define EX1 S1
6420 #define EX2 S2
6421 #define WR S3
6422
6423 // Integer ALU reg operation
6424 pipeline %{
6425
6426 attributes %{
6427 // ARM instructions are of fixed length
6428 fixed_size_instructions; // Fixed size instructions TODO does
6429 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6430 // ARM instructions come in 32-bit word units
6431 instruction_unit_size = 4; // An instruction is 4 bytes long
6432 instruction_fetch_unit_size = 64; // The processor fetches one line
6433 instruction_fetch_units = 1; // of 64 bytes
6434
6435 // List of nop instructions
6436 nops( MachNop );
6437 %}
6438
6439 // We don't use an actual pipeline model so don't care about resources
6440 // or description. we do use pipeline classes to introduce fixed
6441 // latencies
6442
6443 //----------RESOURCES----------------------------------------------------------
6444 // Resources are the functional units available to the machine
6445
6446 resources( INS0, INS1, INS01 = INS0 | INS1,
6447 ALU0, ALU1, ALU = ALU0 | ALU1,
6448 MAC,
6449 DIV,
6450 BRANCH,
6451 LDST,
6452 NEON_FP);
6453
6454 //----------PIPELINE DESCRIPTION-----------------------------------------------
6455 // Pipeline Description specifies the stages in the machine's pipeline
6456
6457 // Define the pipeline as a generic 6 stage pipeline
6458 pipe_desc(S0, S1, S2, S3, S4, S5);
6459
6460 //----------PIPELINE CLASSES---------------------------------------------------
6461 // Pipeline Classes describe the stages in which input and output are
6462 // referenced by the hardware pipeline.
6463
6464 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6465 %{
6466 single_instruction;
6467 src1 : S1(read);
6468 src2 : S2(read);
6469 dst : S5(write);
6470 INS01 : ISS;
6471 NEON_FP : S5;
6472 %}
6473
6474 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6475 %{
6476 single_instruction;
6477 src1 : S1(read);
6478 src2 : S2(read);
6479 dst : S5(write);
6480 INS01 : ISS;
6481 NEON_FP : S5;
6482 %}
6483
6484 pipe_class fp_uop_s(vRegF dst, vRegF src)
6485 %{
6486 single_instruction;
6487 src : S1(read);
6488 dst : S5(write);
6489 INS01 : ISS;
6490 NEON_FP : S5;
6491 %}
6492
6493 pipe_class fp_uop_d(vRegD dst, vRegD src)
6494 %{
6495 single_instruction;
6496 src : S1(read);
6497 dst : S5(write);
6498 INS01 : ISS;
6499 NEON_FP : S5;
6500 %}
6501
6502 pipe_class fp_d2f(vRegF dst, vRegD src)
6503 %{
6504 single_instruction;
6505 src : S1(read);
6506 dst : S5(write);
6507 INS01 : ISS;
6508 NEON_FP : S5;
6509 %}
6510
6511 pipe_class fp_f2d(vRegD dst, vRegF src)
6512 %{
6513 single_instruction;
6514 src : S1(read);
6515 dst : S5(write);
6516 INS01 : ISS;
6517 NEON_FP : S5;
6518 %}
6519
6520 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6521 %{
6522 single_instruction;
6523 src : S1(read);
6524 dst : S5(write);
6525 INS01 : ISS;
6526 NEON_FP : S5;
6527 %}
6528
6529 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6530 %{
6531 single_instruction;
6532 src : S1(read);
6533 dst : S5(write);
6534 INS01 : ISS;
6535 NEON_FP : S5;
6536 %}
6537
6538 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6539 %{
6540 single_instruction;
6541 src : S1(read);
6542 dst : S5(write);
6543 INS01 : ISS;
6544 NEON_FP : S5;
6545 %}
6546
6547 pipe_class fp_l2f(vRegF dst, iRegL src)
6548 %{
6549 single_instruction;
6550 src : S1(read);
6551 dst : S5(write);
6552 INS01 : ISS;
6553 NEON_FP : S5;
6554 %}
6555
6556 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6557 %{
6558 single_instruction;
6559 src : S1(read);
6560 dst : S5(write);
6561 INS01 : ISS;
6562 NEON_FP : S5;
6563 %}
6564
6565 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6566 %{
6567 single_instruction;
6568 src : S1(read);
6569 dst : S5(write);
6570 INS01 : ISS;
6571 NEON_FP : S5;
6572 %}
6573
6574 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6575 %{
6576 single_instruction;
6577 src : S1(read);
6578 dst : S5(write);
6579 INS01 : ISS;
6580 NEON_FP : S5;
6581 %}
6582
6583 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6584 %{
6585 single_instruction;
6586 src : S1(read);
6587 dst : S5(write);
6588 INS01 : ISS;
6589 NEON_FP : S5;
6590 %}
6591
6592 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6593 %{
6594 single_instruction;
6595 src1 : S1(read);
6596 src2 : S2(read);
6597 dst : S5(write);
6598 INS0 : ISS;
6599 NEON_FP : S5;
6600 %}
6601
6602 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6603 %{
6604 single_instruction;
6605 src1 : S1(read);
6606 src2 : S2(read);
6607 dst : S5(write);
6608 INS0 : ISS;
6609 NEON_FP : S5;
6610 %}
6611
6612 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6613 %{
6614 single_instruction;
6615 cr : S1(read);
6616 src1 : S1(read);
6617 src2 : S1(read);
6618 dst : S3(write);
6619 INS01 : ISS;
6620 NEON_FP : S3;
6621 %}
6622
6623 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6624 %{
6625 single_instruction;
6626 cr : S1(read);
6627 src1 : S1(read);
6628 src2 : S1(read);
6629 dst : S3(write);
6630 INS01 : ISS;
6631 NEON_FP : S3;
6632 %}
6633
6634 pipe_class fp_imm_s(vRegF dst)
6635 %{
6636 single_instruction;
6637 dst : S3(write);
6638 INS01 : ISS;
6639 NEON_FP : S3;
6640 %}
6641
6642 pipe_class fp_imm_d(vRegD dst)
6643 %{
6644 single_instruction;
6645 dst : S3(write);
6646 INS01 : ISS;
6647 NEON_FP : S3;
6648 %}
6649
6650 pipe_class fp_load_constant_s(vRegF dst)
6651 %{
6652 single_instruction;
6653 dst : S4(write);
6654 INS01 : ISS;
6655 NEON_FP : S4;
6656 %}
6657
6658 pipe_class fp_load_constant_d(vRegD dst)
6659 %{
6660 single_instruction;
6661 dst : S4(write);
6662 INS01 : ISS;
6663 NEON_FP : S4;
6664 %}
6665
6666 //------- Integer ALU operations --------------------------
6667
6668 // Integer ALU reg-reg operation
6669 // Operands needed in EX1, result generated in EX2
6670 // Eg. ADD x0, x1, x2
6671 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6672 %{
6673 single_instruction;
6674 dst : EX2(write);
6675 src1 : EX1(read);
6676 src2 : EX1(read);
6677 INS01 : ISS; // Dual issue as instruction 0 or 1
6678 ALU : EX2;
6679 %}
6680
6681 // Integer ALU reg-reg operation with constant shift
6682 // Shifted register must be available in LATE_ISS instead of EX1
6683 // Eg. ADD x0, x1, x2, LSL #2
6684 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6685 %{
6686 single_instruction;
6687 dst : EX2(write);
6688 src1 : EX1(read);
6689 src2 : ISS(read);
6690 INS01 : ISS;
6691 ALU : EX2;
6692 %}
6693
6694 // Integer ALU reg operation with constant shift
6695 // Eg. LSL x0, x1, #shift
6696 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6697 %{
6698 single_instruction;
6699 dst : EX2(write);
6700 src1 : ISS(read);
6701 INS01 : ISS;
6702 ALU : EX2;
6703 %}
6704
6705 // Integer ALU reg-reg operation with variable shift
6706 // Both operands must be available in LATE_ISS instead of EX1
6707 // Result is available in EX1 instead of EX2
6708 // Eg. LSLV x0, x1, x2
6709 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6710 %{
6711 single_instruction;
6712 dst : EX1(write);
6713 src1 : ISS(read);
6714 src2 : ISS(read);
6715 INS01 : ISS;
6716 ALU : EX1;
6717 %}
6718
6719 // Integer ALU reg-reg operation with extract
6720 // As for _vshift above, but result generated in EX2
6721 // Eg. EXTR x0, x1, x2, #N
6722 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6723 %{
6724 single_instruction;
6725 dst : EX2(write);
6726 src1 : ISS(read);
6727 src2 : ISS(read);
6728 INS1 : ISS; // Can only dual issue as Instruction 1
6729 ALU : EX1;
6730 %}
6731
6732 // Integer ALU reg operation
6733 // Eg. NEG x0, x1
6734 pipe_class ialu_reg(iRegI dst, iRegI src)
6735 %{
6736 single_instruction;
6737 dst : EX2(write);
6738 src : EX1(read);
6739 INS01 : ISS;
6740 ALU : EX2;
6741 %}
6742
6743 // Integer ALU reg mmediate operation
6744 // Eg. ADD x0, x1, #N
6745 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6746 %{
6747 single_instruction;
6748 dst : EX2(write);
6749 src1 : EX1(read);
6750 INS01 : ISS;
6751 ALU : EX2;
6752 %}
6753
6754 // Integer ALU immediate operation (no source operands)
6755 // Eg. MOV x0, #N
6756 pipe_class ialu_imm(iRegI dst)
6757 %{
6758 single_instruction;
6759 dst : EX1(write);
6760 INS01 : ISS;
6761 ALU : EX1;
6762 %}
6763
6764 //------- Compare operation -------------------------------
6765
6766 // Compare reg-reg
6767 // Eg. CMP x0, x1
6768 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6769 %{
6770 single_instruction;
6771 // fixed_latency(16);
6772 cr : EX2(write);
6773 op1 : EX1(read);
6774 op2 : EX1(read);
6775 INS01 : ISS;
6776 ALU : EX2;
6777 %}
6778
6779 // Compare reg-reg
6780 // Eg. CMP x0, #N
6781 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6782 %{
6783 single_instruction;
6784 // fixed_latency(16);
6785 cr : EX2(write);
6786 op1 : EX1(read);
6787 INS01 : ISS;
6788 ALU : EX2;
6789 %}
6790
6791 //------- Conditional instructions ------------------------
6792
6793 // Conditional no operands
6794 // Eg. CSINC x0, zr, zr, <cond>
6795 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6796 %{
6797 single_instruction;
6798 cr : EX1(read);
6799 dst : EX2(write);
6800 INS01 : ISS;
6801 ALU : EX2;
6802 %}
6803
6804 // Conditional 2 operand
6805 // EG. CSEL X0, X1, X2, <cond>
6806 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6807 %{
6808 single_instruction;
6809 cr : EX1(read);
6810 src1 : EX1(read);
6811 src2 : EX1(read);
6812 dst : EX2(write);
6813 INS01 : ISS;
6814 ALU : EX2;
6815 %}
6816
6817 // Conditional 2 operand
6818 // EG. CSEL X0, X1, X2, <cond>
6819 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6820 %{
6821 single_instruction;
6822 cr : EX1(read);
6823 src : EX1(read);
6824 dst : EX2(write);
6825 INS01 : ISS;
6826 ALU : EX2;
6827 %}
6828
6829 //------- Multiply pipeline operations --------------------
6830
6831 // Multiply reg-reg
6832 // Eg. MUL w0, w1, w2
6833 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6834 %{
6835 single_instruction;
6836 dst : WR(write);
6837 src1 : ISS(read);
6838 src2 : ISS(read);
6839 INS01 : ISS;
6840 MAC : WR;
6841 %}
6842
6843 // Multiply accumulate
6844 // Eg. MADD w0, w1, w2, w3
6845 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6846 %{
6847 single_instruction;
6848 dst : WR(write);
6849 src1 : ISS(read);
6850 src2 : ISS(read);
6851 src3 : ISS(read);
6852 INS01 : ISS;
6853 MAC : WR;
6854 %}
6855
6856 // Eg. MUL w0, w1, w2
6857 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6858 %{
6859 single_instruction;
6860 fixed_latency(3); // Maximum latency for 64 bit mul
6861 dst : WR(write);
6862 src1 : ISS(read);
6863 src2 : ISS(read);
6864 INS01 : ISS;
6865 MAC : WR;
6866 %}
6867
6868 // Multiply accumulate
6869 // Eg. MADD w0, w1, w2, w3
6870 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6871 %{
6872 single_instruction;
6873 fixed_latency(3); // Maximum latency for 64 bit mul
6874 dst : WR(write);
6875 src1 : ISS(read);
6876 src2 : ISS(read);
6877 src3 : ISS(read);
6878 INS01 : ISS;
6879 MAC : WR;
6880 %}
6881
6882 //------- Divide pipeline operations --------------------
6883
6884 // Eg. SDIV w0, w1, w2
6885 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6886 %{
6887 single_instruction;
6888 fixed_latency(8); // Maximum latency for 32 bit divide
6889 dst : WR(write);
6890 src1 : ISS(read);
6891 src2 : ISS(read);
6892 INS0 : ISS; // Can only dual issue as instruction 0
6893 DIV : WR;
6894 %}
6895
6896 // Eg. SDIV x0, x1, x2
6897 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6898 %{
6899 single_instruction;
6900 fixed_latency(16); // Maximum latency for 64 bit divide
6901 dst : WR(write);
6902 src1 : ISS(read);
6903 src2 : ISS(read);
6904 INS0 : ISS; // Can only dual issue as instruction 0
6905 DIV : WR;
6906 %}
6907
6908 //------- Load pipeline operations ------------------------
6909
6910 // Load - prefetch
6911 // Eg. PFRM <mem>
6912 pipe_class iload_prefetch(memory mem)
6913 %{
6914 single_instruction;
6915 mem : ISS(read);
6916 INS01 : ISS;
6917 LDST : WR;
6918 %}
6919
6920 // Load - reg, mem
6921 // Eg. LDR x0, <mem>
6922 pipe_class iload_reg_mem(iRegI dst, memory mem)
6923 %{
6924 single_instruction;
6925 dst : WR(write);
6926 mem : ISS(read);
6927 INS01 : ISS;
6928 LDST : WR;
6929 %}
6930
6931 // Load - reg, reg
6932 // Eg. LDR x0, [sp, x1]
6933 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6934 %{
6935 single_instruction;
6936 dst : WR(write);
6937 src : ISS(read);
6938 INS01 : ISS;
6939 LDST : WR;
6940 %}
6941
6942 //------- Store pipeline operations -----------------------
6943
6944 // Store - zr, mem
6945 // Eg. STR zr, <mem>
6946 pipe_class istore_mem(memory mem)
6947 %{
6948 single_instruction;
6949 mem : ISS(read);
6950 INS01 : ISS;
6951 LDST : WR;
6952 %}
6953
6954 // Store - reg, mem
6955 // Eg. STR x0, <mem>
6956 pipe_class istore_reg_mem(iRegI src, memory mem)
6957 %{
6958 single_instruction;
6959 mem : ISS(read);
6960 src : EX2(read);
6961 INS01 : ISS;
6962 LDST : WR;
6963 %}
6964
6965 // Store - reg, reg
6966 // Eg. STR x0, [sp, x1]
6967 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6968 %{
6969 single_instruction;
6970 dst : ISS(read);
6971 src : EX2(read);
6972 INS01 : ISS;
6973 LDST : WR;
6974 %}
6975
6976 //------- Store pipeline operations -----------------------
6977
6978 // Branch
6979 pipe_class pipe_branch()
6980 %{
6981 single_instruction;
6982 INS01 : ISS;
6983 BRANCH : EX1;
6984 %}
6985
6986 // Conditional branch
6987 pipe_class pipe_branch_cond(rFlagsReg cr)
6988 %{
6989 single_instruction;
6990 cr : EX1(read);
6991 INS01 : ISS;
6992 BRANCH : EX1;
6993 %}
6994
6995 // Compare & Branch
6996 // EG. CBZ/CBNZ
6997 pipe_class pipe_cmp_branch(iRegI op1)
6998 %{
6999 single_instruction;
7000 op1 : EX1(read);
7001 INS01 : ISS;
7002 BRANCH : EX1;
7003 %}
7004
7005 //------- Synchronisation operations ----------------------
7006
7007 // Any operation requiring serialization.
7008 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7009 pipe_class pipe_serial()
7010 %{
7011 single_instruction;
7012 force_serialization;
7013 fixed_latency(16);
7014 INS01 : ISS(2); // Cannot dual issue with any other instruction
7015 LDST : WR;
7016 %}
7017
7018 // Generic big/slow expanded idiom - also serialized
7019 pipe_class pipe_slow()
7020 %{
7021 instruction_count(10);
7022 multiple_bundles;
7023 force_serialization;
7024 fixed_latency(16);
7025 INS01 : ISS(2); // Cannot dual issue with any other instruction
7026 LDST : WR;
7027 %}
7028
7029 // Empty pipeline class
7030 pipe_class pipe_class_empty()
7031 %{
7032 single_instruction;
7033 fixed_latency(0);
7034 %}
7035
7036 // Default pipeline class.
7037 pipe_class pipe_class_default()
7038 %{
7039 single_instruction;
7040 fixed_latency(2);
7041 %}
7042
7043 // Pipeline class for compares.
7044 pipe_class pipe_class_compare()
7045 %{
7046 single_instruction;
7047 fixed_latency(16);
7048 %}
7049
7050 // Pipeline class for memory operations.
7051 pipe_class pipe_class_memory()
7052 %{
7053 single_instruction;
7054 fixed_latency(16);
7055 %}
7056
7057 // Pipeline class for call.
7058 pipe_class pipe_class_call()
7059 %{
7060 single_instruction;
7061 fixed_latency(100);
7062 %}
7063
7064 // Define the class for the Nop node.
7065 define %{
7066 MachNop = pipe_class_empty;
7067 %}
7068
7069 %}
7070 //----------INSTRUCTIONS-------------------------------------------------------
7071 //
7072 // match -- States which machine-independent subtree may be replaced
7073 // by this instruction.
7074 // ins_cost -- The estimated cost of this instruction is used by instruction
7075 // selection to identify a minimum cost tree of machine
7076 // instructions that matches a tree of machine-independent
7077 // instructions.
7078 // format -- A string providing the disassembly for this instruction.
7079 // The value of an instruction's operand may be inserted
7080 // by referring to it with a '$' prefix.
7081 // opcode -- Three instruction opcodes may be provided. These are referred
7082 // to within an encode class as $primary, $secondary, and $tertiary
7083 // rrspectively. The primary opcode is commonly used to
7084 // indicate the type of machine instruction, while secondary
7085 // and tertiary are often used for prefix options or addressing
7086 // modes.
7087 // ins_encode -- A list of encode classes with parameters. The encode class
7088 // name must have been defined in an 'enc_class' specification
7089 // in the encode section of the architecture description.
7090
7091 // ============================================================================
7092 // Memory (Load/Store) Instructions
7093
7094 // Load Instructions
7095
7096 // Load Byte (8 bit signed)
7097 instruct loadB(iRegINoSp dst, memory1 mem)
7098 %{
7099 match(Set dst (LoadB mem));
7100 predicate(!needs_acquiring_load(n));
7101
7102 ins_cost(4 * INSN_COST);
7103 format %{ "ldrsbw $dst, $mem\t# byte" %}
7104
7105 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7106
7107 ins_pipe(iload_reg_mem);
7108 %}
7109
7110 // Load Byte (8 bit signed) into long
7111 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7112 %{
7113 match(Set dst (ConvI2L (LoadB mem)));
7114 predicate(!needs_acquiring_load(n->in(1)));
7115
7116 ins_cost(4 * INSN_COST);
7117 format %{ "ldrsb $dst, $mem\t# byte" %}
7118
7119 ins_encode(aarch64_enc_ldrsb(dst, mem));
7120
7121 ins_pipe(iload_reg_mem);
7122 %}
7123
7124 // Load Byte (8 bit unsigned)
7125 instruct loadUB(iRegINoSp dst, memory1 mem)
7126 %{
7127 match(Set dst (LoadUB mem));
7128 predicate(!needs_acquiring_load(n));
7129
7130 ins_cost(4 * INSN_COST);
7131 format %{ "ldrbw $dst, $mem\t# byte" %}
7132
7133 ins_encode(aarch64_enc_ldrb(dst, mem));
7134
7135 ins_pipe(iload_reg_mem);
7136 %}
7137
7138 // Load Byte (8 bit unsigned) into long
7139 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7140 %{
7141 match(Set dst (ConvI2L (LoadUB mem)));
7142 predicate(!needs_acquiring_load(n->in(1)));
7143
7144 ins_cost(4 * INSN_COST);
7145 format %{ "ldrb $dst, $mem\t# byte" %}
7146
7147 ins_encode(aarch64_enc_ldrb(dst, mem));
7148
7149 ins_pipe(iload_reg_mem);
7150 %}
7151
7152 // Load Short (16 bit signed)
7153 instruct loadS(iRegINoSp dst, memory2 mem)
7154 %{
7155 match(Set dst (LoadS mem));
7156 predicate(!needs_acquiring_load(n));
7157
7158 ins_cost(4 * INSN_COST);
7159 format %{ "ldrshw $dst, $mem\t# short" %}
7160
7161 ins_encode(aarch64_enc_ldrshw(dst, mem));
7162
7163 ins_pipe(iload_reg_mem);
7164 %}
7165
7166 // Load Short (16 bit signed) into long
7167 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7168 %{
7169 match(Set dst (ConvI2L (LoadS mem)));
7170 predicate(!needs_acquiring_load(n->in(1)));
7171
7172 ins_cost(4 * INSN_COST);
7173 format %{ "ldrsh $dst, $mem\t# short" %}
7174
7175 ins_encode(aarch64_enc_ldrsh(dst, mem));
7176
7177 ins_pipe(iload_reg_mem);
7178 %}
7179
7180 // Load Char (16 bit unsigned)
7181 instruct loadUS(iRegINoSp dst, memory2 mem)
7182 %{
7183 match(Set dst (LoadUS mem));
7184 predicate(!needs_acquiring_load(n));
7185
7186 ins_cost(4 * INSN_COST);
7187 format %{ "ldrh $dst, $mem\t# short" %}
7188
7189 ins_encode(aarch64_enc_ldrh(dst, mem));
7190
7191 ins_pipe(iload_reg_mem);
7192 %}
7193
7194 // Load Short/Char (16 bit unsigned) into long
7195 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7196 %{
7197 match(Set dst (ConvI2L (LoadUS mem)));
7198 predicate(!needs_acquiring_load(n->in(1)));
7199
7200 ins_cost(4 * INSN_COST);
7201 format %{ "ldrh $dst, $mem\t# short" %}
7202
7203 ins_encode(aarch64_enc_ldrh(dst, mem));
7204
7205 ins_pipe(iload_reg_mem);
7206 %}
7207
7208 // Load Integer (32 bit signed)
7209 instruct loadI(iRegINoSp dst, memory4 mem)
7210 %{
7211 match(Set dst (LoadI mem));
7212 predicate(!needs_acquiring_load(n));
7213
7214 ins_cost(4 * INSN_COST);
7215 format %{ "ldrw $dst, $mem\t# int" %}
7216
7217 ins_encode(aarch64_enc_ldrw(dst, mem));
7218
7219 ins_pipe(iload_reg_mem);
7220 %}
7221
7222 // Load Integer (32 bit signed) into long
7223 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7224 %{
7225 match(Set dst (ConvI2L (LoadI mem)));
7226 predicate(!needs_acquiring_load(n->in(1)));
7227
7228 ins_cost(4 * INSN_COST);
7229 format %{ "ldrsw $dst, $mem\t# int" %}
7230
7231 ins_encode(aarch64_enc_ldrsw(dst, mem));
7232
7233 ins_pipe(iload_reg_mem);
7234 %}
7235
7236 // Load Integer (32 bit unsigned) into long
7237 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7238 %{
7239 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7240 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7241
7242 ins_cost(4 * INSN_COST);
7243 format %{ "ldrw $dst, $mem\t# int" %}
7244
7245 ins_encode(aarch64_enc_ldrw(dst, mem));
7246
7247 ins_pipe(iload_reg_mem);
7248 %}
7249
7250 // Load Long (64 bit signed)
7251 instruct loadL(iRegLNoSp dst, memory8 mem)
7252 %{
7253 match(Set dst (LoadL mem));
7254 predicate(!needs_acquiring_load(n));
7255
7256 ins_cost(4 * INSN_COST);
7257 format %{ "ldr $dst, $mem\t# int" %}
7258
7259 ins_encode(aarch64_enc_ldr(dst, mem));
7260
7261 ins_pipe(iload_reg_mem);
7262 %}
7263
7264 // Load Range
7265 instruct loadRange(iRegINoSp dst, memory4 mem)
7266 %{
7267 match(Set dst (LoadRange mem));
7268
7269 ins_cost(4 * INSN_COST);
7270 format %{ "ldrw $dst, $mem\t# range" %}
7271
7272 ins_encode(aarch64_enc_ldrw(dst, mem));
7273
7274 ins_pipe(iload_reg_mem);
7275 %}
7276
7277 // Load Pointer
7278 instruct loadP(iRegPNoSp dst, memory8 mem)
7279 %{
7280 match(Set dst (LoadP mem));
7281 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7282
7283 ins_cost(4 * INSN_COST);
7284 format %{ "ldr $dst, $mem\t# ptr" %}
7285
7286 ins_encode(aarch64_enc_ldr(dst, mem));
7287
7288 ins_pipe(iload_reg_mem);
7289 %}
7290
7291 // Load Compressed Pointer
7292 instruct loadN(iRegNNoSp dst, memory4 mem)
7293 %{
7294 match(Set dst (LoadN mem));
7295 predicate(!needs_acquiring_load(n));
7296
7297 ins_cost(4 * INSN_COST);
7298 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7299
7300 ins_encode(aarch64_enc_ldrw(dst, mem));
7301
7302 ins_pipe(iload_reg_mem);
7303 %}
7304
7305 // Load Klass Pointer
7306 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7307 %{
7308 match(Set dst (LoadKlass mem));
7309 predicate(!needs_acquiring_load(n));
7310
7311 ins_cost(4 * INSN_COST);
7312 format %{ "ldr $dst, $mem\t# class" %}
7313
7314 ins_encode(aarch64_enc_ldr(dst, mem));
7315
7316 ins_pipe(iload_reg_mem);
7317 %}
7318
7319 // Load Narrow Klass Pointer
7320 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7321 %{
7322 match(Set dst (LoadNKlass mem));
7323 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
7324
7325 ins_cost(4 * INSN_COST);
7326 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7327
7328 ins_encode(aarch64_enc_ldrw(dst, mem));
7329
7330 ins_pipe(iload_reg_mem);
7331 %}
7332
7333 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
7334 %{
7335 match(Set dst (LoadNKlass mem));
7336 effect(KILL cr);
7337 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
7338
7339 ins_cost(4 * INSN_COST);
7340 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7341 ins_encode %{
7342 assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
7343 assert($mem$$index$$Register == noreg, "expect no index");
7344 __ load_nklass_compact($dst$$Register, $mem$$base$$Register);
7345 %}
7346 ins_pipe(pipe_slow);
7347 %}
7348
7349 // Load Float
7350 instruct loadF(vRegF dst, memory4 mem)
7351 %{
7352 match(Set dst (LoadF mem));
7353 predicate(!needs_acquiring_load(n));
7354
7355 ins_cost(4 * INSN_COST);
7356 format %{ "ldrs $dst, $mem\t# float" %}
7357
7358 ins_encode( aarch64_enc_ldrs(dst, mem) );
7359
7360 ins_pipe(pipe_class_memory);
7361 %}
7362
7363 // Load Double
7364 instruct loadD(vRegD dst, memory8 mem)
7365 %{
7366 match(Set dst (LoadD mem));
7367 predicate(!needs_acquiring_load(n));
7368
7369 ins_cost(4 * INSN_COST);
7370 format %{ "ldrd $dst, $mem\t# double" %}
7371
7372 ins_encode( aarch64_enc_ldrd(dst, mem) );
7373
7374 ins_pipe(pipe_class_memory);
7375 %}
7376
7377
7378 // Load Int Constant
7379 instruct loadConI(iRegINoSp dst, immI src)
7380 %{
7381 match(Set dst src);
7382
7383 ins_cost(INSN_COST);
7384 format %{ "mov $dst, $src\t# int" %}
7385
7386 ins_encode( aarch64_enc_movw_imm(dst, src) );
7387
7388 ins_pipe(ialu_imm);
7389 %}
7390
7391 // Load Long Constant
7392 instruct loadConL(iRegLNoSp dst, immL src)
7393 %{
7394 match(Set dst src);
7395
7396 ins_cost(INSN_COST);
7397 format %{ "mov $dst, $src\t# long" %}
7398
7399 ins_encode( aarch64_enc_mov_imm(dst, src) );
7400
7401 ins_pipe(ialu_imm);
7402 %}
7403
7404 // Load Pointer Constant
7405
7406 instruct loadConP(iRegPNoSp dst, immP con)
7407 %{
7408 match(Set dst con);
7409
7410 ins_cost(INSN_COST * 4);
7411 format %{
7412 "mov $dst, $con\t# ptr\n\t"
7413 %}
7414
7415 ins_encode(aarch64_enc_mov_p(dst, con));
7416
7417 ins_pipe(ialu_imm);
7418 %}
7419
7420 // Load Null Pointer Constant
7421
7422 instruct loadConP0(iRegPNoSp dst, immP0 con)
7423 %{
7424 match(Set dst con);
7425
7426 ins_cost(INSN_COST);
7427 format %{ "mov $dst, $con\t# NULL ptr" %}
7428
7429 ins_encode(aarch64_enc_mov_p0(dst, con));
7430
7431 ins_pipe(ialu_imm);
7432 %}
7433
7434 // Load Pointer Constant One
7435
7436 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7437 %{
7438 match(Set dst con);
7439
7440 ins_cost(INSN_COST);
7441 format %{ "mov $dst, $con\t# NULL ptr" %}
7442
7443 ins_encode(aarch64_enc_mov_p1(dst, con));
7444
7445 ins_pipe(ialu_imm);
7446 %}
7447
7448 // Load Byte Map Base Constant
7449
7450 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7451 %{
7452 match(Set dst con);
7453
7454 ins_cost(INSN_COST);
7455 format %{ "adr $dst, $con\t# Byte Map Base" %}
7456
7457 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7458
7459 ins_pipe(ialu_imm);
7460 %}
7461
7462 // Load Narrow Pointer Constant
7463
7464 instruct loadConN(iRegNNoSp dst, immN con)
7465 %{
7466 match(Set dst con);
7467
7468 ins_cost(INSN_COST * 4);
7469 format %{ "mov $dst, $con\t# compressed ptr" %}
7470
7471 ins_encode(aarch64_enc_mov_n(dst, con));
7472
7473 ins_pipe(ialu_imm);
7474 %}
7475
7476 // Load Narrow Null Pointer Constant
7477
7478 instruct loadConN0(iRegNNoSp dst, immN0 con)
7479 %{
7480 match(Set dst con);
7481
7482 ins_cost(INSN_COST);
7483 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7484
7485 ins_encode(aarch64_enc_mov_n0(dst, con));
7486
7487 ins_pipe(ialu_imm);
7488 %}
7489
7490 // Load Narrow Klass Constant
7491
7492 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7493 %{
7494 match(Set dst con);
7495
7496 ins_cost(INSN_COST);
7497 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7498
7499 ins_encode(aarch64_enc_mov_nk(dst, con));
7500
7501 ins_pipe(ialu_imm);
7502 %}
7503
7504 // Load Packed Float Constant
7505
7506 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7507 match(Set dst con);
7508 ins_cost(INSN_COST * 4);
7509 format %{ "fmovs $dst, $con"%}
7510 ins_encode %{
7511 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7512 %}
7513
7514 ins_pipe(fp_imm_s);
7515 %}
7516
7517 // Load Float Constant
7518
7519 instruct loadConF(vRegF dst, immF con) %{
7520 match(Set dst con);
7521
7522 ins_cost(INSN_COST * 4);
7523
7524 format %{
7525 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7526 %}
7527
7528 ins_encode %{
7529 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7530 %}
7531
7532 ins_pipe(fp_load_constant_s);
7533 %}
7534
7535 // Load Packed Double Constant
7536
7537 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7538 match(Set dst con);
7539 ins_cost(INSN_COST);
7540 format %{ "fmovd $dst, $con"%}
7541 ins_encode %{
7542 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7543 %}
7544
7545 ins_pipe(fp_imm_d);
7546 %}
7547
7548 // Load Double Constant
7549
7550 instruct loadConD(vRegD dst, immD con) %{
7551 match(Set dst con);
7552
7553 ins_cost(INSN_COST * 5);
7554 format %{
7555 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7556 %}
7557
7558 ins_encode %{
7559 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7560 %}
7561
7562 ins_pipe(fp_load_constant_d);
7563 %}
7564
7565 // Store Instructions
7566
7567 // Store CMS card-mark Immediate
7568 instruct storeimmCM0(immI0 zero, memory1 mem)
7569 %{
7570 match(Set mem (StoreCM mem zero));
7571
7572 ins_cost(INSN_COST);
7573 format %{ "storestore (elided)\n\t"
7574 "strb zr, $mem\t# byte" %}
7575
7576 ins_encode(aarch64_enc_strb0(mem));
7577
7578 ins_pipe(istore_mem);
7579 %}
7580
7581 // Store CMS card-mark Immediate with intervening StoreStore
7582 // needed when using CMS with no conditional card marking
7583 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7584 %{
7585 match(Set mem (StoreCM mem zero));
7586
7587 ins_cost(INSN_COST * 2);
7588 format %{ "storestore\n\t"
7589 "dmb ishst"
7590 "\n\tstrb zr, $mem\t# byte" %}
7591
7592 ins_encode(aarch64_enc_strb0_ordered(mem));
7593
7594 ins_pipe(istore_mem);
7595 %}
7596
7597 // Store Byte
7598 instruct storeB(iRegIorL2I src, memory1 mem)
7599 %{
7600 match(Set mem (StoreB mem src));
7601 predicate(!needs_releasing_store(n));
7602
7603 ins_cost(INSN_COST);
7604 format %{ "strb $src, $mem\t# byte" %}
7605
7606 ins_encode(aarch64_enc_strb(src, mem));
7607
7608 ins_pipe(istore_reg_mem);
7609 %}
7610
7611
7612 instruct storeimmB0(immI0 zero, memory1 mem)
7613 %{
7614 match(Set mem (StoreB mem zero));
7615 predicate(!needs_releasing_store(n));
7616
7617 ins_cost(INSN_COST);
7618 format %{ "strb rscractch2, $mem\t# byte" %}
7619
7620 ins_encode(aarch64_enc_strb0(mem));
7621
7622 ins_pipe(istore_mem);
7623 %}
7624
7625 // Store Char/Short
7626 instruct storeC(iRegIorL2I src, memory2 mem)
7627 %{
7628 match(Set mem (StoreC mem src));
7629 predicate(!needs_releasing_store(n));
7630
7631 ins_cost(INSN_COST);
7632 format %{ "strh $src, $mem\t# short" %}
7633
7634 ins_encode(aarch64_enc_strh(src, mem));
7635
7636 ins_pipe(istore_reg_mem);
7637 %}
7638
7639 instruct storeimmC0(immI0 zero, memory2 mem)
7640 %{
7641 match(Set mem (StoreC mem zero));
7642 predicate(!needs_releasing_store(n));
7643
7644 ins_cost(INSN_COST);
7645 format %{ "strh zr, $mem\t# short" %}
7646
7647 ins_encode(aarch64_enc_strh0(mem));
7648
7649 ins_pipe(istore_mem);
7650 %}
7651
7652 // Store Integer
7653
7654 instruct storeI(iRegIorL2I src, memory4 mem)
7655 %{
7656 match(Set mem(StoreI mem src));
7657 predicate(!needs_releasing_store(n));
7658
7659 ins_cost(INSN_COST);
7660 format %{ "strw $src, $mem\t# int" %}
7661
7662 ins_encode(aarch64_enc_strw(src, mem));
7663
7664 ins_pipe(istore_reg_mem);
7665 %}
7666
7667 instruct storeimmI0(immI0 zero, memory4 mem)
7668 %{
7669 match(Set mem(StoreI mem zero));
7670 predicate(!needs_releasing_store(n));
7671
7672 ins_cost(INSN_COST);
7673 format %{ "strw zr, $mem\t# int" %}
7674
7675 ins_encode(aarch64_enc_strw0(mem));
7676
7677 ins_pipe(istore_mem);
7678 %}
7679
7680 // Store Long (64 bit signed)
7681 instruct storeL(iRegL src, memory8 mem)
7682 %{
7683 match(Set mem (StoreL mem src));
7684 predicate(!needs_releasing_store(n));
7685
7686 ins_cost(INSN_COST);
7687 format %{ "str $src, $mem\t# int" %}
7688
7689 ins_encode(aarch64_enc_str(src, mem));
7690
7691 ins_pipe(istore_reg_mem);
7692 %}
7693
7694 // Store Long (64 bit signed)
7695 instruct storeimmL0(immL0 zero, memory8 mem)
7696 %{
7697 match(Set mem (StoreL mem zero));
7698 predicate(!needs_releasing_store(n));
7699
7700 ins_cost(INSN_COST);
7701 format %{ "str zr, $mem\t# int" %}
7702
7703 ins_encode(aarch64_enc_str0(mem));
7704
7705 ins_pipe(istore_mem);
7706 %}
7707
7708 // Store Pointer
7709 instruct storeP(iRegP src, memory8 mem)
7710 %{
7711 match(Set mem (StoreP mem src));
7712 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7713
7714 ins_cost(INSN_COST);
7715 format %{ "str $src, $mem\t# ptr" %}
7716
7717 ins_encode(aarch64_enc_str(src, mem));
7718
7719 ins_pipe(istore_reg_mem);
7720 %}
7721
7722 // Store Pointer
7723 instruct storeimmP0(immP0 zero, memory8 mem)
7724 %{
7725 match(Set mem (StoreP mem zero));
7726 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7727
7728 ins_cost(INSN_COST);
7729 format %{ "str zr, $mem\t# ptr" %}
7730
7731 ins_encode(aarch64_enc_str0(mem));
7732
7733 ins_pipe(istore_mem);
7734 %}
7735
7736 // Store Compressed Pointer
7737 instruct storeN(iRegN src, memory4 mem)
7738 %{
7739 match(Set mem (StoreN mem src));
7740 predicate(!needs_releasing_store(n));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "strw $src, $mem\t# compressed ptr" %}
7744
7745 ins_encode(aarch64_enc_strw(src, mem));
7746
7747 ins_pipe(istore_reg_mem);
7748 %}
7749
7750 instruct storeImmN0(immN0 zero, memory4 mem)
7751 %{
7752 match(Set mem (StoreN mem zero));
7753 predicate(!needs_releasing_store(n));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "strw zr, $mem\t# compressed ptr" %}
7757
7758 ins_encode(aarch64_enc_strw0(mem));
7759
7760 ins_pipe(istore_mem);
7761 %}
7762
7763 // Store Float
7764 instruct storeF(vRegF src, memory4 mem)
7765 %{
7766 match(Set mem (StoreF mem src));
7767 predicate(!needs_releasing_store(n));
7768
7769 ins_cost(INSN_COST);
7770 format %{ "strs $src, $mem\t# float" %}
7771
7772 ins_encode( aarch64_enc_strs(src, mem) );
7773
7774 ins_pipe(pipe_class_memory);
7775 %}
7776
7777 // TODO
7778 // implement storeImmF0 and storeFImmPacked
7779
7780 // Store Double
7781 instruct storeD(vRegD src, memory8 mem)
7782 %{
7783 match(Set mem (StoreD mem src));
7784 predicate(!needs_releasing_store(n));
7785
7786 ins_cost(INSN_COST);
7787 format %{ "strd $src, $mem\t# double" %}
7788
7789 ins_encode( aarch64_enc_strd(src, mem) );
7790
7791 ins_pipe(pipe_class_memory);
7792 %}
7793
7794 // Store Compressed Klass Pointer
7795 instruct storeNKlass(iRegN src, memory4 mem)
7796 %{
7797 predicate(!needs_releasing_store(n));
7798 match(Set mem (StoreNKlass mem src));
7799
7800 ins_cost(INSN_COST);
7801 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7802
7803 ins_encode(aarch64_enc_strw(src, mem));
7804
7805 ins_pipe(istore_reg_mem);
7806 %}
7807
7808 // TODO
7809 // implement storeImmD0 and storeDImmPacked
7810
7811 // prefetch instructions
7812 // Must be safe to execute with invalid address (cannot fault).
7813
7814 instruct prefetchalloc( memory8 mem ) %{
7815 match(PrefetchAllocation mem);
7816
7817 ins_cost(INSN_COST);
7818 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7819
7820 ins_encode( aarch64_enc_prefetchw(mem) );
7821
7822 ins_pipe(iload_prefetch);
7823 %}
7824
7825 // ---------------- volatile loads and stores ----------------
7826
7827 // Load Byte (8 bit signed)
7828 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7829 %{
7830 match(Set dst (LoadB mem));
7831
7832 ins_cost(VOLATILE_REF_COST);
7833 format %{ "ldarsb $dst, $mem\t# byte" %}
7834
7835 ins_encode(aarch64_enc_ldarsb(dst, mem));
7836
7837 ins_pipe(pipe_serial);
7838 %}
7839
7840 // Load Byte (8 bit signed) into long
7841 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7842 %{
7843 match(Set dst (ConvI2L (LoadB mem)));
7844
7845 ins_cost(VOLATILE_REF_COST);
7846 format %{ "ldarsb $dst, $mem\t# byte" %}
7847
7848 ins_encode(aarch64_enc_ldarsb(dst, mem));
7849
7850 ins_pipe(pipe_serial);
7851 %}
7852
7853 // Load Byte (8 bit unsigned)
7854 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7855 %{
7856 match(Set dst (LoadUB mem));
7857
7858 ins_cost(VOLATILE_REF_COST);
7859 format %{ "ldarb $dst, $mem\t# byte" %}
7860
7861 ins_encode(aarch64_enc_ldarb(dst, mem));
7862
7863 ins_pipe(pipe_serial);
7864 %}
7865
7866 // Load Byte (8 bit unsigned) into long
7867 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7868 %{
7869 match(Set dst (ConvI2L (LoadUB mem)));
7870
7871 ins_cost(VOLATILE_REF_COST);
7872 format %{ "ldarb $dst, $mem\t# byte" %}
7873
7874 ins_encode(aarch64_enc_ldarb(dst, mem));
7875
7876 ins_pipe(pipe_serial);
7877 %}
7878
7879 // Load Short (16 bit signed)
7880 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7881 %{
7882 match(Set dst (LoadS mem));
7883
7884 ins_cost(VOLATILE_REF_COST);
7885 format %{ "ldarshw $dst, $mem\t# short" %}
7886
7887 ins_encode(aarch64_enc_ldarshw(dst, mem));
7888
7889 ins_pipe(pipe_serial);
7890 %}
7891
7892 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7893 %{
7894 match(Set dst (LoadUS mem));
7895
7896 ins_cost(VOLATILE_REF_COST);
7897 format %{ "ldarhw $dst, $mem\t# short" %}
7898
7899 ins_encode(aarch64_enc_ldarhw(dst, mem));
7900
7901 ins_pipe(pipe_serial);
7902 %}
7903
7904 // Load Short/Char (16 bit unsigned) into long
7905 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7906 %{
7907 match(Set dst (ConvI2L (LoadUS mem)));
7908
7909 ins_cost(VOLATILE_REF_COST);
7910 format %{ "ldarh $dst, $mem\t# short" %}
7911
7912 ins_encode(aarch64_enc_ldarh(dst, mem));
7913
7914 ins_pipe(pipe_serial);
7915 %}
7916
7917 // Load Short/Char (16 bit signed) into long
7918 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7919 %{
7920 match(Set dst (ConvI2L (LoadS mem)));
7921
7922 ins_cost(VOLATILE_REF_COST);
7923 format %{ "ldarh $dst, $mem\t# short" %}
7924
7925 ins_encode(aarch64_enc_ldarsh(dst, mem));
7926
7927 ins_pipe(pipe_serial);
7928 %}
7929
7930 // Load Integer (32 bit signed)
7931 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7932 %{
7933 match(Set dst (LoadI mem));
7934
7935 ins_cost(VOLATILE_REF_COST);
7936 format %{ "ldarw $dst, $mem\t# int" %}
7937
7938 ins_encode(aarch64_enc_ldarw(dst, mem));
7939
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 // Load Integer (32 bit unsigned) into long
7944 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7945 %{
7946 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7947
7948 ins_cost(VOLATILE_REF_COST);
7949 format %{ "ldarw $dst, $mem\t# int" %}
7950
7951 ins_encode(aarch64_enc_ldarw(dst, mem));
7952
7953 ins_pipe(pipe_serial);
7954 %}
7955
7956 // Load Long (64 bit signed)
7957 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7958 %{
7959 match(Set dst (LoadL mem));
7960
7961 ins_cost(VOLATILE_REF_COST);
7962 format %{ "ldar $dst, $mem\t# int" %}
7963
7964 ins_encode(aarch64_enc_ldar(dst, mem));
7965
7966 ins_pipe(pipe_serial);
7967 %}
7968
7969 // Load Pointer
7970 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7971 %{
7972 match(Set dst (LoadP mem));
7973 predicate(n->as_Load()->barrier_data() == 0);
7974
7975 ins_cost(VOLATILE_REF_COST);
7976 format %{ "ldar $dst, $mem\t# ptr" %}
7977
7978 ins_encode(aarch64_enc_ldar(dst, mem));
7979
7980 ins_pipe(pipe_serial);
7981 %}
7982
7983 // Load Compressed Pointer
7984 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7985 %{
7986 match(Set dst (LoadN mem));
7987
7988 ins_cost(VOLATILE_REF_COST);
7989 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7990
7991 ins_encode(aarch64_enc_ldarw(dst, mem));
7992
7993 ins_pipe(pipe_serial);
7994 %}
7995
7996 // Load Float
7997 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7998 %{
7999 match(Set dst (LoadF mem));
8000
8001 ins_cost(VOLATILE_REF_COST);
8002 format %{ "ldars $dst, $mem\t# float" %}
8003
8004 ins_encode( aarch64_enc_fldars(dst, mem) );
8005
8006 ins_pipe(pipe_serial);
8007 %}
8008
8009 // Load Double
8010 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8011 %{
8012 match(Set dst (LoadD mem));
8013
8014 ins_cost(VOLATILE_REF_COST);
8015 format %{ "ldard $dst, $mem\t# double" %}
8016
8017 ins_encode( aarch64_enc_fldard(dst, mem) );
8018
8019 ins_pipe(pipe_serial);
8020 %}
8021
8022 // Store Byte
8023 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8024 %{
8025 match(Set mem (StoreB mem src));
8026
8027 ins_cost(VOLATILE_REF_COST);
8028 format %{ "stlrb $src, $mem\t# byte" %}
8029
8030 ins_encode(aarch64_enc_stlrb(src, mem));
8031
8032 ins_pipe(pipe_class_memory);
8033 %}
8034
8035 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8036 %{
8037 match(Set mem (StoreB mem zero));
8038
8039 ins_cost(VOLATILE_REF_COST);
8040 format %{ "stlrb zr, $mem\t# byte" %}
8041
8042 ins_encode(aarch64_enc_stlrb0(mem));
8043
8044 ins_pipe(pipe_class_memory);
8045 %}
8046
8047 // Store Char/Short
8048 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8049 %{
8050 match(Set mem (StoreC mem src));
8051
8052 ins_cost(VOLATILE_REF_COST);
8053 format %{ "stlrh $src, $mem\t# short" %}
8054
8055 ins_encode(aarch64_enc_stlrh(src, mem));
8056
8057 ins_pipe(pipe_class_memory);
8058 %}
8059
8060 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8061 %{
8062 match(Set mem (StoreC mem zero));
8063
8064 ins_cost(VOLATILE_REF_COST);
8065 format %{ "stlrh zr, $mem\t# short" %}
8066
8067 ins_encode(aarch64_enc_stlrh0(mem));
8068
8069 ins_pipe(pipe_class_memory);
8070 %}
8071
8072 // Store Integer
8073
8074 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8075 %{
8076 match(Set mem(StoreI mem src));
8077
8078 ins_cost(VOLATILE_REF_COST);
8079 format %{ "stlrw $src, $mem\t# int" %}
8080
8081 ins_encode(aarch64_enc_stlrw(src, mem));
8082
8083 ins_pipe(pipe_class_memory);
8084 %}
8085
8086 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8087 %{
8088 match(Set mem(StoreI mem zero));
8089
8090 ins_cost(VOLATILE_REF_COST);
8091 format %{ "stlrw zr, $mem\t# int" %}
8092
8093 ins_encode(aarch64_enc_stlrw0(mem));
8094
8095 ins_pipe(pipe_class_memory);
8096 %}
8097
8098 // Store Long (64 bit signed)
8099 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8100 %{
8101 match(Set mem (StoreL mem src));
8102
8103 ins_cost(VOLATILE_REF_COST);
8104 format %{ "stlr $src, $mem\t# int" %}
8105
8106 ins_encode(aarch64_enc_stlr(src, mem));
8107
8108 ins_pipe(pipe_class_memory);
8109 %}
8110
8111 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8112 %{
8113 match(Set mem (StoreL mem zero));
8114
8115 ins_cost(VOLATILE_REF_COST);
8116 format %{ "stlr zr, $mem\t# int" %}
8117
8118 ins_encode(aarch64_enc_stlr0(mem));
8119
8120 ins_pipe(pipe_class_memory);
8121 %}
8122
8123 // Store Pointer
8124 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8125 %{
8126 match(Set mem (StoreP mem src));
8127 predicate(n->as_Store()->barrier_data() == 0);
8128
8129 ins_cost(VOLATILE_REF_COST);
8130 format %{ "stlr $src, $mem\t# ptr" %}
8131
8132 ins_encode(aarch64_enc_stlr(src, mem));
8133
8134 ins_pipe(pipe_class_memory);
8135 %}
8136
8137 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8138 %{
8139 match(Set mem (StoreP mem zero));
8140 predicate(n->as_Store()->barrier_data() == 0);
8141
8142 ins_cost(VOLATILE_REF_COST);
8143 format %{ "stlr zr, $mem\t# ptr" %}
8144
8145 ins_encode(aarch64_enc_stlr0(mem));
8146
8147 ins_pipe(pipe_class_memory);
8148 %}
8149
8150 // Store Compressed Pointer
8151 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8152 %{
8153 match(Set mem (StoreN mem src));
8154
8155 ins_cost(VOLATILE_REF_COST);
8156 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8157
8158 ins_encode(aarch64_enc_stlrw(src, mem));
8159
8160 ins_pipe(pipe_class_memory);
8161 %}
8162
8163 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8164 %{
8165 match(Set mem (StoreN mem zero));
8166
8167 ins_cost(VOLATILE_REF_COST);
8168 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8169
8170 ins_encode(aarch64_enc_stlrw0(mem));
8171
8172 ins_pipe(pipe_class_memory);
8173 %}
8174
8175 // Store Float
8176 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8177 %{
8178 match(Set mem (StoreF mem src));
8179
8180 ins_cost(VOLATILE_REF_COST);
8181 format %{ "stlrs $src, $mem\t# float" %}
8182
8183 ins_encode( aarch64_enc_fstlrs(src, mem) );
8184
8185 ins_pipe(pipe_class_memory);
8186 %}
8187
8188 // TODO
8189 // implement storeImmF0 and storeFImmPacked
8190
8191 // Store Double
8192 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8193 %{
8194 match(Set mem (StoreD mem src));
8195
8196 ins_cost(VOLATILE_REF_COST);
8197 format %{ "stlrd $src, $mem\t# double" %}
8198
8199 ins_encode( aarch64_enc_fstlrd(src, mem) );
8200
8201 ins_pipe(pipe_class_memory);
8202 %}
8203
8204 // ---------------- end of volatile loads and stores ----------------
8205
8206 instruct cacheWB(indirect addr)
8207 %{
8208 predicate(VM_Version::supports_data_cache_line_flush());
8209 match(CacheWB addr);
8210
8211 ins_cost(100);
8212 format %{"cache wb $addr" %}
8213 ins_encode %{
8214 assert($addr->index_position() < 0, "should be");
8215 assert($addr$$disp == 0, "should be");
8216 __ cache_wb(Address($addr$$base$$Register, 0));
8217 %}
8218 ins_pipe(pipe_slow); // XXX
8219 %}
8220
8221 instruct cacheWBPreSync()
8222 %{
8223 predicate(VM_Version::supports_data_cache_line_flush());
8224 match(CacheWBPreSync);
8225
8226 ins_cost(100);
8227 format %{"cache wb presync" %}
8228 ins_encode %{
8229 __ cache_wbsync(true);
8230 %}
8231 ins_pipe(pipe_slow); // XXX
8232 %}
8233
8234 instruct cacheWBPostSync()
8235 %{
8236 predicate(VM_Version::supports_data_cache_line_flush());
8237 match(CacheWBPostSync);
8238
8239 ins_cost(100);
8240 format %{"cache wb postsync" %}
8241 ins_encode %{
8242 __ cache_wbsync(false);
8243 %}
8244 ins_pipe(pipe_slow); // XXX
8245 %}
8246
8247 // ============================================================================
8248 // BSWAP Instructions
8249
8250 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8251 match(Set dst (ReverseBytesI src));
8252
8253 ins_cost(INSN_COST);
8254 format %{ "revw $dst, $src" %}
8255
8256 ins_encode %{
8257 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8258 %}
8259
8260 ins_pipe(ialu_reg);
8261 %}
8262
8263 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8264 match(Set dst (ReverseBytesL src));
8265
8266 ins_cost(INSN_COST);
8267 format %{ "rev $dst, $src" %}
8268
8269 ins_encode %{
8270 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8271 %}
8272
8273 ins_pipe(ialu_reg);
8274 %}
8275
8276 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8277 match(Set dst (ReverseBytesUS src));
8278
8279 ins_cost(INSN_COST);
8280 format %{ "rev16w $dst, $src" %}
8281
8282 ins_encode %{
8283 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8284 %}
8285
8286 ins_pipe(ialu_reg);
8287 %}
8288
8289 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8290 match(Set dst (ReverseBytesS src));
8291
8292 ins_cost(INSN_COST);
8293 format %{ "rev16w $dst, $src\n\t"
8294 "sbfmw $dst, $dst, #0, #15" %}
8295
8296 ins_encode %{
8297 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8298 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8299 %}
8300
8301 ins_pipe(ialu_reg);
8302 %}
8303
8304 // ============================================================================
8305 // Zero Count Instructions
8306
8307 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8308 match(Set dst (CountLeadingZerosI src));
8309
8310 ins_cost(INSN_COST);
8311 format %{ "clzw $dst, $src" %}
8312 ins_encode %{
8313 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8314 %}
8315
8316 ins_pipe(ialu_reg);
8317 %}
8318
8319 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8320 match(Set dst (CountLeadingZerosL src));
8321
8322 ins_cost(INSN_COST);
8323 format %{ "clz $dst, $src" %}
8324 ins_encode %{
8325 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8326 %}
8327
8328 ins_pipe(ialu_reg);
8329 %}
8330
8331 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8332 match(Set dst (CountTrailingZerosI src));
8333
8334 ins_cost(INSN_COST * 2);
8335 format %{ "rbitw $dst, $src\n\t"
8336 "clzw $dst, $dst" %}
8337 ins_encode %{
8338 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8339 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8340 %}
8341
8342 ins_pipe(ialu_reg);
8343 %}
8344
8345 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8346 match(Set dst (CountTrailingZerosL src));
8347
8348 ins_cost(INSN_COST * 2);
8349 format %{ "rbit $dst, $src\n\t"
8350 "clz $dst, $dst" %}
8351 ins_encode %{
8352 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8353 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8354 %}
8355
8356 ins_pipe(ialu_reg);
8357 %}
8358
8359 //---------- Population Count Instructions -------------------------------------
8360 //
8361
8362 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8363 match(Set dst (PopCountI src));
8364 effect(TEMP tmp);
8365 ins_cost(INSN_COST * 13);
8366
8367 format %{ "movw $src, $src\n\t"
8368 "mov $tmp, $src\t# vector (1D)\n\t"
8369 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8370 "addv $tmp, $tmp\t# vector (8B)\n\t"
8371 "mov $dst, $tmp\t# vector (1D)" %}
8372 ins_encode %{
8373 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8374 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8375 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8376 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8377 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8378 %}
8379
8380 ins_pipe(pipe_class_default);
8381 %}
8382
8383 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8384 match(Set dst (PopCountI (LoadI mem)));
8385 effect(TEMP tmp);
8386 ins_cost(INSN_COST * 13);
8387
8388 format %{ "ldrs $tmp, $mem\n\t"
8389 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8390 "addv $tmp, $tmp\t# vector (8B)\n\t"
8391 "mov $dst, $tmp\t# vector (1D)" %}
8392 ins_encode %{
8393 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8394 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8395 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8396 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8397 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8398 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8399 %}
8400
8401 ins_pipe(pipe_class_default);
8402 %}
8403
8404 // Note: Long.bitCount(long) returns an int.
8405 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8406 match(Set dst (PopCountL src));
8407 effect(TEMP tmp);
8408 ins_cost(INSN_COST * 13);
8409
8410 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8411 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8412 "addv $tmp, $tmp\t# vector (8B)\n\t"
8413 "mov $dst, $tmp\t# vector (1D)" %}
8414 ins_encode %{
8415 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8416 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8417 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8418 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8419 %}
8420
8421 ins_pipe(pipe_class_default);
8422 %}
8423
8424 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8425 match(Set dst (PopCountL (LoadL mem)));
8426 effect(TEMP tmp);
8427 ins_cost(INSN_COST * 13);
8428
8429 format %{ "ldrd $tmp, $mem\n\t"
8430 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8431 "addv $tmp, $tmp\t# vector (8B)\n\t"
8432 "mov $dst, $tmp\t# vector (1D)" %}
8433 ins_encode %{
8434 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8435 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8436 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8437 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8438 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8439 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8440 %}
8441
8442 ins_pipe(pipe_class_default);
8443 %}
8444
8445 // ============================================================================
8446 // MemBar Instruction
8447
8448 instruct load_fence() %{
8449 match(LoadFence);
8450 ins_cost(VOLATILE_REF_COST);
8451
8452 format %{ "load_fence" %}
8453
8454 ins_encode %{
8455 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8456 %}
8457 ins_pipe(pipe_serial);
8458 %}
8459
8460 instruct unnecessary_membar_acquire() %{
8461 predicate(unnecessary_acquire(n));
8462 match(MemBarAcquire);
8463 ins_cost(0);
8464
8465 format %{ "membar_acquire (elided)" %}
8466
8467 ins_encode %{
8468 __ block_comment("membar_acquire (elided)");
8469 %}
8470
8471 ins_pipe(pipe_class_empty);
8472 %}
8473
8474 instruct membar_acquire() %{
8475 match(MemBarAcquire);
8476 ins_cost(VOLATILE_REF_COST);
8477
8478 format %{ "membar_acquire\n\t"
8479 "dmb ish" %}
8480
8481 ins_encode %{
8482 __ block_comment("membar_acquire");
8483 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8484 %}
8485
8486 ins_pipe(pipe_serial);
8487 %}
8488
8489
8490 instruct membar_acquire_lock() %{
8491 match(MemBarAcquireLock);
8492 ins_cost(VOLATILE_REF_COST);
8493
8494 format %{ "membar_acquire_lock (elided)" %}
8495
8496 ins_encode %{
8497 __ block_comment("membar_acquire_lock (elided)");
8498 %}
8499
8500 ins_pipe(pipe_serial);
8501 %}
8502
8503 instruct store_fence() %{
8504 match(StoreFence);
8505 ins_cost(VOLATILE_REF_COST);
8506
8507 format %{ "store_fence" %}
8508
8509 ins_encode %{
8510 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8511 %}
8512 ins_pipe(pipe_serial);
8513 %}
8514
8515 instruct unnecessary_membar_release() %{
8516 predicate(unnecessary_release(n));
8517 match(MemBarRelease);
8518 ins_cost(0);
8519
8520 format %{ "membar_release (elided)" %}
8521
8522 ins_encode %{
8523 __ block_comment("membar_release (elided)");
8524 %}
8525 ins_pipe(pipe_serial);
8526 %}
8527
8528 instruct membar_release() %{
8529 match(MemBarRelease);
8530 ins_cost(VOLATILE_REF_COST);
8531
8532 format %{ "membar_release\n\t"
8533 "dmb ish" %}
8534
8535 ins_encode %{
8536 __ block_comment("membar_release");
8537 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8538 %}
8539 ins_pipe(pipe_serial);
8540 %}
8541
8542 instruct membar_storestore() %{
8543 match(MemBarStoreStore);
8544 match(StoreStoreFence);
8545 ins_cost(VOLATILE_REF_COST);
8546
8547 format %{ "MEMBAR-store-store" %}
8548
8549 ins_encode %{
8550 __ membar(Assembler::StoreStore);
8551 %}
8552 ins_pipe(pipe_serial);
8553 %}
8554
8555 instruct membar_release_lock() %{
8556 match(MemBarReleaseLock);
8557 ins_cost(VOLATILE_REF_COST);
8558
8559 format %{ "membar_release_lock (elided)" %}
8560
8561 ins_encode %{
8562 __ block_comment("membar_release_lock (elided)");
8563 %}
8564
8565 ins_pipe(pipe_serial);
8566 %}
8567
8568 instruct unnecessary_membar_volatile() %{
8569 predicate(unnecessary_volatile(n));
8570 match(MemBarVolatile);
8571 ins_cost(0);
8572
8573 format %{ "membar_volatile (elided)" %}
8574
8575 ins_encode %{
8576 __ block_comment("membar_volatile (elided)");
8577 %}
8578
8579 ins_pipe(pipe_serial);
8580 %}
8581
8582 instruct membar_volatile() %{
8583 match(MemBarVolatile);
8584 ins_cost(VOLATILE_REF_COST*100);
8585
8586 format %{ "membar_volatile\n\t"
8587 "dmb ish"%}
8588
8589 ins_encode %{
8590 __ block_comment("membar_volatile");
8591 __ membar(Assembler::StoreLoad);
8592 %}
8593
8594 ins_pipe(pipe_serial);
8595 %}
8596
8597 // ============================================================================
8598 // Cast/Convert Instructions
8599
8600 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8601 match(Set dst (CastX2P src));
8602
8603 ins_cost(INSN_COST);
8604 format %{ "mov $dst, $src\t# long -> ptr" %}
8605
8606 ins_encode %{
8607 if ($dst$$reg != $src$$reg) {
8608 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8609 }
8610 %}
8611
8612 ins_pipe(ialu_reg);
8613 %}
8614
8615 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8616 match(Set dst (CastP2X src));
8617
8618 ins_cost(INSN_COST);
8619 format %{ "mov $dst, $src\t# ptr -> long" %}
8620
8621 ins_encode %{
8622 if ($dst$$reg != $src$$reg) {
8623 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8624 }
8625 %}
8626
8627 ins_pipe(ialu_reg);
8628 %}
8629
8630 // Convert oop into int for vectors alignment masking
8631 instruct convP2I(iRegINoSp dst, iRegP src) %{
8632 match(Set dst (ConvL2I (CastP2X src)));
8633
8634 ins_cost(INSN_COST);
8635 format %{ "movw $dst, $src\t# ptr -> int" %}
8636 ins_encode %{
8637 __ movw($dst$$Register, $src$$Register);
8638 %}
8639
8640 ins_pipe(ialu_reg);
8641 %}
8642
8643 // Convert compressed oop into int for vectors alignment masking
8644 // in case of 32bit oops (heap < 4Gb).
8645 instruct convN2I(iRegINoSp dst, iRegN src)
8646 %{
8647 predicate(CompressedOops::shift() == 0);
8648 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8649
8650 ins_cost(INSN_COST);
8651 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8652 ins_encode %{
8653 __ movw($dst$$Register, $src$$Register);
8654 %}
8655
8656 ins_pipe(ialu_reg);
8657 %}
8658
8659
8660 // Convert oop pointer into compressed form
8661 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8662 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8663 match(Set dst (EncodeP src));
8664 effect(KILL cr);
8665 ins_cost(INSN_COST * 3);
8666 format %{ "encode_heap_oop $dst, $src" %}
8667 ins_encode %{
8668 Register s = $src$$Register;
8669 Register d = $dst$$Register;
8670 __ encode_heap_oop(d, s);
8671 %}
8672 ins_pipe(ialu_reg);
8673 %}
8674
8675 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8676 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8677 match(Set dst (EncodeP src));
8678 ins_cost(INSN_COST * 3);
8679 format %{ "encode_heap_oop_not_null $dst, $src" %}
8680 ins_encode %{
8681 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8682 %}
8683 ins_pipe(ialu_reg);
8684 %}
8685
8686 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8687 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8688 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8689 match(Set dst (DecodeN src));
8690 ins_cost(INSN_COST * 3);
8691 format %{ "decode_heap_oop $dst, $src" %}
8692 ins_encode %{
8693 Register s = $src$$Register;
8694 Register d = $dst$$Register;
8695 __ decode_heap_oop(d, s);
8696 %}
8697 ins_pipe(ialu_reg);
8698 %}
8699
8700 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8701 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8702 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8703 match(Set dst (DecodeN src));
8704 ins_cost(INSN_COST * 3);
8705 format %{ "decode_heap_oop_not_null $dst, $src" %}
8706 ins_encode %{
8707 Register s = $src$$Register;
8708 Register d = $dst$$Register;
8709 __ decode_heap_oop_not_null(d, s);
8710 %}
8711 ins_pipe(ialu_reg);
8712 %}
8713
8714 // n.b. AArch64 implementations of encode_klass_not_null and
8715 // decode_klass_not_null do not modify the flags register so, unlike
8716 // Intel, we don't kill CR as a side effect here
8717
8718 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8719 match(Set dst (EncodePKlass src));
8720
8721 ins_cost(INSN_COST * 3);
8722 format %{ "encode_klass_not_null $dst,$src" %}
8723
8724 ins_encode %{
8725 Register src_reg = as_Register($src$$reg);
8726 Register dst_reg = as_Register($dst$$reg);
8727 __ encode_klass_not_null(dst_reg, src_reg);
8728 %}
8729
8730 ins_pipe(ialu_reg);
8731 %}
8732
8733 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8734 match(Set dst (DecodeNKlass src));
8735
8736 ins_cost(INSN_COST * 3);
8737 format %{ "decode_klass_not_null $dst,$src" %}
8738
8739 ins_encode %{
8740 Register src_reg = as_Register($src$$reg);
8741 Register dst_reg = as_Register($dst$$reg);
8742 if (dst_reg != src_reg) {
8743 __ decode_klass_not_null(dst_reg, src_reg);
8744 } else {
8745 __ decode_klass_not_null(dst_reg);
8746 }
8747 %}
8748
8749 ins_pipe(ialu_reg);
8750 %}
8751
8752 instruct checkCastPP(iRegPNoSp dst)
8753 %{
8754 match(Set dst (CheckCastPP dst));
8755
8756 size(0);
8757 format %{ "# checkcastPP of $dst" %}
8758 ins_encode(/* empty encoding */);
8759 ins_pipe(pipe_class_empty);
8760 %}
8761
8762 instruct castPP(iRegPNoSp dst)
8763 %{
8764 match(Set dst (CastPP dst));
8765
8766 size(0);
8767 format %{ "# castPP of $dst" %}
8768 ins_encode(/* empty encoding */);
8769 ins_pipe(pipe_class_empty);
8770 %}
8771
8772 instruct castII(iRegI dst)
8773 %{
8774 match(Set dst (CastII dst));
8775
8776 size(0);
8777 format %{ "# castII of $dst" %}
8778 ins_encode(/* empty encoding */);
8779 ins_cost(0);
8780 ins_pipe(pipe_class_empty);
8781 %}
8782
8783 instruct castLL(iRegL dst)
8784 %{
8785 match(Set dst (CastLL dst));
8786
8787 size(0);
8788 format %{ "# castLL of $dst" %}
8789 ins_encode(/* empty encoding */);
8790 ins_cost(0);
8791 ins_pipe(pipe_class_empty);
8792 %}
8793
8794 instruct castFF(vRegF dst)
8795 %{
8796 match(Set dst (CastFF dst));
8797
8798 size(0);
8799 format %{ "# castFF of $dst" %}
8800 ins_encode(/* empty encoding */);
8801 ins_cost(0);
8802 ins_pipe(pipe_class_empty);
8803 %}
8804
8805 instruct castDD(vRegD dst)
8806 %{
8807 match(Set dst (CastDD dst));
8808
8809 size(0);
8810 format %{ "# castDD of $dst" %}
8811 ins_encode(/* empty encoding */);
8812 ins_cost(0);
8813 ins_pipe(pipe_class_empty);
8814 %}
8815
8816 instruct castVV(vReg dst)
8817 %{
8818 match(Set dst (CastVV dst));
8819
8820 size(0);
8821 format %{ "# castVV of $dst" %}
8822 ins_encode(/* empty encoding */);
8823 ins_cost(0);
8824 ins_pipe(pipe_class_empty);
8825 %}
8826
8827 instruct castVVMask(pRegGov dst)
8828 %{
8829 match(Set dst (CastVV dst));
8830
8831 size(0);
8832 format %{ "# castVV of $dst" %}
8833 ins_encode(/* empty encoding */);
8834 ins_cost(0);
8835 ins_pipe(pipe_class_empty);
8836 %}
8837
8838 // ============================================================================
8839 // Atomic operation instructions
8840 //
8841
8842 // standard CompareAndSwapX when we are using barriers
8843 // these have higher priority than the rules selected by a predicate
8844
8845 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8846 // can't match them
8847
8848 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8849
8850 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8851 ins_cost(2 * VOLATILE_REF_COST);
8852
8853 effect(KILL cr);
8854
8855 format %{
8856 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8857 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8858 %}
8859
8860 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8861 aarch64_enc_cset_eq(res));
8862
8863 ins_pipe(pipe_slow);
8864 %}
8865
8866 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8867
8868 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8869 ins_cost(2 * VOLATILE_REF_COST);
8870
8871 effect(KILL cr);
8872
8873 format %{
8874 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8875 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8876 %}
8877
8878 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8879 aarch64_enc_cset_eq(res));
8880
8881 ins_pipe(pipe_slow);
8882 %}
8883
8884 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8885
8886 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8887 ins_cost(2 * VOLATILE_REF_COST);
8888
8889 effect(KILL cr);
8890
8891 format %{
8892 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8893 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8894 %}
8895
8896 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8897 aarch64_enc_cset_eq(res));
8898
8899 ins_pipe(pipe_slow);
8900 %}
8901
8902 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8903
8904 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8905 ins_cost(2 * VOLATILE_REF_COST);
8906
8907 effect(KILL cr);
8908
8909 format %{
8910 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8911 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8912 %}
8913
8914 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8915 aarch64_enc_cset_eq(res));
8916
8917 ins_pipe(pipe_slow);
8918 %}
8919
8920 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8921
8922 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8923 predicate(n->as_LoadStore()->barrier_data() == 0);
8924 ins_cost(2 * VOLATILE_REF_COST);
8925
8926 effect(KILL cr);
8927
8928 format %{
8929 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8930 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8931 %}
8932
8933 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8934 aarch64_enc_cset_eq(res));
8935
8936 ins_pipe(pipe_slow);
8937 %}
8938
8939 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8940
8941 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8942 ins_cost(2 * VOLATILE_REF_COST);
8943
8944 effect(KILL cr);
8945
8946 format %{
8947 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8948 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8949 %}
8950
8951 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8952 aarch64_enc_cset_eq(res));
8953
8954 ins_pipe(pipe_slow);
8955 %}
8956
8957 // alternative CompareAndSwapX when we are eliding barriers
8958
8959 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8960
8961 predicate(needs_acquiring_load_exclusive(n));
8962 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8963 ins_cost(VOLATILE_REF_COST);
8964
8965 effect(KILL cr);
8966
8967 format %{
8968 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8969 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8970 %}
8971
8972 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8973 aarch64_enc_cset_eq(res));
8974
8975 ins_pipe(pipe_slow);
8976 %}
8977
8978 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8979
8980 predicate(needs_acquiring_load_exclusive(n));
8981 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8982 ins_cost(VOLATILE_REF_COST);
8983
8984 effect(KILL cr);
8985
8986 format %{
8987 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8988 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8989 %}
8990
8991 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8992 aarch64_enc_cset_eq(res));
8993
8994 ins_pipe(pipe_slow);
8995 %}
8996
8997 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8998
8999 predicate(needs_acquiring_load_exclusive(n));
9000 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9001 ins_cost(VOLATILE_REF_COST);
9002
9003 effect(KILL cr);
9004
9005 format %{
9006 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9007 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9008 %}
9009
9010 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9011 aarch64_enc_cset_eq(res));
9012
9013 ins_pipe(pipe_slow);
9014 %}
9015
9016 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9017
9018 predicate(needs_acquiring_load_exclusive(n));
9019 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9020 ins_cost(VOLATILE_REF_COST);
9021
9022 effect(KILL cr);
9023
9024 format %{
9025 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9026 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9027 %}
9028
9029 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9030 aarch64_enc_cset_eq(res));
9031
9032 ins_pipe(pipe_slow);
9033 %}
9034
9035 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9036
9037 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9038 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9039 ins_cost(VOLATILE_REF_COST);
9040
9041 effect(KILL cr);
9042
9043 format %{
9044 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9045 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9046 %}
9047
9048 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9049 aarch64_enc_cset_eq(res));
9050
9051 ins_pipe(pipe_slow);
9052 %}
9053
9054 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9055
9056 predicate(needs_acquiring_load_exclusive(n));
9057 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9058 ins_cost(VOLATILE_REF_COST);
9059
9060 effect(KILL cr);
9061
9062 format %{
9063 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9064 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9065 %}
9066
9067 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9068 aarch64_enc_cset_eq(res));
9069
9070 ins_pipe(pipe_slow);
9071 %}
9072
9073
9074 // ---------------------------------------------------------------------
9075
9076 // BEGIN This section of the file is automatically generated. Do not edit --------------
9077
9078 // Sundry CAS operations. Note that release is always true,
9079 // regardless of the memory ordering of the CAS. This is because we
9080 // need the volatile case to be sequentially consistent but there is
9081 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9082 // can't check the type of memory ordering here, so we always emit a
9083 // STLXR.
9084
9085 // This section is generated from cas.m4
9086
9087
9088 // This pattern is generated automatically from cas.m4.
9089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9090 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9091 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9092 ins_cost(2 * VOLATILE_REF_COST);
9093 effect(TEMP_DEF res, KILL cr);
9094 format %{
9095 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9096 %}
9097 ins_encode %{
9098 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9099 Assembler::byte, /*acquire*/ false, /*release*/ true,
9100 /*weak*/ false, $res$$Register);
9101 __ sxtbw($res$$Register, $res$$Register);
9102 %}
9103 ins_pipe(pipe_slow);
9104 %}
9105
9106 // This pattern is generated automatically from cas.m4.
9107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9108 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9109 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9110 ins_cost(2 * VOLATILE_REF_COST);
9111 effect(TEMP_DEF res, KILL cr);
9112 format %{
9113 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9114 %}
9115 ins_encode %{
9116 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9117 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9118 /*weak*/ false, $res$$Register);
9119 __ sxthw($res$$Register, $res$$Register);
9120 %}
9121 ins_pipe(pipe_slow);
9122 %}
9123
9124 // This pattern is generated automatically from cas.m4.
9125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9126 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9127 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9128 ins_cost(2 * VOLATILE_REF_COST);
9129 effect(TEMP_DEF res, KILL cr);
9130 format %{
9131 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9132 %}
9133 ins_encode %{
9134 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9135 Assembler::word, /*acquire*/ false, /*release*/ true,
9136 /*weak*/ false, $res$$Register);
9137 %}
9138 ins_pipe(pipe_slow);
9139 %}
9140
9141 // This pattern is generated automatically from cas.m4.
9142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9143 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9144 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9145 ins_cost(2 * VOLATILE_REF_COST);
9146 effect(TEMP_DEF res, KILL cr);
9147 format %{
9148 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9149 %}
9150 ins_encode %{
9151 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9152 Assembler::xword, /*acquire*/ false, /*release*/ true,
9153 /*weak*/ false, $res$$Register);
9154 %}
9155 ins_pipe(pipe_slow);
9156 %}
9157
9158 // This pattern is generated automatically from cas.m4.
9159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9160 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9161 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9162 ins_cost(2 * VOLATILE_REF_COST);
9163 effect(TEMP_DEF res, KILL cr);
9164 format %{
9165 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9166 %}
9167 ins_encode %{
9168 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9169 Assembler::word, /*acquire*/ false, /*release*/ true,
9170 /*weak*/ false, $res$$Register);
9171 %}
9172 ins_pipe(pipe_slow);
9173 %}
9174
9175 // This pattern is generated automatically from cas.m4.
9176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9177 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9178 predicate(n->as_LoadStore()->barrier_data() == 0);
9179 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9180 ins_cost(2 * VOLATILE_REF_COST);
9181 effect(TEMP_DEF res, KILL cr);
9182 format %{
9183 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9184 %}
9185 ins_encode %{
9186 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9187 Assembler::xword, /*acquire*/ false, /*release*/ true,
9188 /*weak*/ false, $res$$Register);
9189 %}
9190 ins_pipe(pipe_slow);
9191 %}
9192
9193 // This pattern is generated automatically from cas.m4.
9194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9195 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9196 predicate(needs_acquiring_load_exclusive(n));
9197 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9198 ins_cost(VOLATILE_REF_COST);
9199 effect(TEMP_DEF res, KILL cr);
9200 format %{
9201 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9202 %}
9203 ins_encode %{
9204 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9205 Assembler::byte, /*acquire*/ true, /*release*/ true,
9206 /*weak*/ false, $res$$Register);
9207 __ sxtbw($res$$Register, $res$$Register);
9208 %}
9209 ins_pipe(pipe_slow);
9210 %}
9211
9212 // This pattern is generated automatically from cas.m4.
9213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9214 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9215 predicate(needs_acquiring_load_exclusive(n));
9216 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9217 ins_cost(VOLATILE_REF_COST);
9218 effect(TEMP_DEF res, KILL cr);
9219 format %{
9220 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9221 %}
9222 ins_encode %{
9223 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9224 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9225 /*weak*/ false, $res$$Register);
9226 __ sxthw($res$$Register, $res$$Register);
9227 %}
9228 ins_pipe(pipe_slow);
9229 %}
9230
9231 // This pattern is generated automatically from cas.m4.
9232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9233 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9234 predicate(needs_acquiring_load_exclusive(n));
9235 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9236 ins_cost(VOLATILE_REF_COST);
9237 effect(TEMP_DEF res, KILL cr);
9238 format %{
9239 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9240 %}
9241 ins_encode %{
9242 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9243 Assembler::word, /*acquire*/ true, /*release*/ true,
9244 /*weak*/ false, $res$$Register);
9245 %}
9246 ins_pipe(pipe_slow);
9247 %}
9248
9249 // This pattern is generated automatically from cas.m4.
9250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9251 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9252 predicate(needs_acquiring_load_exclusive(n));
9253 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9254 ins_cost(VOLATILE_REF_COST);
9255 effect(TEMP_DEF res, KILL cr);
9256 format %{
9257 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9258 %}
9259 ins_encode %{
9260 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9261 Assembler::xword, /*acquire*/ true, /*release*/ true,
9262 /*weak*/ false, $res$$Register);
9263 %}
9264 ins_pipe(pipe_slow);
9265 %}
9266
9267 // This pattern is generated automatically from cas.m4.
9268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9269 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9270 predicate(needs_acquiring_load_exclusive(n));
9271 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9272 ins_cost(VOLATILE_REF_COST);
9273 effect(TEMP_DEF res, KILL cr);
9274 format %{
9275 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9276 %}
9277 ins_encode %{
9278 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9279 Assembler::word, /*acquire*/ true, /*release*/ true,
9280 /*weak*/ false, $res$$Register);
9281 %}
9282 ins_pipe(pipe_slow);
9283 %}
9284
9285 // This pattern is generated automatically from cas.m4.
9286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9287 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9288 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9289 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9290 ins_cost(VOLATILE_REF_COST);
9291 effect(TEMP_DEF res, KILL cr);
9292 format %{
9293 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9294 %}
9295 ins_encode %{
9296 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9297 Assembler::xword, /*acquire*/ true, /*release*/ true,
9298 /*weak*/ false, $res$$Register);
9299 %}
9300 ins_pipe(pipe_slow);
9301 %}
9302
9303 // This pattern is generated automatically from cas.m4.
9304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9305 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9306 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9307 ins_cost(2 * VOLATILE_REF_COST);
9308 effect(KILL cr);
9309 format %{
9310 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9311 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9312 %}
9313 ins_encode %{
9314 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9315 Assembler::byte, /*acquire*/ false, /*release*/ true,
9316 /*weak*/ true, noreg);
9317 __ csetw($res$$Register, Assembler::EQ);
9318 %}
9319 ins_pipe(pipe_slow);
9320 %}
9321
9322 // This pattern is generated automatically from cas.m4.
9323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9324 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9325 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9326 ins_cost(2 * VOLATILE_REF_COST);
9327 effect(KILL cr);
9328 format %{
9329 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9330 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9331 %}
9332 ins_encode %{
9333 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9334 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9335 /*weak*/ true, noreg);
9336 __ csetw($res$$Register, Assembler::EQ);
9337 %}
9338 ins_pipe(pipe_slow);
9339 %}
9340
9341 // This pattern is generated automatically from cas.m4.
9342 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9343 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9344 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9345 ins_cost(2 * VOLATILE_REF_COST);
9346 effect(KILL cr);
9347 format %{
9348 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9349 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9350 %}
9351 ins_encode %{
9352 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9353 Assembler::word, /*acquire*/ false, /*release*/ true,
9354 /*weak*/ true, noreg);
9355 __ csetw($res$$Register, Assembler::EQ);
9356 %}
9357 ins_pipe(pipe_slow);
9358 %}
9359
9360 // This pattern is generated automatically from cas.m4.
9361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9362 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9363 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9364 ins_cost(2 * VOLATILE_REF_COST);
9365 effect(KILL cr);
9366 format %{
9367 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9368 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9369 %}
9370 ins_encode %{
9371 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9372 Assembler::xword, /*acquire*/ false, /*release*/ true,
9373 /*weak*/ true, noreg);
9374 __ csetw($res$$Register, Assembler::EQ);
9375 %}
9376 ins_pipe(pipe_slow);
9377 %}
9378
9379 // This pattern is generated automatically from cas.m4.
9380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9381 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9382 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9383 ins_cost(2 * VOLATILE_REF_COST);
9384 effect(KILL cr);
9385 format %{
9386 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9387 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9388 %}
9389 ins_encode %{
9390 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9391 Assembler::word, /*acquire*/ false, /*release*/ true,
9392 /*weak*/ true, noreg);
9393 __ csetw($res$$Register, Assembler::EQ);
9394 %}
9395 ins_pipe(pipe_slow);
9396 %}
9397
9398 // This pattern is generated automatically from cas.m4.
9399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9400 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9401 predicate(n->as_LoadStore()->barrier_data() == 0);
9402 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9403 ins_cost(2 * VOLATILE_REF_COST);
9404 effect(KILL cr);
9405 format %{
9406 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9407 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9408 %}
9409 ins_encode %{
9410 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9411 Assembler::xword, /*acquire*/ false, /*release*/ true,
9412 /*weak*/ true, noreg);
9413 __ csetw($res$$Register, Assembler::EQ);
9414 %}
9415 ins_pipe(pipe_slow);
9416 %}
9417
9418 // This pattern is generated automatically from cas.m4.
9419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9420 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9421 predicate(needs_acquiring_load_exclusive(n));
9422 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9423 ins_cost(VOLATILE_REF_COST);
9424 effect(KILL cr);
9425 format %{
9426 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9427 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9428 %}
9429 ins_encode %{
9430 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9431 Assembler::byte, /*acquire*/ true, /*release*/ true,
9432 /*weak*/ true, noreg);
9433 __ csetw($res$$Register, Assembler::EQ);
9434 %}
9435 ins_pipe(pipe_slow);
9436 %}
9437
9438 // This pattern is generated automatically from cas.m4.
9439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9440 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9441 predicate(needs_acquiring_load_exclusive(n));
9442 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9443 ins_cost(VOLATILE_REF_COST);
9444 effect(KILL cr);
9445 format %{
9446 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9447 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9448 %}
9449 ins_encode %{
9450 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9451 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9452 /*weak*/ true, noreg);
9453 __ csetw($res$$Register, Assembler::EQ);
9454 %}
9455 ins_pipe(pipe_slow);
9456 %}
9457
9458 // This pattern is generated automatically from cas.m4.
9459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9460 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9461 predicate(needs_acquiring_load_exclusive(n));
9462 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9463 ins_cost(VOLATILE_REF_COST);
9464 effect(KILL cr);
9465 format %{
9466 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9467 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9468 %}
9469 ins_encode %{
9470 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9471 Assembler::word, /*acquire*/ true, /*release*/ true,
9472 /*weak*/ true, noreg);
9473 __ csetw($res$$Register, Assembler::EQ);
9474 %}
9475 ins_pipe(pipe_slow);
9476 %}
9477
9478 // This pattern is generated automatically from cas.m4.
9479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9480 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9481 predicate(needs_acquiring_load_exclusive(n));
9482 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9483 ins_cost(VOLATILE_REF_COST);
9484 effect(KILL cr);
9485 format %{
9486 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9487 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9488 %}
9489 ins_encode %{
9490 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9491 Assembler::xword, /*acquire*/ true, /*release*/ true,
9492 /*weak*/ true, noreg);
9493 __ csetw($res$$Register, Assembler::EQ);
9494 %}
9495 ins_pipe(pipe_slow);
9496 %}
9497
9498 // This pattern is generated automatically from cas.m4.
9499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9500 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9501 predicate(needs_acquiring_load_exclusive(n));
9502 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9503 ins_cost(VOLATILE_REF_COST);
9504 effect(KILL cr);
9505 format %{
9506 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9507 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9508 %}
9509 ins_encode %{
9510 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9511 Assembler::word, /*acquire*/ true, /*release*/ true,
9512 /*weak*/ true, noreg);
9513 __ csetw($res$$Register, Assembler::EQ);
9514 %}
9515 ins_pipe(pipe_slow);
9516 %}
9517
9518 // This pattern is generated automatically from cas.m4.
9519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9520 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9521 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9522 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9523 ins_cost(VOLATILE_REF_COST);
9524 effect(KILL cr);
9525 format %{
9526 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9527 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9528 %}
9529 ins_encode %{
9530 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9531 Assembler::xword, /*acquire*/ true, /*release*/ true,
9532 /*weak*/ true, noreg);
9533 __ csetw($res$$Register, Assembler::EQ);
9534 %}
9535 ins_pipe(pipe_slow);
9536 %}
9537
9538 // END This section of the file is automatically generated. Do not edit --------------
9539 // ---------------------------------------------------------------------
9540
9541 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9542 match(Set prev (GetAndSetI mem newv));
9543 ins_cost(2 * VOLATILE_REF_COST);
9544 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9545 ins_encode %{
9546 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9547 %}
9548 ins_pipe(pipe_serial);
9549 %}
9550
9551 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9552 match(Set prev (GetAndSetL mem newv));
9553 ins_cost(2 * VOLATILE_REF_COST);
9554 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9555 ins_encode %{
9556 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9557 %}
9558 ins_pipe(pipe_serial);
9559 %}
9560
9561 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9562 match(Set prev (GetAndSetN mem newv));
9563 ins_cost(2 * VOLATILE_REF_COST);
9564 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9565 ins_encode %{
9566 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9567 %}
9568 ins_pipe(pipe_serial);
9569 %}
9570
9571 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9572 predicate(n->as_LoadStore()->barrier_data() == 0);
9573 match(Set prev (GetAndSetP mem newv));
9574 ins_cost(2 * VOLATILE_REF_COST);
9575 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9576 ins_encode %{
9577 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9578 %}
9579 ins_pipe(pipe_serial);
9580 %}
9581
9582 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9583 predicate(needs_acquiring_load_exclusive(n));
9584 match(Set prev (GetAndSetI mem newv));
9585 ins_cost(VOLATILE_REF_COST);
9586 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9587 ins_encode %{
9588 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9589 %}
9590 ins_pipe(pipe_serial);
9591 %}
9592
9593 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9594 predicate(needs_acquiring_load_exclusive(n));
9595 match(Set prev (GetAndSetL mem newv));
9596 ins_cost(VOLATILE_REF_COST);
9597 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9598 ins_encode %{
9599 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9600 %}
9601 ins_pipe(pipe_serial);
9602 %}
9603
9604 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9605 predicate(needs_acquiring_load_exclusive(n));
9606 match(Set prev (GetAndSetN mem newv));
9607 ins_cost(VOLATILE_REF_COST);
9608 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9609 ins_encode %{
9610 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9611 %}
9612 ins_pipe(pipe_serial);
9613 %}
9614
9615 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9616 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9617 match(Set prev (GetAndSetP mem newv));
9618 ins_cost(VOLATILE_REF_COST);
9619 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9620 ins_encode %{
9621 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9622 %}
9623 ins_pipe(pipe_serial);
9624 %}
9625
9626
9627 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9628 match(Set newval (GetAndAddL mem incr));
9629 ins_cost(2 * VOLATILE_REF_COST + 1);
9630 format %{ "get_and_addL $newval, [$mem], $incr" %}
9631 ins_encode %{
9632 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9633 %}
9634 ins_pipe(pipe_serial);
9635 %}
9636
9637 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9638 predicate(n->as_LoadStore()->result_not_used());
9639 match(Set dummy (GetAndAddL mem incr));
9640 ins_cost(2 * VOLATILE_REF_COST);
9641 format %{ "get_and_addL [$mem], $incr" %}
9642 ins_encode %{
9643 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9644 %}
9645 ins_pipe(pipe_serial);
9646 %}
9647
9648 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9649 match(Set newval (GetAndAddL mem incr));
9650 ins_cost(2 * VOLATILE_REF_COST + 1);
9651 format %{ "get_and_addL $newval, [$mem], $incr" %}
9652 ins_encode %{
9653 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9654 %}
9655 ins_pipe(pipe_serial);
9656 %}
9657
9658 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9659 predicate(n->as_LoadStore()->result_not_used());
9660 match(Set dummy (GetAndAddL mem incr));
9661 ins_cost(2 * VOLATILE_REF_COST);
9662 format %{ "get_and_addL [$mem], $incr" %}
9663 ins_encode %{
9664 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9665 %}
9666 ins_pipe(pipe_serial);
9667 %}
9668
9669 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9670 match(Set newval (GetAndAddI mem incr));
9671 ins_cost(2 * VOLATILE_REF_COST + 1);
9672 format %{ "get_and_addI $newval, [$mem], $incr" %}
9673 ins_encode %{
9674 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9675 %}
9676 ins_pipe(pipe_serial);
9677 %}
9678
9679 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9680 predicate(n->as_LoadStore()->result_not_used());
9681 match(Set dummy (GetAndAddI mem incr));
9682 ins_cost(2 * VOLATILE_REF_COST);
9683 format %{ "get_and_addI [$mem], $incr" %}
9684 ins_encode %{
9685 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9686 %}
9687 ins_pipe(pipe_serial);
9688 %}
9689
9690 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9691 match(Set newval (GetAndAddI mem incr));
9692 ins_cost(2 * VOLATILE_REF_COST + 1);
9693 format %{ "get_and_addI $newval, [$mem], $incr" %}
9694 ins_encode %{
9695 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9696 %}
9697 ins_pipe(pipe_serial);
9698 %}
9699
9700 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9701 predicate(n->as_LoadStore()->result_not_used());
9702 match(Set dummy (GetAndAddI mem incr));
9703 ins_cost(2 * VOLATILE_REF_COST);
9704 format %{ "get_and_addI [$mem], $incr" %}
9705 ins_encode %{
9706 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9707 %}
9708 ins_pipe(pipe_serial);
9709 %}
9710
9711 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9712 predicate(needs_acquiring_load_exclusive(n));
9713 match(Set newval (GetAndAddL mem incr));
9714 ins_cost(VOLATILE_REF_COST + 1);
9715 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9716 ins_encode %{
9717 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9718 %}
9719 ins_pipe(pipe_serial);
9720 %}
9721
9722 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9723 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9724 match(Set dummy (GetAndAddL mem incr));
9725 ins_cost(VOLATILE_REF_COST);
9726 format %{ "get_and_addL_acq [$mem], $incr" %}
9727 ins_encode %{
9728 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9729 %}
9730 ins_pipe(pipe_serial);
9731 %}
9732
9733 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9734 predicate(needs_acquiring_load_exclusive(n));
9735 match(Set newval (GetAndAddL mem incr));
9736 ins_cost(VOLATILE_REF_COST + 1);
9737 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9738 ins_encode %{
9739 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9740 %}
9741 ins_pipe(pipe_serial);
9742 %}
9743
9744 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9745 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9746 match(Set dummy (GetAndAddL mem incr));
9747 ins_cost(VOLATILE_REF_COST);
9748 format %{ "get_and_addL_acq [$mem], $incr" %}
9749 ins_encode %{
9750 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9751 %}
9752 ins_pipe(pipe_serial);
9753 %}
9754
9755 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9756 predicate(needs_acquiring_load_exclusive(n));
9757 match(Set newval (GetAndAddI mem incr));
9758 ins_cost(VOLATILE_REF_COST + 1);
9759 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9760 ins_encode %{
9761 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9762 %}
9763 ins_pipe(pipe_serial);
9764 %}
9765
9766 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9767 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9768 match(Set dummy (GetAndAddI mem incr));
9769 ins_cost(VOLATILE_REF_COST);
9770 format %{ "get_and_addI_acq [$mem], $incr" %}
9771 ins_encode %{
9772 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9773 %}
9774 ins_pipe(pipe_serial);
9775 %}
9776
9777 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9778 predicate(needs_acquiring_load_exclusive(n));
9779 match(Set newval (GetAndAddI mem incr));
9780 ins_cost(VOLATILE_REF_COST + 1);
9781 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9782 ins_encode %{
9783 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9784 %}
9785 ins_pipe(pipe_serial);
9786 %}
9787
9788 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9789 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9790 match(Set dummy (GetAndAddI mem incr));
9791 ins_cost(VOLATILE_REF_COST);
9792 format %{ "get_and_addI_acq [$mem], $incr" %}
9793 ins_encode %{
9794 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9795 %}
9796 ins_pipe(pipe_serial);
9797 %}
9798
9799 // Manifest a CmpU result in an integer register.
9800 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9801 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9802 %{
9803 match(Set dst (CmpU3 src1 src2));
9804 effect(KILL flags);
9805
9806 ins_cost(INSN_COST * 3);
9807 format %{
9808 "cmpw $src1, $src2\n\t"
9809 "csetw $dst, ne\n\t"
9810 "cnegw $dst, lo\t# CmpU3(reg)"
9811 %}
9812 ins_encode %{
9813 __ cmpw($src1$$Register, $src2$$Register);
9814 __ csetw($dst$$Register, Assembler::NE);
9815 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9816 %}
9817
9818 ins_pipe(pipe_class_default);
9819 %}
9820
9821 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9822 %{
9823 match(Set dst (CmpU3 src1 src2));
9824 effect(KILL flags);
9825
9826 ins_cost(INSN_COST * 3);
9827 format %{
9828 "subsw zr, $src1, $src2\n\t"
9829 "csetw $dst, ne\n\t"
9830 "cnegw $dst, lo\t# CmpU3(imm)"
9831 %}
9832 ins_encode %{
9833 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9834 __ csetw($dst$$Register, Assembler::NE);
9835 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9836 %}
9837
9838 ins_pipe(pipe_class_default);
9839 %}
9840
9841 // Manifest a CmpUL result in an integer register.
9842 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9843 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9844 %{
9845 match(Set dst (CmpUL3 src1 src2));
9846 effect(KILL flags);
9847
9848 ins_cost(INSN_COST * 3);
9849 format %{
9850 "cmp $src1, $src2\n\t"
9851 "csetw $dst, ne\n\t"
9852 "cnegw $dst, lo\t# CmpUL3(reg)"
9853 %}
9854 ins_encode %{
9855 __ cmp($src1$$Register, $src2$$Register);
9856 __ csetw($dst$$Register, Assembler::NE);
9857 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9858 %}
9859
9860 ins_pipe(pipe_class_default);
9861 %}
9862
9863 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9864 %{
9865 match(Set dst (CmpUL3 src1 src2));
9866 effect(KILL flags);
9867
9868 ins_cost(INSN_COST * 3);
9869 format %{
9870 "subs zr, $src1, $src2\n\t"
9871 "csetw $dst, ne\n\t"
9872 "cnegw $dst, lo\t# CmpUL3(imm)"
9873 %}
9874 ins_encode %{
9875 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9876 __ csetw($dst$$Register, Assembler::NE);
9877 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9878 %}
9879
9880 ins_pipe(pipe_class_default);
9881 %}
9882
9883 // Manifest a CmpL result in an integer register.
9884 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9885 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9886 %{
9887 match(Set dst (CmpL3 src1 src2));
9888 effect(KILL flags);
9889
9890 ins_cost(INSN_COST * 3);
9891 format %{
9892 "cmp $src1, $src2\n\t"
9893 "csetw $dst, ne\n\t"
9894 "cnegw $dst, lt\t# CmpL3(reg)"
9895 %}
9896 ins_encode %{
9897 __ cmp($src1$$Register, $src2$$Register);
9898 __ csetw($dst$$Register, Assembler::NE);
9899 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9900 %}
9901
9902 ins_pipe(pipe_class_default);
9903 %}
9904
9905 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9906 %{
9907 match(Set dst (CmpL3 src1 src2));
9908 effect(KILL flags);
9909
9910 ins_cost(INSN_COST * 3);
9911 format %{
9912 "subs zr, $src1, $src2\n\t"
9913 "csetw $dst, ne\n\t"
9914 "cnegw $dst, lt\t# CmpL3(imm)"
9915 %}
9916 ins_encode %{
9917 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9918 __ csetw($dst$$Register, Assembler::NE);
9919 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9920 %}
9921
9922 ins_pipe(pipe_class_default);
9923 %}
9924
9925 // ============================================================================
9926 // Conditional Move Instructions
9927
9928 // n.b. we have identical rules for both a signed compare op (cmpOp)
9929 // and an unsigned compare op (cmpOpU). it would be nice if we could
9930 // define an op class which merged both inputs and use it to type the
9931 // argument to a single rule. unfortunatelyt his fails because the
9932 // opclass does not live up to the COND_INTER interface of its
9933 // component operands. When the generic code tries to negate the
9934 // operand it ends up running the generci Machoper::negate method
9935 // which throws a ShouldNotHappen. So, we have to provide two flavours
9936 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9937
9938 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9939 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9940
9941 ins_cost(INSN_COST * 2);
9942 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9943
9944 ins_encode %{
9945 __ cselw(as_Register($dst$$reg),
9946 as_Register($src2$$reg),
9947 as_Register($src1$$reg),
9948 (Assembler::Condition)$cmp$$cmpcode);
9949 %}
9950
9951 ins_pipe(icond_reg_reg);
9952 %}
9953
9954 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9955 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9956
9957 ins_cost(INSN_COST * 2);
9958 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9959
9960 ins_encode %{
9961 __ cselw(as_Register($dst$$reg),
9962 as_Register($src2$$reg),
9963 as_Register($src1$$reg),
9964 (Assembler::Condition)$cmp$$cmpcode);
9965 %}
9966
9967 ins_pipe(icond_reg_reg);
9968 %}
9969
9970 // special cases where one arg is zero
9971
9972 // n.b. this is selected in preference to the rule above because it
9973 // avoids loading constant 0 into a source register
9974
9975 // TODO
9976 // we ought only to be able to cull one of these variants as the ideal
9977 // transforms ought always to order the zero consistently (to left/right?)
9978
9979 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9980 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9981
9982 ins_cost(INSN_COST * 2);
9983 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9984
9985 ins_encode %{
9986 __ cselw(as_Register($dst$$reg),
9987 as_Register($src$$reg),
9988 zr,
9989 (Assembler::Condition)$cmp$$cmpcode);
9990 %}
9991
9992 ins_pipe(icond_reg);
9993 %}
9994
9995 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9996 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9997
9998 ins_cost(INSN_COST * 2);
9999 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10000
10001 ins_encode %{
10002 __ cselw(as_Register($dst$$reg),
10003 as_Register($src$$reg),
10004 zr,
10005 (Assembler::Condition)$cmp$$cmpcode);
10006 %}
10007
10008 ins_pipe(icond_reg);
10009 %}
10010
10011 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10012 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10013
10014 ins_cost(INSN_COST * 2);
10015 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10016
10017 ins_encode %{
10018 __ cselw(as_Register($dst$$reg),
10019 zr,
10020 as_Register($src$$reg),
10021 (Assembler::Condition)$cmp$$cmpcode);
10022 %}
10023
10024 ins_pipe(icond_reg);
10025 %}
10026
10027 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10028 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10029
10030 ins_cost(INSN_COST * 2);
10031 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10032
10033 ins_encode %{
10034 __ cselw(as_Register($dst$$reg),
10035 zr,
10036 as_Register($src$$reg),
10037 (Assembler::Condition)$cmp$$cmpcode);
10038 %}
10039
10040 ins_pipe(icond_reg);
10041 %}
10042
10043 // special case for creating a boolean 0 or 1
10044
10045 // n.b. this is selected in preference to the rule above because it
10046 // avoids loading constants 0 and 1 into a source register
10047
10048 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10049 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10050
10051 ins_cost(INSN_COST * 2);
10052 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10053
10054 ins_encode %{
10055 // equivalently
10056 // cset(as_Register($dst$$reg),
10057 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10058 __ csincw(as_Register($dst$$reg),
10059 zr,
10060 zr,
10061 (Assembler::Condition)$cmp$$cmpcode);
10062 %}
10063
10064 ins_pipe(icond_none);
10065 %}
10066
10067 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10068 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10069
10070 ins_cost(INSN_COST * 2);
10071 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10072
10073 ins_encode %{
10074 // equivalently
10075 // cset(as_Register($dst$$reg),
10076 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10077 __ csincw(as_Register($dst$$reg),
10078 zr,
10079 zr,
10080 (Assembler::Condition)$cmp$$cmpcode);
10081 %}
10082
10083 ins_pipe(icond_none);
10084 %}
10085
10086 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10087 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10088
10089 ins_cost(INSN_COST * 2);
10090 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10091
10092 ins_encode %{
10093 __ csel(as_Register($dst$$reg),
10094 as_Register($src2$$reg),
10095 as_Register($src1$$reg),
10096 (Assembler::Condition)$cmp$$cmpcode);
10097 %}
10098
10099 ins_pipe(icond_reg_reg);
10100 %}
10101
10102 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10103 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10104
10105 ins_cost(INSN_COST * 2);
10106 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10107
10108 ins_encode %{
10109 __ csel(as_Register($dst$$reg),
10110 as_Register($src2$$reg),
10111 as_Register($src1$$reg),
10112 (Assembler::Condition)$cmp$$cmpcode);
10113 %}
10114
10115 ins_pipe(icond_reg_reg);
10116 %}
10117
10118 // special cases where one arg is zero
10119
10120 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10121 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10122
10123 ins_cost(INSN_COST * 2);
10124 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10125
10126 ins_encode %{
10127 __ csel(as_Register($dst$$reg),
10128 zr,
10129 as_Register($src$$reg),
10130 (Assembler::Condition)$cmp$$cmpcode);
10131 %}
10132
10133 ins_pipe(icond_reg);
10134 %}
10135
10136 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10137 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10138
10139 ins_cost(INSN_COST * 2);
10140 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10141
10142 ins_encode %{
10143 __ csel(as_Register($dst$$reg),
10144 zr,
10145 as_Register($src$$reg),
10146 (Assembler::Condition)$cmp$$cmpcode);
10147 %}
10148
10149 ins_pipe(icond_reg);
10150 %}
10151
10152 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10153 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10154
10155 ins_cost(INSN_COST * 2);
10156 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10157
10158 ins_encode %{
10159 __ csel(as_Register($dst$$reg),
10160 as_Register($src$$reg),
10161 zr,
10162 (Assembler::Condition)$cmp$$cmpcode);
10163 %}
10164
10165 ins_pipe(icond_reg);
10166 %}
10167
10168 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10169 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10170
10171 ins_cost(INSN_COST * 2);
10172 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10173
10174 ins_encode %{
10175 __ csel(as_Register($dst$$reg),
10176 as_Register($src$$reg),
10177 zr,
10178 (Assembler::Condition)$cmp$$cmpcode);
10179 %}
10180
10181 ins_pipe(icond_reg);
10182 %}
10183
10184 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10185 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10186
10187 ins_cost(INSN_COST * 2);
10188 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10189
10190 ins_encode %{
10191 __ csel(as_Register($dst$$reg),
10192 as_Register($src2$$reg),
10193 as_Register($src1$$reg),
10194 (Assembler::Condition)$cmp$$cmpcode);
10195 %}
10196
10197 ins_pipe(icond_reg_reg);
10198 %}
10199
10200 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10201 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10202
10203 ins_cost(INSN_COST * 2);
10204 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10205
10206 ins_encode %{
10207 __ csel(as_Register($dst$$reg),
10208 as_Register($src2$$reg),
10209 as_Register($src1$$reg),
10210 (Assembler::Condition)$cmp$$cmpcode);
10211 %}
10212
10213 ins_pipe(icond_reg_reg);
10214 %}
10215
10216 // special cases where one arg is zero
10217
10218 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10219 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10220
10221 ins_cost(INSN_COST * 2);
10222 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10223
10224 ins_encode %{
10225 __ csel(as_Register($dst$$reg),
10226 zr,
10227 as_Register($src$$reg),
10228 (Assembler::Condition)$cmp$$cmpcode);
10229 %}
10230
10231 ins_pipe(icond_reg);
10232 %}
10233
10234 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10235 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10236
10237 ins_cost(INSN_COST * 2);
10238 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10239
10240 ins_encode %{
10241 __ csel(as_Register($dst$$reg),
10242 zr,
10243 as_Register($src$$reg),
10244 (Assembler::Condition)$cmp$$cmpcode);
10245 %}
10246
10247 ins_pipe(icond_reg);
10248 %}
10249
10250 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10251 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10252
10253 ins_cost(INSN_COST * 2);
10254 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10255
10256 ins_encode %{
10257 __ csel(as_Register($dst$$reg),
10258 as_Register($src$$reg),
10259 zr,
10260 (Assembler::Condition)$cmp$$cmpcode);
10261 %}
10262
10263 ins_pipe(icond_reg);
10264 %}
10265
10266 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10267 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10268
10269 ins_cost(INSN_COST * 2);
10270 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10271
10272 ins_encode %{
10273 __ csel(as_Register($dst$$reg),
10274 as_Register($src$$reg),
10275 zr,
10276 (Assembler::Condition)$cmp$$cmpcode);
10277 %}
10278
10279 ins_pipe(icond_reg);
10280 %}
10281
10282 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10283 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10284
10285 ins_cost(INSN_COST * 2);
10286 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10287
10288 ins_encode %{
10289 __ cselw(as_Register($dst$$reg),
10290 as_Register($src2$$reg),
10291 as_Register($src1$$reg),
10292 (Assembler::Condition)$cmp$$cmpcode);
10293 %}
10294
10295 ins_pipe(icond_reg_reg);
10296 %}
10297
10298 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10299 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10300
10301 ins_cost(INSN_COST * 2);
10302 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10303
10304 ins_encode %{
10305 __ cselw(as_Register($dst$$reg),
10306 as_Register($src2$$reg),
10307 as_Register($src1$$reg),
10308 (Assembler::Condition)$cmp$$cmpcode);
10309 %}
10310
10311 ins_pipe(icond_reg_reg);
10312 %}
10313
10314 // special cases where one arg is zero
10315
10316 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10317 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10318
10319 ins_cost(INSN_COST * 2);
10320 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10321
10322 ins_encode %{
10323 __ cselw(as_Register($dst$$reg),
10324 zr,
10325 as_Register($src$$reg),
10326 (Assembler::Condition)$cmp$$cmpcode);
10327 %}
10328
10329 ins_pipe(icond_reg);
10330 %}
10331
10332 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10333 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10334
10335 ins_cost(INSN_COST * 2);
10336 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10337
10338 ins_encode %{
10339 __ cselw(as_Register($dst$$reg),
10340 zr,
10341 as_Register($src$$reg),
10342 (Assembler::Condition)$cmp$$cmpcode);
10343 %}
10344
10345 ins_pipe(icond_reg);
10346 %}
10347
10348 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10349 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10350
10351 ins_cost(INSN_COST * 2);
10352 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10353
10354 ins_encode %{
10355 __ cselw(as_Register($dst$$reg),
10356 as_Register($src$$reg),
10357 zr,
10358 (Assembler::Condition)$cmp$$cmpcode);
10359 %}
10360
10361 ins_pipe(icond_reg);
10362 %}
10363
10364 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10365 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10366
10367 ins_cost(INSN_COST * 2);
10368 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10369
10370 ins_encode %{
10371 __ cselw(as_Register($dst$$reg),
10372 as_Register($src$$reg),
10373 zr,
10374 (Assembler::Condition)$cmp$$cmpcode);
10375 %}
10376
10377 ins_pipe(icond_reg);
10378 %}
10379
10380 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10381 %{
10382 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10383
10384 ins_cost(INSN_COST * 3);
10385
10386 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10387 ins_encode %{
10388 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10389 __ fcsels(as_FloatRegister($dst$$reg),
10390 as_FloatRegister($src2$$reg),
10391 as_FloatRegister($src1$$reg),
10392 cond);
10393 %}
10394
10395 ins_pipe(fp_cond_reg_reg_s);
10396 %}
10397
10398 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10399 %{
10400 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10401
10402 ins_cost(INSN_COST * 3);
10403
10404 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10405 ins_encode %{
10406 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10407 __ fcsels(as_FloatRegister($dst$$reg),
10408 as_FloatRegister($src2$$reg),
10409 as_FloatRegister($src1$$reg),
10410 cond);
10411 %}
10412
10413 ins_pipe(fp_cond_reg_reg_s);
10414 %}
10415
10416 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10417 %{
10418 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10419
10420 ins_cost(INSN_COST * 3);
10421
10422 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10423 ins_encode %{
10424 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10425 __ fcseld(as_FloatRegister($dst$$reg),
10426 as_FloatRegister($src2$$reg),
10427 as_FloatRegister($src1$$reg),
10428 cond);
10429 %}
10430
10431 ins_pipe(fp_cond_reg_reg_d);
10432 %}
10433
10434 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10435 %{
10436 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10437
10438 ins_cost(INSN_COST * 3);
10439
10440 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10441 ins_encode %{
10442 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10443 __ fcseld(as_FloatRegister($dst$$reg),
10444 as_FloatRegister($src2$$reg),
10445 as_FloatRegister($src1$$reg),
10446 cond);
10447 %}
10448
10449 ins_pipe(fp_cond_reg_reg_d);
10450 %}
10451
10452 // ============================================================================
10453 // Arithmetic Instructions
10454 //
10455
10456 // Integer Addition
10457
10458 // TODO
10459 // these currently employ operations which do not set CR and hence are
10460 // not flagged as killing CR but we would like to isolate the cases
10461 // where we want to set flags from those where we don't. need to work
10462 // out how to do that.
10463
10464 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10465 match(Set dst (AddI src1 src2));
10466
10467 ins_cost(INSN_COST);
10468 format %{ "addw $dst, $src1, $src2" %}
10469
10470 ins_encode %{
10471 __ addw(as_Register($dst$$reg),
10472 as_Register($src1$$reg),
10473 as_Register($src2$$reg));
10474 %}
10475
10476 ins_pipe(ialu_reg_reg);
10477 %}
10478
10479 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10480 match(Set dst (AddI src1 src2));
10481
10482 ins_cost(INSN_COST);
10483 format %{ "addw $dst, $src1, $src2" %}
10484
10485 // use opcode to indicate that this is an add not a sub
10486 opcode(0x0);
10487
10488 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10489
10490 ins_pipe(ialu_reg_imm);
10491 %}
10492
10493 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10494 match(Set dst (AddI (ConvL2I src1) src2));
10495
10496 ins_cost(INSN_COST);
10497 format %{ "addw $dst, $src1, $src2" %}
10498
10499 // use opcode to indicate that this is an add not a sub
10500 opcode(0x0);
10501
10502 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10503
10504 ins_pipe(ialu_reg_imm);
10505 %}
10506
10507 // Pointer Addition
10508 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10509 match(Set dst (AddP src1 src2));
10510
10511 ins_cost(INSN_COST);
10512 format %{ "add $dst, $src1, $src2\t# ptr" %}
10513
10514 ins_encode %{
10515 __ add(as_Register($dst$$reg),
10516 as_Register($src1$$reg),
10517 as_Register($src2$$reg));
10518 %}
10519
10520 ins_pipe(ialu_reg_reg);
10521 %}
10522
10523 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10524 match(Set dst (AddP src1 (ConvI2L src2)));
10525
10526 ins_cost(1.9 * INSN_COST);
10527 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10528
10529 ins_encode %{
10530 __ add(as_Register($dst$$reg),
10531 as_Register($src1$$reg),
10532 as_Register($src2$$reg), ext::sxtw);
10533 %}
10534
10535 ins_pipe(ialu_reg_reg);
10536 %}
10537
10538 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10539 match(Set dst (AddP src1 (LShiftL src2 scale)));
10540
10541 ins_cost(1.9 * INSN_COST);
10542 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10543
10544 ins_encode %{
10545 __ lea(as_Register($dst$$reg),
10546 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10547 Address::lsl($scale$$constant)));
10548 %}
10549
10550 ins_pipe(ialu_reg_reg_shift);
10551 %}
10552
10553 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10554 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10555
10556 ins_cost(1.9 * INSN_COST);
10557 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10558
10559 ins_encode %{
10560 __ lea(as_Register($dst$$reg),
10561 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10562 Address::sxtw($scale$$constant)));
10563 %}
10564
10565 ins_pipe(ialu_reg_reg_shift);
10566 %}
10567
10568 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10569 match(Set dst (LShiftL (ConvI2L src) scale));
10570
10571 ins_cost(INSN_COST);
10572 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10573
10574 ins_encode %{
10575 __ sbfiz(as_Register($dst$$reg),
10576 as_Register($src$$reg),
10577 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10578 %}
10579
10580 ins_pipe(ialu_reg_shift);
10581 %}
10582
10583 // Pointer Immediate Addition
10584 // n.b. this needs to be more expensive than using an indirect memory
10585 // operand
10586 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10587 match(Set dst (AddP src1 src2));
10588
10589 ins_cost(INSN_COST);
10590 format %{ "add $dst, $src1, $src2\t# ptr" %}
10591
10592 // use opcode to indicate that this is an add not a sub
10593 opcode(0x0);
10594
10595 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10596
10597 ins_pipe(ialu_reg_imm);
10598 %}
10599
10600 // Long Addition
10601 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10602
10603 match(Set dst (AddL src1 src2));
10604
10605 ins_cost(INSN_COST);
10606 format %{ "add $dst, $src1, $src2" %}
10607
10608 ins_encode %{
10609 __ add(as_Register($dst$$reg),
10610 as_Register($src1$$reg),
10611 as_Register($src2$$reg));
10612 %}
10613
10614 ins_pipe(ialu_reg_reg);
10615 %}
10616
10617 // No constant pool entries requiredLong Immediate Addition.
10618 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10619 match(Set dst (AddL src1 src2));
10620
10621 ins_cost(INSN_COST);
10622 format %{ "add $dst, $src1, $src2" %}
10623
10624 // use opcode to indicate that this is an add not a sub
10625 opcode(0x0);
10626
10627 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10628
10629 ins_pipe(ialu_reg_imm);
10630 %}
10631
10632 // Integer Subtraction
10633 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10634 match(Set dst (SubI src1 src2));
10635
10636 ins_cost(INSN_COST);
10637 format %{ "subw $dst, $src1, $src2" %}
10638
10639 ins_encode %{
10640 __ subw(as_Register($dst$$reg),
10641 as_Register($src1$$reg),
10642 as_Register($src2$$reg));
10643 %}
10644
10645 ins_pipe(ialu_reg_reg);
10646 %}
10647
10648 // Immediate Subtraction
10649 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10650 match(Set dst (SubI src1 src2));
10651
10652 ins_cost(INSN_COST);
10653 format %{ "subw $dst, $src1, $src2" %}
10654
10655 // use opcode to indicate that this is a sub not an add
10656 opcode(0x1);
10657
10658 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10659
10660 ins_pipe(ialu_reg_imm);
10661 %}
10662
10663 // Long Subtraction
10664 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10665
10666 match(Set dst (SubL src1 src2));
10667
10668 ins_cost(INSN_COST);
10669 format %{ "sub $dst, $src1, $src2" %}
10670
10671 ins_encode %{
10672 __ sub(as_Register($dst$$reg),
10673 as_Register($src1$$reg),
10674 as_Register($src2$$reg));
10675 %}
10676
10677 ins_pipe(ialu_reg_reg);
10678 %}
10679
10680 // No constant pool entries requiredLong Immediate Subtraction.
10681 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10682 match(Set dst (SubL src1 src2));
10683
10684 ins_cost(INSN_COST);
10685 format %{ "sub$dst, $src1, $src2" %}
10686
10687 // use opcode to indicate that this is a sub not an add
10688 opcode(0x1);
10689
10690 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10691
10692 ins_pipe(ialu_reg_imm);
10693 %}
10694
10695 // Integer Negation (special case for sub)
10696
10697 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10698 match(Set dst (SubI zero src));
10699
10700 ins_cost(INSN_COST);
10701 format %{ "negw $dst, $src\t# int" %}
10702
10703 ins_encode %{
10704 __ negw(as_Register($dst$$reg),
10705 as_Register($src$$reg));
10706 %}
10707
10708 ins_pipe(ialu_reg);
10709 %}
10710
10711 // Long Negation
10712
10713 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10714 match(Set dst (SubL zero src));
10715
10716 ins_cost(INSN_COST);
10717 format %{ "neg $dst, $src\t# long" %}
10718
10719 ins_encode %{
10720 __ neg(as_Register($dst$$reg),
10721 as_Register($src$$reg));
10722 %}
10723
10724 ins_pipe(ialu_reg);
10725 %}
10726
10727 // Integer Multiply
10728
10729 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10730 match(Set dst (MulI src1 src2));
10731
10732 ins_cost(INSN_COST * 3);
10733 format %{ "mulw $dst, $src1, $src2" %}
10734
10735 ins_encode %{
10736 __ mulw(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 as_Register($src2$$reg));
10739 %}
10740
10741 ins_pipe(imul_reg_reg);
10742 %}
10743
10744 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10745 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10746
10747 ins_cost(INSN_COST * 3);
10748 format %{ "smull $dst, $src1, $src2" %}
10749
10750 ins_encode %{
10751 __ smull(as_Register($dst$$reg),
10752 as_Register($src1$$reg),
10753 as_Register($src2$$reg));
10754 %}
10755
10756 ins_pipe(imul_reg_reg);
10757 %}
10758
10759 // Long Multiply
10760
10761 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10762 match(Set dst (MulL src1 src2));
10763
10764 ins_cost(INSN_COST * 5);
10765 format %{ "mul $dst, $src1, $src2" %}
10766
10767 ins_encode %{
10768 __ mul(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg));
10771 %}
10772
10773 ins_pipe(lmul_reg_reg);
10774 %}
10775
10776 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10777 %{
10778 match(Set dst (MulHiL src1 src2));
10779
10780 ins_cost(INSN_COST * 7);
10781 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10782
10783 ins_encode %{
10784 __ smulh(as_Register($dst$$reg),
10785 as_Register($src1$$reg),
10786 as_Register($src2$$reg));
10787 %}
10788
10789 ins_pipe(lmul_reg_reg);
10790 %}
10791
10792 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10793 %{
10794 match(Set dst (UMulHiL src1 src2));
10795
10796 ins_cost(INSN_COST * 7);
10797 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10798
10799 ins_encode %{
10800 __ umulh(as_Register($dst$$reg),
10801 as_Register($src1$$reg),
10802 as_Register($src2$$reg));
10803 %}
10804
10805 ins_pipe(lmul_reg_reg);
10806 %}
10807
10808 // Combined Integer Multiply & Add/Sub
10809
10810 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10811 match(Set dst (AddI src3 (MulI src1 src2)));
10812
10813 ins_cost(INSN_COST * 3);
10814 format %{ "madd $dst, $src1, $src2, $src3" %}
10815
10816 ins_encode %{
10817 __ maddw(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg),
10820 as_Register($src3$$reg));
10821 %}
10822
10823 ins_pipe(imac_reg_reg);
10824 %}
10825
10826 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10827 match(Set dst (SubI src3 (MulI src1 src2)));
10828
10829 ins_cost(INSN_COST * 3);
10830 format %{ "msub $dst, $src1, $src2, $src3" %}
10831
10832 ins_encode %{
10833 __ msubw(as_Register($dst$$reg),
10834 as_Register($src1$$reg),
10835 as_Register($src2$$reg),
10836 as_Register($src3$$reg));
10837 %}
10838
10839 ins_pipe(imac_reg_reg);
10840 %}
10841
10842 // Combined Integer Multiply & Neg
10843
10844 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10845 match(Set dst (MulI (SubI zero src1) src2));
10846
10847 ins_cost(INSN_COST * 3);
10848 format %{ "mneg $dst, $src1, $src2" %}
10849
10850 ins_encode %{
10851 __ mnegw(as_Register($dst$$reg),
10852 as_Register($src1$$reg),
10853 as_Register($src2$$reg));
10854 %}
10855
10856 ins_pipe(imac_reg_reg);
10857 %}
10858
10859 // Combined Long Multiply & Add/Sub
10860
10861 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10862 match(Set dst (AddL src3 (MulL src1 src2)));
10863
10864 ins_cost(INSN_COST * 5);
10865 format %{ "madd $dst, $src1, $src2, $src3" %}
10866
10867 ins_encode %{
10868 __ madd(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg),
10871 as_Register($src3$$reg));
10872 %}
10873
10874 ins_pipe(lmac_reg_reg);
10875 %}
10876
10877 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10878 match(Set dst (SubL src3 (MulL src1 src2)));
10879
10880 ins_cost(INSN_COST * 5);
10881 format %{ "msub $dst, $src1, $src2, $src3" %}
10882
10883 ins_encode %{
10884 __ msub(as_Register($dst$$reg),
10885 as_Register($src1$$reg),
10886 as_Register($src2$$reg),
10887 as_Register($src3$$reg));
10888 %}
10889
10890 ins_pipe(lmac_reg_reg);
10891 %}
10892
10893 // Combined Long Multiply & Neg
10894
10895 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10896 match(Set dst (MulL (SubL zero src1) src2));
10897
10898 ins_cost(INSN_COST * 5);
10899 format %{ "mneg $dst, $src1, $src2" %}
10900
10901 ins_encode %{
10902 __ mneg(as_Register($dst$$reg),
10903 as_Register($src1$$reg),
10904 as_Register($src2$$reg));
10905 %}
10906
10907 ins_pipe(lmac_reg_reg);
10908 %}
10909
10910 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10911
10912 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10913 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10914
10915 ins_cost(INSN_COST * 3);
10916 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10917
10918 ins_encode %{
10919 __ smaddl(as_Register($dst$$reg),
10920 as_Register($src1$$reg),
10921 as_Register($src2$$reg),
10922 as_Register($src3$$reg));
10923 %}
10924
10925 ins_pipe(imac_reg_reg);
10926 %}
10927
10928 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10929 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10930
10931 ins_cost(INSN_COST * 3);
10932 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10933
10934 ins_encode %{
10935 __ smsubl(as_Register($dst$$reg),
10936 as_Register($src1$$reg),
10937 as_Register($src2$$reg),
10938 as_Register($src3$$reg));
10939 %}
10940
10941 ins_pipe(imac_reg_reg);
10942 %}
10943
10944 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10945 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10946
10947 ins_cost(INSN_COST * 3);
10948 format %{ "smnegl $dst, $src1, $src2" %}
10949
10950 ins_encode %{
10951 __ smnegl(as_Register($dst$$reg),
10952 as_Register($src1$$reg),
10953 as_Register($src2$$reg));
10954 %}
10955
10956 ins_pipe(imac_reg_reg);
10957 %}
10958
10959 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10960
10961 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10962 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10963
10964 ins_cost(INSN_COST * 5);
10965 format %{ "mulw rscratch1, $src1, $src2\n\t"
10966 "maddw $dst, $src3, $src4, rscratch1" %}
10967
10968 ins_encode %{
10969 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10970 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10971
10972 ins_pipe(imac_reg_reg);
10973 %}
10974
10975 // Integer Divide
10976
10977 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10978 match(Set dst (DivI src1 src2));
10979
10980 ins_cost(INSN_COST * 19);
10981 format %{ "sdivw $dst, $src1, $src2" %}
10982
10983 ins_encode(aarch64_enc_divw(dst, src1, src2));
10984 ins_pipe(idiv_reg_reg);
10985 %}
10986
10987 // Long Divide
10988
10989 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10990 match(Set dst (DivL src1 src2));
10991
10992 ins_cost(INSN_COST * 35);
10993 format %{ "sdiv $dst, $src1, $src2" %}
10994
10995 ins_encode(aarch64_enc_div(dst, src1, src2));
10996 ins_pipe(ldiv_reg_reg);
10997 %}
10998
10999 // Integer Remainder
11000
11001 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11002 match(Set dst (ModI src1 src2));
11003
11004 ins_cost(INSN_COST * 22);
11005 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11006 "msubw $dst, rscratch1, $src2, $src1" %}
11007
11008 ins_encode(aarch64_enc_modw(dst, src1, src2));
11009 ins_pipe(idiv_reg_reg);
11010 %}
11011
11012 // Long Remainder
11013
11014 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11015 match(Set dst (ModL src1 src2));
11016
11017 ins_cost(INSN_COST * 38);
11018 format %{ "sdiv rscratch1, $src1, $src2\n"
11019 "msub $dst, rscratch1, $src2, $src1" %}
11020
11021 ins_encode(aarch64_enc_mod(dst, src1, src2));
11022 ins_pipe(ldiv_reg_reg);
11023 %}
11024
11025 // Unsigned Integer Divide
11026
11027 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11028 match(Set dst (UDivI src1 src2));
11029
11030 ins_cost(INSN_COST * 19);
11031 format %{ "udivw $dst, $src1, $src2" %}
11032
11033 ins_encode %{
11034 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
11035 %}
11036
11037 ins_pipe(idiv_reg_reg);
11038 %}
11039
11040 // Unsigned Long Divide
11041
11042 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11043 match(Set dst (UDivL src1 src2));
11044
11045 ins_cost(INSN_COST * 35);
11046 format %{ "udiv $dst, $src1, $src2" %}
11047
11048 ins_encode %{
11049 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
11050 %}
11051
11052 ins_pipe(ldiv_reg_reg);
11053 %}
11054
11055 // Unsigned Integer Remainder
11056
11057 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11058 match(Set dst (UModI src1 src2));
11059
11060 ins_cost(INSN_COST * 22);
11061 format %{ "udivw rscratch1, $src1, $src2\n\t"
11062 "msubw $dst, rscratch1, $src2, $src1" %}
11063
11064 ins_encode %{
11065 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
11066 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11067 %}
11068
11069 ins_pipe(idiv_reg_reg);
11070 %}
11071
11072 // Unsigned Long Remainder
11073
11074 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11075 match(Set dst (UModL src1 src2));
11076
11077 ins_cost(INSN_COST * 38);
11078 format %{ "udiv rscratch1, $src1, $src2\n"
11079 "msub $dst, rscratch1, $src2, $src1" %}
11080
11081 ins_encode %{
11082 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
11083 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
11084 %}
11085
11086 ins_pipe(ldiv_reg_reg);
11087 %}
11088
11089 // Integer Shifts
11090
11091 // Shift Left Register
11092 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11093 match(Set dst (LShiftI src1 src2));
11094
11095 ins_cost(INSN_COST * 2);
11096 format %{ "lslvw $dst, $src1, $src2" %}
11097
11098 ins_encode %{
11099 __ lslvw(as_Register($dst$$reg),
11100 as_Register($src1$$reg),
11101 as_Register($src2$$reg));
11102 %}
11103
11104 ins_pipe(ialu_reg_reg_vshift);
11105 %}
11106
11107 // Shift Left Immediate
11108 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11109 match(Set dst (LShiftI src1 src2));
11110
11111 ins_cost(INSN_COST);
11112 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11113
11114 ins_encode %{
11115 __ lslw(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 $src2$$constant & 0x1f);
11118 %}
11119
11120 ins_pipe(ialu_reg_shift);
11121 %}
11122
11123 // Shift Right Logical Register
11124 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11125 match(Set dst (URShiftI src1 src2));
11126
11127 ins_cost(INSN_COST * 2);
11128 format %{ "lsrvw $dst, $src1, $src2" %}
11129
11130 ins_encode %{
11131 __ lsrvw(as_Register($dst$$reg),
11132 as_Register($src1$$reg),
11133 as_Register($src2$$reg));
11134 %}
11135
11136 ins_pipe(ialu_reg_reg_vshift);
11137 %}
11138
11139 // Shift Right Logical Immediate
11140 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11141 match(Set dst (URShiftI src1 src2));
11142
11143 ins_cost(INSN_COST);
11144 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11145
11146 ins_encode %{
11147 __ lsrw(as_Register($dst$$reg),
11148 as_Register($src1$$reg),
11149 $src2$$constant & 0x1f);
11150 %}
11151
11152 ins_pipe(ialu_reg_shift);
11153 %}
11154
11155 // Shift Right Arithmetic Register
11156 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11157 match(Set dst (RShiftI src1 src2));
11158
11159 ins_cost(INSN_COST * 2);
11160 format %{ "asrvw $dst, $src1, $src2" %}
11161
11162 ins_encode %{
11163 __ asrvw(as_Register($dst$$reg),
11164 as_Register($src1$$reg),
11165 as_Register($src2$$reg));
11166 %}
11167
11168 ins_pipe(ialu_reg_reg_vshift);
11169 %}
11170
11171 // Shift Right Arithmetic Immediate
11172 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11173 match(Set dst (RShiftI src1 src2));
11174
11175 ins_cost(INSN_COST);
11176 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11177
11178 ins_encode %{
11179 __ asrw(as_Register($dst$$reg),
11180 as_Register($src1$$reg),
11181 $src2$$constant & 0x1f);
11182 %}
11183
11184 ins_pipe(ialu_reg_shift);
11185 %}
11186
11187 // Combined Int Mask and Right Shift (using UBFM)
11188 // TODO
11189
11190 // Long Shifts
11191
11192 // Shift Left Register
11193 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11194 match(Set dst (LShiftL src1 src2));
11195
11196 ins_cost(INSN_COST * 2);
11197 format %{ "lslv $dst, $src1, $src2" %}
11198
11199 ins_encode %{
11200 __ lslv(as_Register($dst$$reg),
11201 as_Register($src1$$reg),
11202 as_Register($src2$$reg));
11203 %}
11204
11205 ins_pipe(ialu_reg_reg_vshift);
11206 %}
11207
11208 // Shift Left Immediate
11209 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11210 match(Set dst (LShiftL src1 src2));
11211
11212 ins_cost(INSN_COST);
11213 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11214
11215 ins_encode %{
11216 __ lsl(as_Register($dst$$reg),
11217 as_Register($src1$$reg),
11218 $src2$$constant & 0x3f);
11219 %}
11220
11221 ins_pipe(ialu_reg_shift);
11222 %}
11223
11224 // Shift Right Logical Register
11225 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11226 match(Set dst (URShiftL src1 src2));
11227
11228 ins_cost(INSN_COST * 2);
11229 format %{ "lsrv $dst, $src1, $src2" %}
11230
11231 ins_encode %{
11232 __ lsrv(as_Register($dst$$reg),
11233 as_Register($src1$$reg),
11234 as_Register($src2$$reg));
11235 %}
11236
11237 ins_pipe(ialu_reg_reg_vshift);
11238 %}
11239
11240 // Shift Right Logical Immediate
11241 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11242 match(Set dst (URShiftL src1 src2));
11243
11244 ins_cost(INSN_COST);
11245 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11246
11247 ins_encode %{
11248 __ lsr(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 $src2$$constant & 0x3f);
11251 %}
11252
11253 ins_pipe(ialu_reg_shift);
11254 %}
11255
11256 // A special-case pattern for card table stores.
11257 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11258 match(Set dst (URShiftL (CastP2X src1) src2));
11259
11260 ins_cost(INSN_COST);
11261 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11262
11263 ins_encode %{
11264 __ lsr(as_Register($dst$$reg),
11265 as_Register($src1$$reg),
11266 $src2$$constant & 0x3f);
11267 %}
11268
11269 ins_pipe(ialu_reg_shift);
11270 %}
11271
11272 // Shift Right Arithmetic Register
11273 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11274 match(Set dst (RShiftL src1 src2));
11275
11276 ins_cost(INSN_COST * 2);
11277 format %{ "asrv $dst, $src1, $src2" %}
11278
11279 ins_encode %{
11280 __ asrv(as_Register($dst$$reg),
11281 as_Register($src1$$reg),
11282 as_Register($src2$$reg));
11283 %}
11284
11285 ins_pipe(ialu_reg_reg_vshift);
11286 %}
11287
11288 // Shift Right Arithmetic Immediate
11289 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11290 match(Set dst (RShiftL src1 src2));
11291
11292 ins_cost(INSN_COST);
11293 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11294
11295 ins_encode %{
11296 __ asr(as_Register($dst$$reg),
11297 as_Register($src1$$reg),
11298 $src2$$constant & 0x3f);
11299 %}
11300
11301 ins_pipe(ialu_reg_shift);
11302 %}
11303
11304 // BEGIN This section of the file is automatically generated. Do not edit --------------
11305 // This section is generated from aarch64_ad.m4
11306
11307 // This pattern is automatically generated from aarch64_ad.m4
11308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11309 instruct regL_not_reg(iRegLNoSp dst,
11310 iRegL src1, immL_M1 m1,
11311 rFlagsReg cr) %{
11312 match(Set dst (XorL src1 m1));
11313 ins_cost(INSN_COST);
11314 format %{ "eon $dst, $src1, zr" %}
11315
11316 ins_encode %{
11317 __ eon(as_Register($dst$$reg),
11318 as_Register($src1$$reg),
11319 zr,
11320 Assembler::LSL, 0);
11321 %}
11322
11323 ins_pipe(ialu_reg);
11324 %}
11325
11326 // This pattern is automatically generated from aarch64_ad.m4
11327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11328 instruct regI_not_reg(iRegINoSp dst,
11329 iRegIorL2I src1, immI_M1 m1,
11330 rFlagsReg cr) %{
11331 match(Set dst (XorI src1 m1));
11332 ins_cost(INSN_COST);
11333 format %{ "eonw $dst, $src1, zr" %}
11334
11335 ins_encode %{
11336 __ eonw(as_Register($dst$$reg),
11337 as_Register($src1$$reg),
11338 zr,
11339 Assembler::LSL, 0);
11340 %}
11341
11342 ins_pipe(ialu_reg);
11343 %}
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11348 immI0 zero, iRegIorL2I src1, immI src2) %{
11349 match(Set dst (SubI zero (URShiftI src1 src2)));
11350
11351 ins_cost(1.9 * INSN_COST);
11352 format %{ "negw $dst, $src1, LSR $src2" %}
11353
11354 ins_encode %{
11355 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11356 Assembler::LSR, $src2$$constant & 0x1f);
11357 %}
11358
11359 ins_pipe(ialu_reg_shift);
11360 %}
11361
11362 // This pattern is automatically generated from aarch64_ad.m4
11363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11364 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11365 immI0 zero, iRegIorL2I src1, immI src2) %{
11366 match(Set dst (SubI zero (RShiftI src1 src2)));
11367
11368 ins_cost(1.9 * INSN_COST);
11369 format %{ "negw $dst, $src1, ASR $src2" %}
11370
11371 ins_encode %{
11372 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11373 Assembler::ASR, $src2$$constant & 0x1f);
11374 %}
11375
11376 ins_pipe(ialu_reg_shift);
11377 %}
11378
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11382 immI0 zero, iRegIorL2I src1, immI src2) %{
11383 match(Set dst (SubI zero (LShiftI src1 src2)));
11384
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "negw $dst, $src1, LSL $src2" %}
11387
11388 ins_encode %{
11389 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11390 Assembler::LSL, $src2$$constant & 0x1f);
11391 %}
11392
11393 ins_pipe(ialu_reg_shift);
11394 %}
11395
11396 // This pattern is automatically generated from aarch64_ad.m4
11397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11398 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11399 immL0 zero, iRegL src1, immI src2) %{
11400 match(Set dst (SubL zero (URShiftL src1 src2)));
11401
11402 ins_cost(1.9 * INSN_COST);
11403 format %{ "neg $dst, $src1, LSR $src2" %}
11404
11405 ins_encode %{
11406 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11407 Assembler::LSR, $src2$$constant & 0x3f);
11408 %}
11409
11410 ins_pipe(ialu_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11416 immL0 zero, iRegL src1, immI src2) %{
11417 match(Set dst (SubL zero (RShiftL src1 src2)));
11418
11419 ins_cost(1.9 * INSN_COST);
11420 format %{ "neg $dst, $src1, ASR $src2" %}
11421
11422 ins_encode %{
11423 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11424 Assembler::ASR, $src2$$constant & 0x3f);
11425 %}
11426
11427 ins_pipe(ialu_reg_shift);
11428 %}
11429
11430 // This pattern is automatically generated from aarch64_ad.m4
11431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11432 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11433 immL0 zero, iRegL src1, immI src2) %{
11434 match(Set dst (SubL zero (LShiftL src1 src2)));
11435
11436 ins_cost(1.9 * INSN_COST);
11437 format %{ "neg $dst, $src1, LSL $src2" %}
11438
11439 ins_encode %{
11440 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11441 Assembler::LSL, $src2$$constant & 0x3f);
11442 %}
11443
11444 ins_pipe(ialu_reg_shift);
11445 %}
11446
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449 instruct AndI_reg_not_reg(iRegINoSp dst,
11450 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11451 match(Set dst (AndI src1 (XorI src2 m1)));
11452 ins_cost(INSN_COST);
11453 format %{ "bicw $dst, $src1, $src2" %}
11454
11455 ins_encode %{
11456 __ bicw(as_Register($dst$$reg),
11457 as_Register($src1$$reg),
11458 as_Register($src2$$reg),
11459 Assembler::LSL, 0);
11460 %}
11461
11462 ins_pipe(ialu_reg_reg);
11463 %}
11464
11465 // This pattern is automatically generated from aarch64_ad.m4
11466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11467 instruct AndL_reg_not_reg(iRegLNoSp dst,
11468 iRegL src1, iRegL src2, immL_M1 m1) %{
11469 match(Set dst (AndL src1 (XorL src2 m1)));
11470 ins_cost(INSN_COST);
11471 format %{ "bic $dst, $src1, $src2" %}
11472
11473 ins_encode %{
11474 __ bic(as_Register($dst$$reg),
11475 as_Register($src1$$reg),
11476 as_Register($src2$$reg),
11477 Assembler::LSL, 0);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 instruct OrI_reg_not_reg(iRegINoSp dst,
11486 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11487 match(Set dst (OrI src1 (XorI src2 m1)));
11488 ins_cost(INSN_COST);
11489 format %{ "ornw $dst, $src1, $src2" %}
11490
11491 ins_encode %{
11492 __ ornw(as_Register($dst$$reg),
11493 as_Register($src1$$reg),
11494 as_Register($src2$$reg),
11495 Assembler::LSL, 0);
11496 %}
11497
11498 ins_pipe(ialu_reg_reg);
11499 %}
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503 instruct OrL_reg_not_reg(iRegLNoSp dst,
11504 iRegL src1, iRegL src2, immL_M1 m1) %{
11505 match(Set dst (OrL src1 (XorL src2 m1)));
11506 ins_cost(INSN_COST);
11507 format %{ "orn $dst, $src1, $src2" %}
11508
11509 ins_encode %{
11510 __ orn(as_Register($dst$$reg),
11511 as_Register($src1$$reg),
11512 as_Register($src2$$reg),
11513 Assembler::LSL, 0);
11514 %}
11515
11516 ins_pipe(ialu_reg_reg);
11517 %}
11518
11519 // This pattern is automatically generated from aarch64_ad.m4
11520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11521 instruct XorI_reg_not_reg(iRegINoSp dst,
11522 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11523 match(Set dst (XorI m1 (XorI src2 src1)));
11524 ins_cost(INSN_COST);
11525 format %{ "eonw $dst, $src1, $src2" %}
11526
11527 ins_encode %{
11528 __ eonw(as_Register($dst$$reg),
11529 as_Register($src1$$reg),
11530 as_Register($src2$$reg),
11531 Assembler::LSL, 0);
11532 %}
11533
11534 ins_pipe(ialu_reg_reg);
11535 %}
11536
11537 // This pattern is automatically generated from aarch64_ad.m4
11538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11539 instruct XorL_reg_not_reg(iRegLNoSp dst,
11540 iRegL src1, iRegL src2, immL_M1 m1) %{
11541 match(Set dst (XorL m1 (XorL src2 src1)));
11542 ins_cost(INSN_COST);
11543 format %{ "eon $dst, $src1, $src2" %}
11544
11545 ins_encode %{
11546 __ eon(as_Register($dst$$reg),
11547 as_Register($src1$$reg),
11548 as_Register($src2$$reg),
11549 Assembler::LSL, 0);
11550 %}
11551
11552 ins_pipe(ialu_reg_reg);
11553 %}
11554
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 // val & (-1 ^ (val >>> shift)) ==> bicw
11558 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11559 iRegIorL2I src1, iRegIorL2I src2,
11560 immI src3, immI_M1 src4) %{
11561 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11562 ins_cost(1.9 * INSN_COST);
11563 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11564
11565 ins_encode %{
11566 __ bicw(as_Register($dst$$reg),
11567 as_Register($src1$$reg),
11568 as_Register($src2$$reg),
11569 Assembler::LSR,
11570 $src3$$constant & 0x1f);
11571 %}
11572
11573 ins_pipe(ialu_reg_reg_shift);
11574 %}
11575
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 // val & (-1 ^ (val >>> shift)) ==> bic
11579 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11580 iRegL src1, iRegL src2,
11581 immI src3, immL_M1 src4) %{
11582 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11583 ins_cost(1.9 * INSN_COST);
11584 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11585
11586 ins_encode %{
11587 __ bic(as_Register($dst$$reg),
11588 as_Register($src1$$reg),
11589 as_Register($src2$$reg),
11590 Assembler::LSR,
11591 $src3$$constant & 0x3f);
11592 %}
11593
11594 ins_pipe(ialu_reg_reg_shift);
11595 %}
11596
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 // val & (-1 ^ (val >> shift)) ==> bicw
11600 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11601 iRegIorL2I src1, iRegIorL2I src2,
11602 immI src3, immI_M1 src4) %{
11603 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11604 ins_cost(1.9 * INSN_COST);
11605 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11606
11607 ins_encode %{
11608 __ bicw(as_Register($dst$$reg),
11609 as_Register($src1$$reg),
11610 as_Register($src2$$reg),
11611 Assembler::ASR,
11612 $src3$$constant & 0x1f);
11613 %}
11614
11615 ins_pipe(ialu_reg_reg_shift);
11616 %}
11617
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620 // val & (-1 ^ (val >> shift)) ==> bic
11621 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11622 iRegL src1, iRegL src2,
11623 immI src3, immL_M1 src4) %{
11624 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11625 ins_cost(1.9 * INSN_COST);
11626 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11627
11628 ins_encode %{
11629 __ bic(as_Register($dst$$reg),
11630 as_Register($src1$$reg),
11631 as_Register($src2$$reg),
11632 Assembler::ASR,
11633 $src3$$constant & 0x3f);
11634 %}
11635
11636 ins_pipe(ialu_reg_reg_shift);
11637 %}
11638
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641 // val & (-1 ^ (val ror shift)) ==> bicw
11642 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11643 iRegIorL2I src1, iRegIorL2I src2,
11644 immI src3, immI_M1 src4) %{
11645 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11646 ins_cost(1.9 * INSN_COST);
11647 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11648
11649 ins_encode %{
11650 __ bicw(as_Register($dst$$reg),
11651 as_Register($src1$$reg),
11652 as_Register($src2$$reg),
11653 Assembler::ROR,
11654 $src3$$constant & 0x1f);
11655 %}
11656
11657 ins_pipe(ialu_reg_reg_shift);
11658 %}
11659
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662 // val & (-1 ^ (val ror shift)) ==> bic
11663 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11664 iRegL src1, iRegL src2,
11665 immI src3, immL_M1 src4) %{
11666 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11667 ins_cost(1.9 * INSN_COST);
11668 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11669
11670 ins_encode %{
11671 __ bic(as_Register($dst$$reg),
11672 as_Register($src1$$reg),
11673 as_Register($src2$$reg),
11674 Assembler::ROR,
11675 $src3$$constant & 0x3f);
11676 %}
11677
11678 ins_pipe(ialu_reg_reg_shift);
11679 %}
11680
11681 // This pattern is automatically generated from aarch64_ad.m4
11682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11683 // val & (-1 ^ (val << shift)) ==> bicw
11684 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11685 iRegIorL2I src1, iRegIorL2I src2,
11686 immI src3, immI_M1 src4) %{
11687 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11688 ins_cost(1.9 * INSN_COST);
11689 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11690
11691 ins_encode %{
11692 __ bicw(as_Register($dst$$reg),
11693 as_Register($src1$$reg),
11694 as_Register($src2$$reg),
11695 Assembler::LSL,
11696 $src3$$constant & 0x1f);
11697 %}
11698
11699 ins_pipe(ialu_reg_reg_shift);
11700 %}
11701
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 // val & (-1 ^ (val << shift)) ==> bic
11705 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11706 iRegL src1, iRegL src2,
11707 immI src3, immL_M1 src4) %{
11708 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11709 ins_cost(1.9 * INSN_COST);
11710 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11711
11712 ins_encode %{
11713 __ bic(as_Register($dst$$reg),
11714 as_Register($src1$$reg),
11715 as_Register($src2$$reg),
11716 Assembler::LSL,
11717 $src3$$constant & 0x3f);
11718 %}
11719
11720 ins_pipe(ialu_reg_reg_shift);
11721 %}
11722
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11726 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11727 iRegIorL2I src1, iRegIorL2I src2,
11728 immI src3, immI_M1 src4) %{
11729 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11730 ins_cost(1.9 * INSN_COST);
11731 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11732
11733 ins_encode %{
11734 __ eonw(as_Register($dst$$reg),
11735 as_Register($src1$$reg),
11736 as_Register($src2$$reg),
11737 Assembler::LSR,
11738 $src3$$constant & 0x1f);
11739 %}
11740
11741 ins_pipe(ialu_reg_reg_shift);
11742 %}
11743
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 // val ^ (-1 ^ (val >>> shift)) ==> eon
11747 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11748 iRegL src1, iRegL src2,
11749 immI src3, immL_M1 src4) %{
11750 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11751 ins_cost(1.9 * INSN_COST);
11752 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11753
11754 ins_encode %{
11755 __ eon(as_Register($dst$$reg),
11756 as_Register($src1$$reg),
11757 as_Register($src2$$reg),
11758 Assembler::LSR,
11759 $src3$$constant & 0x3f);
11760 %}
11761
11762 ins_pipe(ialu_reg_reg_shift);
11763 %}
11764
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 // val ^ (-1 ^ (val >> shift)) ==> eonw
11768 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11769 iRegIorL2I src1, iRegIorL2I src2,
11770 immI src3, immI_M1 src4) %{
11771 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11772 ins_cost(1.9 * INSN_COST);
11773 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11774
11775 ins_encode %{
11776 __ eonw(as_Register($dst$$reg),
11777 as_Register($src1$$reg),
11778 as_Register($src2$$reg),
11779 Assembler::ASR,
11780 $src3$$constant & 0x1f);
11781 %}
11782
11783 ins_pipe(ialu_reg_reg_shift);
11784 %}
11785
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 // val ^ (-1 ^ (val >> shift)) ==> eon
11789 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11790 iRegL src1, iRegL src2,
11791 immI src3, immL_M1 src4) %{
11792 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11793 ins_cost(1.9 * INSN_COST);
11794 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11795
11796 ins_encode %{
11797 __ eon(as_Register($dst$$reg),
11798 as_Register($src1$$reg),
11799 as_Register($src2$$reg),
11800 Assembler::ASR,
11801 $src3$$constant & 0x3f);
11802 %}
11803
11804 ins_pipe(ialu_reg_reg_shift);
11805 %}
11806
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 // val ^ (-1 ^ (val ror shift)) ==> eonw
11810 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11811 iRegIorL2I src1, iRegIorL2I src2,
11812 immI src3, immI_M1 src4) %{
11813 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11814 ins_cost(1.9 * INSN_COST);
11815 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11816
11817 ins_encode %{
11818 __ eonw(as_Register($dst$$reg),
11819 as_Register($src1$$reg),
11820 as_Register($src2$$reg),
11821 Assembler::ROR,
11822 $src3$$constant & 0x1f);
11823 %}
11824
11825 ins_pipe(ialu_reg_reg_shift);
11826 %}
11827
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 // val ^ (-1 ^ (val ror shift)) ==> eon
11831 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11832 iRegL src1, iRegL src2,
11833 immI src3, immL_M1 src4) %{
11834 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11835 ins_cost(1.9 * INSN_COST);
11836 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11837
11838 ins_encode %{
11839 __ eon(as_Register($dst$$reg),
11840 as_Register($src1$$reg),
11841 as_Register($src2$$reg),
11842 Assembler::ROR,
11843 $src3$$constant & 0x3f);
11844 %}
11845
11846 ins_pipe(ialu_reg_reg_shift);
11847 %}
11848
11849 // This pattern is automatically generated from aarch64_ad.m4
11850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11851 // val ^ (-1 ^ (val << shift)) ==> eonw
11852 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11853 iRegIorL2I src1, iRegIorL2I src2,
11854 immI src3, immI_M1 src4) %{
11855 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11856 ins_cost(1.9 * INSN_COST);
11857 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11858
11859 ins_encode %{
11860 __ eonw(as_Register($dst$$reg),
11861 as_Register($src1$$reg),
11862 as_Register($src2$$reg),
11863 Assembler::LSL,
11864 $src3$$constant & 0x1f);
11865 %}
11866
11867 ins_pipe(ialu_reg_reg_shift);
11868 %}
11869
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 // val ^ (-1 ^ (val << shift)) ==> eon
11873 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11874 iRegL src1, iRegL src2,
11875 immI src3, immL_M1 src4) %{
11876 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11877 ins_cost(1.9 * INSN_COST);
11878 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11879
11880 ins_encode %{
11881 __ eon(as_Register($dst$$reg),
11882 as_Register($src1$$reg),
11883 as_Register($src2$$reg),
11884 Assembler::LSL,
11885 $src3$$constant & 0x3f);
11886 %}
11887
11888 ins_pipe(ialu_reg_reg_shift);
11889 %}
11890
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 // val | (-1 ^ (val >>> shift)) ==> ornw
11894 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11895 iRegIorL2I src1, iRegIorL2I src2,
11896 immI src3, immI_M1 src4) %{
11897 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11898 ins_cost(1.9 * INSN_COST);
11899 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11900
11901 ins_encode %{
11902 __ ornw(as_Register($dst$$reg),
11903 as_Register($src1$$reg),
11904 as_Register($src2$$reg),
11905 Assembler::LSR,
11906 $src3$$constant & 0x1f);
11907 %}
11908
11909 ins_pipe(ialu_reg_reg_shift);
11910 %}
11911
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 // val | (-1 ^ (val >>> shift)) ==> orn
11915 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11916 iRegL src1, iRegL src2,
11917 immI src3, immL_M1 src4) %{
11918 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11919 ins_cost(1.9 * INSN_COST);
11920 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11921
11922 ins_encode %{
11923 __ orn(as_Register($dst$$reg),
11924 as_Register($src1$$reg),
11925 as_Register($src2$$reg),
11926 Assembler::LSR,
11927 $src3$$constant & 0x3f);
11928 %}
11929
11930 ins_pipe(ialu_reg_reg_shift);
11931 %}
11932
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 // val | (-1 ^ (val >> shift)) ==> ornw
11936 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11937 iRegIorL2I src1, iRegIorL2I src2,
11938 immI src3, immI_M1 src4) %{
11939 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11940 ins_cost(1.9 * INSN_COST);
11941 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11942
11943 ins_encode %{
11944 __ ornw(as_Register($dst$$reg),
11945 as_Register($src1$$reg),
11946 as_Register($src2$$reg),
11947 Assembler::ASR,
11948 $src3$$constant & 0x1f);
11949 %}
11950
11951 ins_pipe(ialu_reg_reg_shift);
11952 %}
11953
11954 // This pattern is automatically generated from aarch64_ad.m4
11955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11956 // val | (-1 ^ (val >> shift)) ==> orn
11957 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11958 iRegL src1, iRegL src2,
11959 immI src3, immL_M1 src4) %{
11960 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11961 ins_cost(1.9 * INSN_COST);
11962 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11963
11964 ins_encode %{
11965 __ orn(as_Register($dst$$reg),
11966 as_Register($src1$$reg),
11967 as_Register($src2$$reg),
11968 Assembler::ASR,
11969 $src3$$constant & 0x3f);
11970 %}
11971
11972 ins_pipe(ialu_reg_reg_shift);
11973 %}
11974
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 // val | (-1 ^ (val ror shift)) ==> ornw
11978 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11979 iRegIorL2I src1, iRegIorL2I src2,
11980 immI src3, immI_M1 src4) %{
11981 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11982 ins_cost(1.9 * INSN_COST);
11983 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11984
11985 ins_encode %{
11986 __ ornw(as_Register($dst$$reg),
11987 as_Register($src1$$reg),
11988 as_Register($src2$$reg),
11989 Assembler::ROR,
11990 $src3$$constant & 0x1f);
11991 %}
11992
11993 ins_pipe(ialu_reg_reg_shift);
11994 %}
11995
11996 // This pattern is automatically generated from aarch64_ad.m4
11997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11998 // val | (-1 ^ (val ror shift)) ==> orn
11999 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12000 iRegL src1, iRegL src2,
12001 immI src3, immL_M1 src4) %{
12002 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12003 ins_cost(1.9 * INSN_COST);
12004 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12005
12006 ins_encode %{
12007 __ orn(as_Register($dst$$reg),
12008 as_Register($src1$$reg),
12009 as_Register($src2$$reg),
12010 Assembler::ROR,
12011 $src3$$constant & 0x3f);
12012 %}
12013
12014 ins_pipe(ialu_reg_reg_shift);
12015 %}
12016
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 // val | (-1 ^ (val << shift)) ==> ornw
12020 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12021 iRegIorL2I src1, iRegIorL2I src2,
12022 immI src3, immI_M1 src4) %{
12023 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12024 ins_cost(1.9 * INSN_COST);
12025 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12026
12027 ins_encode %{
12028 __ ornw(as_Register($dst$$reg),
12029 as_Register($src1$$reg),
12030 as_Register($src2$$reg),
12031 Assembler::LSL,
12032 $src3$$constant & 0x1f);
12033 %}
12034
12035 ins_pipe(ialu_reg_reg_shift);
12036 %}
12037
12038 // This pattern is automatically generated from aarch64_ad.m4
12039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12040 // val | (-1 ^ (val << shift)) ==> orn
12041 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12042 iRegL src1, iRegL src2,
12043 immI src3, immL_M1 src4) %{
12044 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12045 ins_cost(1.9 * INSN_COST);
12046 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12047
12048 ins_encode %{
12049 __ orn(as_Register($dst$$reg),
12050 as_Register($src1$$reg),
12051 as_Register($src2$$reg),
12052 Assembler::LSL,
12053 $src3$$constant & 0x3f);
12054 %}
12055
12056 ins_pipe(ialu_reg_reg_shift);
12057 %}
12058
12059 // This pattern is automatically generated from aarch64_ad.m4
12060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12061 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12062 iRegIorL2I src1, iRegIorL2I src2,
12063 immI src3) %{
12064 match(Set dst (AndI src1 (URShiftI src2 src3)));
12065
12066 ins_cost(1.9 * INSN_COST);
12067 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12068
12069 ins_encode %{
12070 __ andw(as_Register($dst$$reg),
12071 as_Register($src1$$reg),
12072 as_Register($src2$$reg),
12073 Assembler::LSR,
12074 $src3$$constant & 0x1f);
12075 %}
12076
12077 ins_pipe(ialu_reg_reg_shift);
12078 %}
12079
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12083 iRegL src1, iRegL src2,
12084 immI src3) %{
12085 match(Set dst (AndL src1 (URShiftL src2 src3)));
12086
12087 ins_cost(1.9 * INSN_COST);
12088 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12089
12090 ins_encode %{
12091 __ andr(as_Register($dst$$reg),
12092 as_Register($src1$$reg),
12093 as_Register($src2$$reg),
12094 Assembler::LSR,
12095 $src3$$constant & 0x3f);
12096 %}
12097
12098 ins_pipe(ialu_reg_reg_shift);
12099 %}
12100
12101 // This pattern is automatically generated from aarch64_ad.m4
12102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12103 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12104 iRegIorL2I src1, iRegIorL2I src2,
12105 immI src3) %{
12106 match(Set dst (AndI src1 (RShiftI src2 src3)));
12107
12108 ins_cost(1.9 * INSN_COST);
12109 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12110
12111 ins_encode %{
12112 __ andw(as_Register($dst$$reg),
12113 as_Register($src1$$reg),
12114 as_Register($src2$$reg),
12115 Assembler::ASR,
12116 $src3$$constant & 0x1f);
12117 %}
12118
12119 ins_pipe(ialu_reg_reg_shift);
12120 %}
12121
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12125 iRegL src1, iRegL src2,
12126 immI src3) %{
12127 match(Set dst (AndL src1 (RShiftL src2 src3)));
12128
12129 ins_cost(1.9 * INSN_COST);
12130 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12131
12132 ins_encode %{
12133 __ andr(as_Register($dst$$reg),
12134 as_Register($src1$$reg),
12135 as_Register($src2$$reg),
12136 Assembler::ASR,
12137 $src3$$constant & 0x3f);
12138 %}
12139
12140 ins_pipe(ialu_reg_reg_shift);
12141 %}
12142
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12146 iRegIorL2I src1, iRegIorL2I src2,
12147 immI src3) %{
12148 match(Set dst (AndI src1 (LShiftI src2 src3)));
12149
12150 ins_cost(1.9 * INSN_COST);
12151 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12152
12153 ins_encode %{
12154 __ andw(as_Register($dst$$reg),
12155 as_Register($src1$$reg),
12156 as_Register($src2$$reg),
12157 Assembler::LSL,
12158 $src3$$constant & 0x1f);
12159 %}
12160
12161 ins_pipe(ialu_reg_reg_shift);
12162 %}
12163
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12167 iRegL src1, iRegL src2,
12168 immI src3) %{
12169 match(Set dst (AndL src1 (LShiftL src2 src3)));
12170
12171 ins_cost(1.9 * INSN_COST);
12172 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12173
12174 ins_encode %{
12175 __ andr(as_Register($dst$$reg),
12176 as_Register($src1$$reg),
12177 as_Register($src2$$reg),
12178 Assembler::LSL,
12179 $src3$$constant & 0x3f);
12180 %}
12181
12182 ins_pipe(ialu_reg_reg_shift);
12183 %}
12184
12185 // This pattern is automatically generated from aarch64_ad.m4
12186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12187 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12188 iRegIorL2I src1, iRegIorL2I src2,
12189 immI src3) %{
12190 match(Set dst (AndI src1 (RotateRight src2 src3)));
12191
12192 ins_cost(1.9 * INSN_COST);
12193 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12194
12195 ins_encode %{
12196 __ andw(as_Register($dst$$reg),
12197 as_Register($src1$$reg),
12198 as_Register($src2$$reg),
12199 Assembler::ROR,
12200 $src3$$constant & 0x1f);
12201 %}
12202
12203 ins_pipe(ialu_reg_reg_shift);
12204 %}
12205
12206 // This pattern is automatically generated from aarch64_ad.m4
12207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12208 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12209 iRegL src1, iRegL src2,
12210 immI src3) %{
12211 match(Set dst (AndL src1 (RotateRight src2 src3)));
12212
12213 ins_cost(1.9 * INSN_COST);
12214 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12215
12216 ins_encode %{
12217 __ andr(as_Register($dst$$reg),
12218 as_Register($src1$$reg),
12219 as_Register($src2$$reg),
12220 Assembler::ROR,
12221 $src3$$constant & 0x3f);
12222 %}
12223
12224 ins_pipe(ialu_reg_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12230 iRegIorL2I src1, iRegIorL2I src2,
12231 immI src3) %{
12232 match(Set dst (XorI src1 (URShiftI src2 src3)));
12233
12234 ins_cost(1.9 * INSN_COST);
12235 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12236
12237 ins_encode %{
12238 __ eorw(as_Register($dst$$reg),
12239 as_Register($src1$$reg),
12240 as_Register($src2$$reg),
12241 Assembler::LSR,
12242 $src3$$constant & 0x1f);
12243 %}
12244
12245 ins_pipe(ialu_reg_reg_shift);
12246 %}
12247
12248 // This pattern is automatically generated from aarch64_ad.m4
12249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12250 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12251 iRegL src1, iRegL src2,
12252 immI src3) %{
12253 match(Set dst (XorL src1 (URShiftL src2 src3)));
12254
12255 ins_cost(1.9 * INSN_COST);
12256 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12257
12258 ins_encode %{
12259 __ eor(as_Register($dst$$reg),
12260 as_Register($src1$$reg),
12261 as_Register($src2$$reg),
12262 Assembler::LSR,
12263 $src3$$constant & 0x3f);
12264 %}
12265
12266 ins_pipe(ialu_reg_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12272 iRegIorL2I src1, iRegIorL2I src2,
12273 immI src3) %{
12274 match(Set dst (XorI src1 (RShiftI src2 src3)));
12275
12276 ins_cost(1.9 * INSN_COST);
12277 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12278
12279 ins_encode %{
12280 __ eorw(as_Register($dst$$reg),
12281 as_Register($src1$$reg),
12282 as_Register($src2$$reg),
12283 Assembler::ASR,
12284 $src3$$constant & 0x1f);
12285 %}
12286
12287 ins_pipe(ialu_reg_reg_shift);
12288 %}
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12293 iRegL src1, iRegL src2,
12294 immI src3) %{
12295 match(Set dst (XorL src1 (RShiftL src2 src3)));
12296
12297 ins_cost(1.9 * INSN_COST);
12298 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12299
12300 ins_encode %{
12301 __ eor(as_Register($dst$$reg),
12302 as_Register($src1$$reg),
12303 as_Register($src2$$reg),
12304 Assembler::ASR,
12305 $src3$$constant & 0x3f);
12306 %}
12307
12308 ins_pipe(ialu_reg_reg_shift);
12309 %}
12310
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12314 iRegIorL2I src1, iRegIorL2I src2,
12315 immI src3) %{
12316 match(Set dst (XorI src1 (LShiftI src2 src3)));
12317
12318 ins_cost(1.9 * INSN_COST);
12319 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12320
12321 ins_encode %{
12322 __ eorw(as_Register($dst$$reg),
12323 as_Register($src1$$reg),
12324 as_Register($src2$$reg),
12325 Assembler::LSL,
12326 $src3$$constant & 0x1f);
12327 %}
12328
12329 ins_pipe(ialu_reg_reg_shift);
12330 %}
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12335 iRegL src1, iRegL src2,
12336 immI src3) %{
12337 match(Set dst (XorL src1 (LShiftL src2 src3)));
12338
12339 ins_cost(1.9 * INSN_COST);
12340 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12341
12342 ins_encode %{
12343 __ eor(as_Register($dst$$reg),
12344 as_Register($src1$$reg),
12345 as_Register($src2$$reg),
12346 Assembler::LSL,
12347 $src3$$constant & 0x3f);
12348 %}
12349
12350 ins_pipe(ialu_reg_reg_shift);
12351 %}
12352
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12356 iRegIorL2I src1, iRegIorL2I src2,
12357 immI src3) %{
12358 match(Set dst (XorI src1 (RotateRight src2 src3)));
12359
12360 ins_cost(1.9 * INSN_COST);
12361 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12362
12363 ins_encode %{
12364 __ eorw(as_Register($dst$$reg),
12365 as_Register($src1$$reg),
12366 as_Register($src2$$reg),
12367 Assembler::ROR,
12368 $src3$$constant & 0x1f);
12369 %}
12370
12371 ins_pipe(ialu_reg_reg_shift);
12372 %}
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12377 iRegL src1, iRegL src2,
12378 immI src3) %{
12379 match(Set dst (XorL src1 (RotateRight src2 src3)));
12380
12381 ins_cost(1.9 * INSN_COST);
12382 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12383
12384 ins_encode %{
12385 __ eor(as_Register($dst$$reg),
12386 as_Register($src1$$reg),
12387 as_Register($src2$$reg),
12388 Assembler::ROR,
12389 $src3$$constant & 0x3f);
12390 %}
12391
12392 ins_pipe(ialu_reg_reg_shift);
12393 %}
12394
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12398 iRegIorL2I src1, iRegIorL2I src2,
12399 immI src3) %{
12400 match(Set dst (OrI src1 (URShiftI src2 src3)));
12401
12402 ins_cost(1.9 * INSN_COST);
12403 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12404
12405 ins_encode %{
12406 __ orrw(as_Register($dst$$reg),
12407 as_Register($src1$$reg),
12408 as_Register($src2$$reg),
12409 Assembler::LSR,
12410 $src3$$constant & 0x1f);
12411 %}
12412
12413 ins_pipe(ialu_reg_reg_shift);
12414 %}
12415
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12419 iRegL src1, iRegL src2,
12420 immI src3) %{
12421 match(Set dst (OrL src1 (URShiftL src2 src3)));
12422
12423 ins_cost(1.9 * INSN_COST);
12424 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12425
12426 ins_encode %{
12427 __ orr(as_Register($dst$$reg),
12428 as_Register($src1$$reg),
12429 as_Register($src2$$reg),
12430 Assembler::LSR,
12431 $src3$$constant & 0x3f);
12432 %}
12433
12434 ins_pipe(ialu_reg_reg_shift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12440 iRegIorL2I src1, iRegIorL2I src2,
12441 immI src3) %{
12442 match(Set dst (OrI src1 (RShiftI src2 src3)));
12443
12444 ins_cost(1.9 * INSN_COST);
12445 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12446
12447 ins_encode %{
12448 __ orrw(as_Register($dst$$reg),
12449 as_Register($src1$$reg),
12450 as_Register($src2$$reg),
12451 Assembler::ASR,
12452 $src3$$constant & 0x1f);
12453 %}
12454
12455 ins_pipe(ialu_reg_reg_shift);
12456 %}
12457
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12461 iRegL src1, iRegL src2,
12462 immI src3) %{
12463 match(Set dst (OrL src1 (RShiftL src2 src3)));
12464
12465 ins_cost(1.9 * INSN_COST);
12466 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12467
12468 ins_encode %{
12469 __ orr(as_Register($dst$$reg),
12470 as_Register($src1$$reg),
12471 as_Register($src2$$reg),
12472 Assembler::ASR,
12473 $src3$$constant & 0x3f);
12474 %}
12475
12476 ins_pipe(ialu_reg_reg_shift);
12477 %}
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12482 iRegIorL2I src1, iRegIorL2I src2,
12483 immI src3) %{
12484 match(Set dst (OrI src1 (LShiftI src2 src3)));
12485
12486 ins_cost(1.9 * INSN_COST);
12487 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12488
12489 ins_encode %{
12490 __ orrw(as_Register($dst$$reg),
12491 as_Register($src1$$reg),
12492 as_Register($src2$$reg),
12493 Assembler::LSL,
12494 $src3$$constant & 0x1f);
12495 %}
12496
12497 ins_pipe(ialu_reg_reg_shift);
12498 %}
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12503 iRegL src1, iRegL src2,
12504 immI src3) %{
12505 match(Set dst (OrL src1 (LShiftL src2 src3)));
12506
12507 ins_cost(1.9 * INSN_COST);
12508 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12509
12510 ins_encode %{
12511 __ orr(as_Register($dst$$reg),
12512 as_Register($src1$$reg),
12513 as_Register($src2$$reg),
12514 Assembler::LSL,
12515 $src3$$constant & 0x3f);
12516 %}
12517
12518 ins_pipe(ialu_reg_reg_shift);
12519 %}
12520
12521 // This pattern is automatically generated from aarch64_ad.m4
12522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12523 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12524 iRegIorL2I src1, iRegIorL2I src2,
12525 immI src3) %{
12526 match(Set dst (OrI src1 (RotateRight src2 src3)));
12527
12528 ins_cost(1.9 * INSN_COST);
12529 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12530
12531 ins_encode %{
12532 __ orrw(as_Register($dst$$reg),
12533 as_Register($src1$$reg),
12534 as_Register($src2$$reg),
12535 Assembler::ROR,
12536 $src3$$constant & 0x1f);
12537 %}
12538
12539 ins_pipe(ialu_reg_reg_shift);
12540 %}
12541
12542 // This pattern is automatically generated from aarch64_ad.m4
12543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12544 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12545 iRegL src1, iRegL src2,
12546 immI src3) %{
12547 match(Set dst (OrL src1 (RotateRight src2 src3)));
12548
12549 ins_cost(1.9 * INSN_COST);
12550 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12551
12552 ins_encode %{
12553 __ orr(as_Register($dst$$reg),
12554 as_Register($src1$$reg),
12555 as_Register($src2$$reg),
12556 Assembler::ROR,
12557 $src3$$constant & 0x3f);
12558 %}
12559
12560 ins_pipe(ialu_reg_reg_shift);
12561 %}
12562
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12566 iRegIorL2I src1, iRegIorL2I src2,
12567 immI src3) %{
12568 match(Set dst (AddI src1 (URShiftI src2 src3)));
12569
12570 ins_cost(1.9 * INSN_COST);
12571 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12572
12573 ins_encode %{
12574 __ addw(as_Register($dst$$reg),
12575 as_Register($src1$$reg),
12576 as_Register($src2$$reg),
12577 Assembler::LSR,
12578 $src3$$constant & 0x1f);
12579 %}
12580
12581 ins_pipe(ialu_reg_reg_shift);
12582 %}
12583
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12587 iRegL src1, iRegL src2,
12588 immI src3) %{
12589 match(Set dst (AddL src1 (URShiftL src2 src3)));
12590
12591 ins_cost(1.9 * INSN_COST);
12592 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12593
12594 ins_encode %{
12595 __ add(as_Register($dst$$reg),
12596 as_Register($src1$$reg),
12597 as_Register($src2$$reg),
12598 Assembler::LSR,
12599 $src3$$constant & 0x3f);
12600 %}
12601
12602 ins_pipe(ialu_reg_reg_shift);
12603 %}
12604
12605 // This pattern is automatically generated from aarch64_ad.m4
12606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12607 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12608 iRegIorL2I src1, iRegIorL2I src2,
12609 immI src3) %{
12610 match(Set dst (AddI src1 (RShiftI src2 src3)));
12611
12612 ins_cost(1.9 * INSN_COST);
12613 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12614
12615 ins_encode %{
12616 __ addw(as_Register($dst$$reg),
12617 as_Register($src1$$reg),
12618 as_Register($src2$$reg),
12619 Assembler::ASR,
12620 $src3$$constant & 0x1f);
12621 %}
12622
12623 ins_pipe(ialu_reg_reg_shift);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12629 iRegL src1, iRegL src2,
12630 immI src3) %{
12631 match(Set dst (AddL src1 (RShiftL src2 src3)));
12632
12633 ins_cost(1.9 * INSN_COST);
12634 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12635
12636 ins_encode %{
12637 __ add(as_Register($dst$$reg),
12638 as_Register($src1$$reg),
12639 as_Register($src2$$reg),
12640 Assembler::ASR,
12641 $src3$$constant & 0x3f);
12642 %}
12643
12644 ins_pipe(ialu_reg_reg_shift);
12645 %}
12646
12647 // This pattern is automatically generated from aarch64_ad.m4
12648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12649 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12650 iRegIorL2I src1, iRegIorL2I src2,
12651 immI src3) %{
12652 match(Set dst (AddI src1 (LShiftI src2 src3)));
12653
12654 ins_cost(1.9 * INSN_COST);
12655 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12656
12657 ins_encode %{
12658 __ addw(as_Register($dst$$reg),
12659 as_Register($src1$$reg),
12660 as_Register($src2$$reg),
12661 Assembler::LSL,
12662 $src3$$constant & 0x1f);
12663 %}
12664
12665 ins_pipe(ialu_reg_reg_shift);
12666 %}
12667
12668 // This pattern is automatically generated from aarch64_ad.m4
12669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12670 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12671 iRegL src1, iRegL src2,
12672 immI src3) %{
12673 match(Set dst (AddL src1 (LShiftL src2 src3)));
12674
12675 ins_cost(1.9 * INSN_COST);
12676 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12677
12678 ins_encode %{
12679 __ add(as_Register($dst$$reg),
12680 as_Register($src1$$reg),
12681 as_Register($src2$$reg),
12682 Assembler::LSL,
12683 $src3$$constant & 0x3f);
12684 %}
12685
12686 ins_pipe(ialu_reg_reg_shift);
12687 %}
12688
12689 // This pattern is automatically generated from aarch64_ad.m4
12690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12691 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12692 iRegIorL2I src1, iRegIorL2I src2,
12693 immI src3) %{
12694 match(Set dst (SubI src1 (URShiftI src2 src3)));
12695
12696 ins_cost(1.9 * INSN_COST);
12697 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12698
12699 ins_encode %{
12700 __ subw(as_Register($dst$$reg),
12701 as_Register($src1$$reg),
12702 as_Register($src2$$reg),
12703 Assembler::LSR,
12704 $src3$$constant & 0x1f);
12705 %}
12706
12707 ins_pipe(ialu_reg_reg_shift);
12708 %}
12709
12710 // This pattern is automatically generated from aarch64_ad.m4
12711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12712 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12713 iRegL src1, iRegL src2,
12714 immI src3) %{
12715 match(Set dst (SubL src1 (URShiftL src2 src3)));
12716
12717 ins_cost(1.9 * INSN_COST);
12718 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12719
12720 ins_encode %{
12721 __ sub(as_Register($dst$$reg),
12722 as_Register($src1$$reg),
12723 as_Register($src2$$reg),
12724 Assembler::LSR,
12725 $src3$$constant & 0x3f);
12726 %}
12727
12728 ins_pipe(ialu_reg_reg_shift);
12729 %}
12730
12731 // This pattern is automatically generated from aarch64_ad.m4
12732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12733 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12734 iRegIorL2I src1, iRegIorL2I src2,
12735 immI src3) %{
12736 match(Set dst (SubI src1 (RShiftI src2 src3)));
12737
12738 ins_cost(1.9 * INSN_COST);
12739 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12740
12741 ins_encode %{
12742 __ subw(as_Register($dst$$reg),
12743 as_Register($src1$$reg),
12744 as_Register($src2$$reg),
12745 Assembler::ASR,
12746 $src3$$constant & 0x1f);
12747 %}
12748
12749 ins_pipe(ialu_reg_reg_shift);
12750 %}
12751
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12755 iRegL src1, iRegL src2,
12756 immI src3) %{
12757 match(Set dst (SubL src1 (RShiftL src2 src3)));
12758
12759 ins_cost(1.9 * INSN_COST);
12760 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12761
12762 ins_encode %{
12763 __ sub(as_Register($dst$$reg),
12764 as_Register($src1$$reg),
12765 as_Register($src2$$reg),
12766 Assembler::ASR,
12767 $src3$$constant & 0x3f);
12768 %}
12769
12770 ins_pipe(ialu_reg_reg_shift);
12771 %}
12772
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12776 iRegIorL2I src1, iRegIorL2I src2,
12777 immI src3) %{
12778 match(Set dst (SubI src1 (LShiftI src2 src3)));
12779
12780 ins_cost(1.9 * INSN_COST);
12781 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12782
12783 ins_encode %{
12784 __ subw(as_Register($dst$$reg),
12785 as_Register($src1$$reg),
12786 as_Register($src2$$reg),
12787 Assembler::LSL,
12788 $src3$$constant & 0x1f);
12789 %}
12790
12791 ins_pipe(ialu_reg_reg_shift);
12792 %}
12793
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12797 iRegL src1, iRegL src2,
12798 immI src3) %{
12799 match(Set dst (SubL src1 (LShiftL src2 src3)));
12800
12801 ins_cost(1.9 * INSN_COST);
12802 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12803
12804 ins_encode %{
12805 __ sub(as_Register($dst$$reg),
12806 as_Register($src1$$reg),
12807 as_Register($src2$$reg),
12808 Assembler::LSL,
12809 $src3$$constant & 0x3f);
12810 %}
12811
12812 ins_pipe(ialu_reg_reg_shift);
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817
12818 // Shift Left followed by Shift Right.
12819 // This idiom is used by the compiler for the i2b bytecode etc.
12820 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12821 %{
12822 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12823 ins_cost(INSN_COST * 2);
12824 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12825 ins_encode %{
12826 int lshift = $lshift_count$$constant & 63;
12827 int rshift = $rshift_count$$constant & 63;
12828 int s = 63 - lshift;
12829 int r = (rshift - lshift) & 63;
12830 __ sbfm(as_Register($dst$$reg),
12831 as_Register($src$$reg),
12832 r, s);
12833 %}
12834
12835 ins_pipe(ialu_reg_shift);
12836 %}
12837
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840
12841 // Shift Left followed by Shift Right.
12842 // This idiom is used by the compiler for the i2b bytecode etc.
12843 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12844 %{
12845 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12846 ins_cost(INSN_COST * 2);
12847 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12848 ins_encode %{
12849 int lshift = $lshift_count$$constant & 31;
12850 int rshift = $rshift_count$$constant & 31;
12851 int s = 31 - lshift;
12852 int r = (rshift - lshift) & 31;
12853 __ sbfmw(as_Register($dst$$reg),
12854 as_Register($src$$reg),
12855 r, s);
12856 %}
12857
12858 ins_pipe(ialu_reg_shift);
12859 %}
12860
12861 // This pattern is automatically generated from aarch64_ad.m4
12862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12863
12864 // Shift Left followed by Shift Right.
12865 // This idiom is used by the compiler for the i2b bytecode etc.
12866 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12867 %{
12868 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12869 ins_cost(INSN_COST * 2);
12870 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12871 ins_encode %{
12872 int lshift = $lshift_count$$constant & 63;
12873 int rshift = $rshift_count$$constant & 63;
12874 int s = 63 - lshift;
12875 int r = (rshift - lshift) & 63;
12876 __ ubfm(as_Register($dst$$reg),
12877 as_Register($src$$reg),
12878 r, s);
12879 %}
12880
12881 ins_pipe(ialu_reg_shift);
12882 %}
12883
12884 // This pattern is automatically generated from aarch64_ad.m4
12885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12886
12887 // Shift Left followed by Shift Right.
12888 // This idiom is used by the compiler for the i2b bytecode etc.
12889 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12890 %{
12891 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12892 ins_cost(INSN_COST * 2);
12893 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12894 ins_encode %{
12895 int lshift = $lshift_count$$constant & 31;
12896 int rshift = $rshift_count$$constant & 31;
12897 int s = 31 - lshift;
12898 int r = (rshift - lshift) & 31;
12899 __ ubfmw(as_Register($dst$$reg),
12900 as_Register($src$$reg),
12901 r, s);
12902 %}
12903
12904 ins_pipe(ialu_reg_shift);
12905 %}
12906
12907 // Bitfield extract with shift & mask
12908
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12912 %{
12913 match(Set dst (AndI (URShiftI src rshift) mask));
12914 // Make sure we are not going to exceed what ubfxw can do.
12915 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12916
12917 ins_cost(INSN_COST);
12918 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12919 ins_encode %{
12920 int rshift = $rshift$$constant & 31;
12921 intptr_t mask = $mask$$constant;
12922 int width = exact_log2(mask+1);
12923 __ ubfxw(as_Register($dst$$reg),
12924 as_Register($src$$reg), rshift, width);
12925 %}
12926 ins_pipe(ialu_reg_shift);
12927 %}
12928
12929 // This pattern is automatically generated from aarch64_ad.m4
12930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12931 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12932 %{
12933 match(Set dst (AndL (URShiftL src rshift) mask));
12934 // Make sure we are not going to exceed what ubfx can do.
12935 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12936
12937 ins_cost(INSN_COST);
12938 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12939 ins_encode %{
12940 int rshift = $rshift$$constant & 63;
12941 intptr_t mask = $mask$$constant;
12942 int width = exact_log2_long(mask+1);
12943 __ ubfx(as_Register($dst$$reg),
12944 as_Register($src$$reg), rshift, width);
12945 %}
12946 ins_pipe(ialu_reg_shift);
12947 %}
12948
12949
12950 // This pattern is automatically generated from aarch64_ad.m4
12951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12952
12953 // We can use ubfx when extending an And with a mask when we know mask
12954 // is positive. We know that because immI_bitmask guarantees it.
12955 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12956 %{
12957 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12958 // Make sure we are not going to exceed what ubfxw can do.
12959 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12960
12961 ins_cost(INSN_COST * 2);
12962 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12963 ins_encode %{
12964 int rshift = $rshift$$constant & 31;
12965 intptr_t mask = $mask$$constant;
12966 int width = exact_log2(mask+1);
12967 __ ubfx(as_Register($dst$$reg),
12968 as_Register($src$$reg), rshift, width);
12969 %}
12970 ins_pipe(ialu_reg_shift);
12971 %}
12972
12973
12974 // This pattern is automatically generated from aarch64_ad.m4
12975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12976
12977 // We can use ubfiz when masking by a positive number and then left shifting the result.
12978 // We know that the mask is positive because immI_bitmask guarantees it.
12979 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12980 %{
12981 match(Set dst (LShiftI (AndI src mask) lshift));
12982 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12983
12984 ins_cost(INSN_COST);
12985 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12986 ins_encode %{
12987 int lshift = $lshift$$constant & 31;
12988 intptr_t mask = $mask$$constant;
12989 int width = exact_log2(mask+1);
12990 __ ubfizw(as_Register($dst$$reg),
12991 as_Register($src$$reg), lshift, width);
12992 %}
12993 ins_pipe(ialu_reg_shift);
12994 %}
12995
12996 // This pattern is automatically generated from aarch64_ad.m4
12997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12998
12999 // We can use ubfiz when masking by a positive number and then left shifting the result.
13000 // We know that the mask is positive because immL_bitmask guarantees it.
13001 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13002 %{
13003 match(Set dst (LShiftL (AndL src mask) lshift));
13004 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13005
13006 ins_cost(INSN_COST);
13007 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13008 ins_encode %{
13009 int lshift = $lshift$$constant & 63;
13010 intptr_t mask = $mask$$constant;
13011 int width = exact_log2_long(mask+1);
13012 __ ubfiz(as_Register($dst$$reg),
13013 as_Register($src$$reg), lshift, width);
13014 %}
13015 ins_pipe(ialu_reg_shift);
13016 %}
13017
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020
13021 // We can use ubfiz when masking by a positive number and then left shifting the result.
13022 // We know that the mask is positive because immI_bitmask guarantees it.
13023 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13024 %{
13025 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13026 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13027
13028 ins_cost(INSN_COST);
13029 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13030 ins_encode %{
13031 int lshift = $lshift$$constant & 31;
13032 intptr_t mask = $mask$$constant;
13033 int width = exact_log2(mask+1);
13034 __ ubfizw(as_Register($dst$$reg),
13035 as_Register($src$$reg), lshift, width);
13036 %}
13037 ins_pipe(ialu_reg_shift);
13038 %}
13039
13040 // This pattern is automatically generated from aarch64_ad.m4
13041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13042
13043 // We can use ubfiz when masking by a positive number and then left shifting the result.
13044 // We know that the mask is positive because immL_bitmask guarantees it.
13045 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13046 %{
13047 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13048 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13049
13050 ins_cost(INSN_COST);
13051 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13052 ins_encode %{
13053 int lshift = $lshift$$constant & 63;
13054 intptr_t mask = $mask$$constant;
13055 int width = exact_log2_long(mask+1);
13056 __ ubfiz(as_Register($dst$$reg),
13057 as_Register($src$$reg), lshift, width);
13058 %}
13059 ins_pipe(ialu_reg_shift);
13060 %}
13061
13062
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065
13066 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13067 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13068 %{
13069 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13070 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13071
13072 ins_cost(INSN_COST);
13073 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13074 ins_encode %{
13075 int lshift = $lshift$$constant & 63;
13076 intptr_t mask = $mask$$constant;
13077 int width = exact_log2(mask+1);
13078 __ ubfiz(as_Register($dst$$reg),
13079 as_Register($src$$reg), lshift, width);
13080 %}
13081 ins_pipe(ialu_reg_shift);
13082 %}
13083
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086
13087 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13088 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13089 %{
13090 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13091 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13092
13093 ins_cost(INSN_COST);
13094 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13095 ins_encode %{
13096 int lshift = $lshift$$constant & 31;
13097 intptr_t mask = $mask$$constant;
13098 int width = exact_log2(mask+1);
13099 __ ubfiz(as_Register($dst$$reg),
13100 as_Register($src$$reg), lshift, width);
13101 %}
13102 ins_pipe(ialu_reg_shift);
13103 %}
13104
13105 // This pattern is automatically generated from aarch64_ad.m4
13106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13107
13108 // Can skip int2long conversions after AND with small bitmask
13109 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13110 %{
13111 match(Set dst (ConvI2L (AndI src msk)));
13112 ins_cost(INSN_COST);
13113 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13114 ins_encode %{
13115 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13116 %}
13117 ins_pipe(ialu_reg_shift);
13118 %}
13119
13120
13121 // Rotations
13122
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13126 %{
13127 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13128 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13129
13130 ins_cost(INSN_COST);
13131 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13132
13133 ins_encode %{
13134 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13135 $rshift$$constant & 63);
13136 %}
13137 ins_pipe(ialu_reg_reg_extr);
13138 %}
13139
13140
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13144 %{
13145 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13146 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13147
13148 ins_cost(INSN_COST);
13149 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13150
13151 ins_encode %{
13152 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13153 $rshift$$constant & 31);
13154 %}
13155 ins_pipe(ialu_reg_reg_extr);
13156 %}
13157
13158
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13162 %{
13163 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13164 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13165
13166 ins_cost(INSN_COST);
13167 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13168
13169 ins_encode %{
13170 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13171 $rshift$$constant & 63);
13172 %}
13173 ins_pipe(ialu_reg_reg_extr);
13174 %}
13175
13176
13177 // This pattern is automatically generated from aarch64_ad.m4
13178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13179 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13180 %{
13181 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13182 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13183
13184 ins_cost(INSN_COST);
13185 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13186
13187 ins_encode %{
13188 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13189 $rshift$$constant & 31);
13190 %}
13191 ins_pipe(ialu_reg_reg_extr);
13192 %}
13193
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13197 %{
13198 match(Set dst (RotateRight src shift));
13199
13200 ins_cost(INSN_COST);
13201 format %{ "ror $dst, $src, $shift" %}
13202
13203 ins_encode %{
13204 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13205 $shift$$constant & 0x1f);
13206 %}
13207 ins_pipe(ialu_reg_reg_vshift);
13208 %}
13209
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13213 %{
13214 match(Set dst (RotateRight src shift));
13215
13216 ins_cost(INSN_COST);
13217 format %{ "ror $dst, $src, $shift" %}
13218
13219 ins_encode %{
13220 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13221 $shift$$constant & 0x3f);
13222 %}
13223 ins_pipe(ialu_reg_reg_vshift);
13224 %}
13225
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13229 %{
13230 match(Set dst (RotateRight src shift));
13231
13232 ins_cost(INSN_COST);
13233 format %{ "ror $dst, $src, $shift" %}
13234
13235 ins_encode %{
13236 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13237 %}
13238 ins_pipe(ialu_reg_reg_vshift);
13239 %}
13240
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13244 %{
13245 match(Set dst (RotateRight src shift));
13246
13247 ins_cost(INSN_COST);
13248 format %{ "ror $dst, $src, $shift" %}
13249
13250 ins_encode %{
13251 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13252 %}
13253 ins_pipe(ialu_reg_reg_vshift);
13254 %}
13255
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13259 %{
13260 match(Set dst (RotateLeft src shift));
13261
13262 ins_cost(INSN_COST);
13263 format %{ "rol $dst, $src, $shift" %}
13264
13265 ins_encode %{
13266 __ subw(rscratch1, zr, as_Register($shift$$reg));
13267 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13268 %}
13269 ins_pipe(ialu_reg_reg_vshift);
13270 %}
13271
13272 // This pattern is automatically generated from aarch64_ad.m4
13273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13274 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13275 %{
13276 match(Set dst (RotateLeft src shift));
13277
13278 ins_cost(INSN_COST);
13279 format %{ "rol $dst, $src, $shift" %}
13280
13281 ins_encode %{
13282 __ subw(rscratch1, zr, as_Register($shift$$reg));
13283 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13284 %}
13285 ins_pipe(ialu_reg_reg_vshift);
13286 %}
13287
13288
13289 // Add/subtract (extended)
13290
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13294 %{
13295 match(Set dst (AddL src1 (ConvI2L src2)));
13296 ins_cost(INSN_COST);
13297 format %{ "add $dst, $src1, $src2, sxtw" %}
13298
13299 ins_encode %{
13300 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13301 as_Register($src2$$reg), ext::sxtw);
13302 %}
13303 ins_pipe(ialu_reg_reg);
13304 %}
13305
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13309 %{
13310 match(Set dst (SubL src1 (ConvI2L src2)));
13311 ins_cost(INSN_COST);
13312 format %{ "sub $dst, $src1, $src2, sxtw" %}
13313
13314 ins_encode %{
13315 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13316 as_Register($src2$$reg), ext::sxtw);
13317 %}
13318 ins_pipe(ialu_reg_reg);
13319 %}
13320
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13324 %{
13325 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13326 ins_cost(INSN_COST);
13327 format %{ "add $dst, $src1, $src2, sxth" %}
13328
13329 ins_encode %{
13330 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13331 as_Register($src2$$reg), ext::sxth);
13332 %}
13333 ins_pipe(ialu_reg_reg);
13334 %}
13335
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13339 %{
13340 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13341 ins_cost(INSN_COST);
13342 format %{ "add $dst, $src1, $src2, sxtb" %}
13343
13344 ins_encode %{
13345 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13346 as_Register($src2$$reg), ext::sxtb);
13347 %}
13348 ins_pipe(ialu_reg_reg);
13349 %}
13350
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13354 %{
13355 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13356 ins_cost(INSN_COST);
13357 format %{ "add $dst, $src1, $src2, uxtb" %}
13358
13359 ins_encode %{
13360 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13361 as_Register($src2$$reg), ext::uxtb);
13362 %}
13363 ins_pipe(ialu_reg_reg);
13364 %}
13365
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13369 %{
13370 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13371 ins_cost(INSN_COST);
13372 format %{ "add $dst, $src1, $src2, sxth" %}
13373
13374 ins_encode %{
13375 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13376 as_Register($src2$$reg), ext::sxth);
13377 %}
13378 ins_pipe(ialu_reg_reg);
13379 %}
13380
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13384 %{
13385 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13386 ins_cost(INSN_COST);
13387 format %{ "add $dst, $src1, $src2, sxtw" %}
13388
13389 ins_encode %{
13390 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13391 as_Register($src2$$reg), ext::sxtw);
13392 %}
13393 ins_pipe(ialu_reg_reg);
13394 %}
13395
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13399 %{
13400 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13401 ins_cost(INSN_COST);
13402 format %{ "add $dst, $src1, $src2, sxtb" %}
13403
13404 ins_encode %{
13405 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13406 as_Register($src2$$reg), ext::sxtb);
13407 %}
13408 ins_pipe(ialu_reg_reg);
13409 %}
13410
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13414 %{
13415 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13416 ins_cost(INSN_COST);
13417 format %{ "add $dst, $src1, $src2, uxtb" %}
13418
13419 ins_encode %{
13420 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13421 as_Register($src2$$reg), ext::uxtb);
13422 %}
13423 ins_pipe(ialu_reg_reg);
13424 %}
13425
13426 // This pattern is automatically generated from aarch64_ad.m4
13427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13428 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13429 %{
13430 match(Set dst (AddI src1 (AndI src2 mask)));
13431 ins_cost(INSN_COST);
13432 format %{ "addw $dst, $src1, $src2, uxtb" %}
13433
13434 ins_encode %{
13435 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13436 as_Register($src2$$reg), ext::uxtb);
13437 %}
13438 ins_pipe(ialu_reg_reg);
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13444 %{
13445 match(Set dst (AddI src1 (AndI src2 mask)));
13446 ins_cost(INSN_COST);
13447 format %{ "addw $dst, $src1, $src2, uxth" %}
13448
13449 ins_encode %{
13450 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13451 as_Register($src2$$reg), ext::uxth);
13452 %}
13453 ins_pipe(ialu_reg_reg);
13454 %}
13455
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13459 %{
13460 match(Set dst (AddL src1 (AndL src2 mask)));
13461 ins_cost(INSN_COST);
13462 format %{ "add $dst, $src1, $src2, uxtb" %}
13463
13464 ins_encode %{
13465 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13466 as_Register($src2$$reg), ext::uxtb);
13467 %}
13468 ins_pipe(ialu_reg_reg);
13469 %}
13470
13471 // This pattern is automatically generated from aarch64_ad.m4
13472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13473 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13474 %{
13475 match(Set dst (AddL src1 (AndL src2 mask)));
13476 ins_cost(INSN_COST);
13477 format %{ "add $dst, $src1, $src2, uxth" %}
13478
13479 ins_encode %{
13480 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13481 as_Register($src2$$reg), ext::uxth);
13482 %}
13483 ins_pipe(ialu_reg_reg);
13484 %}
13485
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13489 %{
13490 match(Set dst (AddL src1 (AndL src2 mask)));
13491 ins_cost(INSN_COST);
13492 format %{ "add $dst, $src1, $src2, uxtw" %}
13493
13494 ins_encode %{
13495 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13496 as_Register($src2$$reg), ext::uxtw);
13497 %}
13498 ins_pipe(ialu_reg_reg);
13499 %}
13500
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13504 %{
13505 match(Set dst (SubI src1 (AndI src2 mask)));
13506 ins_cost(INSN_COST);
13507 format %{ "subw $dst, $src1, $src2, uxtb" %}
13508
13509 ins_encode %{
13510 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13511 as_Register($src2$$reg), ext::uxtb);
13512 %}
13513 ins_pipe(ialu_reg_reg);
13514 %}
13515
13516 // This pattern is automatically generated from aarch64_ad.m4
13517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13518 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13519 %{
13520 match(Set dst (SubI src1 (AndI src2 mask)));
13521 ins_cost(INSN_COST);
13522 format %{ "subw $dst, $src1, $src2, uxth" %}
13523
13524 ins_encode %{
13525 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13526 as_Register($src2$$reg), ext::uxth);
13527 %}
13528 ins_pipe(ialu_reg_reg);
13529 %}
13530
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13534 %{
13535 match(Set dst (SubL src1 (AndL src2 mask)));
13536 ins_cost(INSN_COST);
13537 format %{ "sub $dst, $src1, $src2, uxtb" %}
13538
13539 ins_encode %{
13540 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13541 as_Register($src2$$reg), ext::uxtb);
13542 %}
13543 ins_pipe(ialu_reg_reg);
13544 %}
13545
13546 // This pattern is automatically generated from aarch64_ad.m4
13547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13548 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13549 %{
13550 match(Set dst (SubL src1 (AndL src2 mask)));
13551 ins_cost(INSN_COST);
13552 format %{ "sub $dst, $src1, $src2, uxth" %}
13553
13554 ins_encode %{
13555 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13556 as_Register($src2$$reg), ext::uxth);
13557 %}
13558 ins_pipe(ialu_reg_reg);
13559 %}
13560
13561 // This pattern is automatically generated from aarch64_ad.m4
13562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13563 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13564 %{
13565 match(Set dst (SubL src1 (AndL src2 mask)));
13566 ins_cost(INSN_COST);
13567 format %{ "sub $dst, $src1, $src2, uxtw" %}
13568
13569 ins_encode %{
13570 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13571 as_Register($src2$$reg), ext::uxtw);
13572 %}
13573 ins_pipe(ialu_reg_reg);
13574 %}
13575
13576
13577 // This pattern is automatically generated from aarch64_ad.m4
13578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13579 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13580 %{
13581 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13582 ins_cost(1.9 * INSN_COST);
13583 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13584
13585 ins_encode %{
13586 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13587 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13588 %}
13589 ins_pipe(ialu_reg_reg_shift);
13590 %}
13591
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13595 %{
13596 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13597 ins_cost(1.9 * INSN_COST);
13598 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13599
13600 ins_encode %{
13601 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13602 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13603 %}
13604 ins_pipe(ialu_reg_reg_shift);
13605 %}
13606
13607 // This pattern is automatically generated from aarch64_ad.m4
13608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13609 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13610 %{
13611 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13612 ins_cost(1.9 * INSN_COST);
13613 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13614
13615 ins_encode %{
13616 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13617 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13618 %}
13619 ins_pipe(ialu_reg_reg_shift);
13620 %}
13621
13622 // This pattern is automatically generated from aarch64_ad.m4
13623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13624 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13625 %{
13626 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13627 ins_cost(1.9 * INSN_COST);
13628 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13629
13630 ins_encode %{
13631 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13632 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13633 %}
13634 ins_pipe(ialu_reg_reg_shift);
13635 %}
13636
13637 // This pattern is automatically generated from aarch64_ad.m4
13638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13639 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13640 %{
13641 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13642 ins_cost(1.9 * INSN_COST);
13643 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13644
13645 ins_encode %{
13646 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13647 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13648 %}
13649 ins_pipe(ialu_reg_reg_shift);
13650 %}
13651
13652 // This pattern is automatically generated from aarch64_ad.m4
13653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13654 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13655 %{
13656 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13657 ins_cost(1.9 * INSN_COST);
13658 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13659
13660 ins_encode %{
13661 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13662 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13663 %}
13664 ins_pipe(ialu_reg_reg_shift);
13665 %}
13666
13667 // This pattern is automatically generated from aarch64_ad.m4
13668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13669 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13670 %{
13671 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13672 ins_cost(1.9 * INSN_COST);
13673 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13674
13675 ins_encode %{
13676 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13677 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13678 %}
13679 ins_pipe(ialu_reg_reg_shift);
13680 %}
13681
13682 // This pattern is automatically generated from aarch64_ad.m4
13683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13684 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13685 %{
13686 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13687 ins_cost(1.9 * INSN_COST);
13688 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13689
13690 ins_encode %{
13691 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13692 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13693 %}
13694 ins_pipe(ialu_reg_reg_shift);
13695 %}
13696
13697 // This pattern is automatically generated from aarch64_ad.m4
13698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13699 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13700 %{
13701 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13702 ins_cost(1.9 * INSN_COST);
13703 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13704
13705 ins_encode %{
13706 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13707 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13708 %}
13709 ins_pipe(ialu_reg_reg_shift);
13710 %}
13711
13712 // This pattern is automatically generated from aarch64_ad.m4
13713 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13714 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13715 %{
13716 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13717 ins_cost(1.9 * INSN_COST);
13718 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13719
13720 ins_encode %{
13721 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13722 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13723 %}
13724 ins_pipe(ialu_reg_reg_shift);
13725 %}
13726
13727 // This pattern is automatically generated from aarch64_ad.m4
13728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13729 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13730 %{
13731 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13732 ins_cost(1.9 * INSN_COST);
13733 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13734
13735 ins_encode %{
13736 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13737 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13738 %}
13739 ins_pipe(ialu_reg_reg_shift);
13740 %}
13741
13742 // This pattern is automatically generated from aarch64_ad.m4
13743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13744 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13745 %{
13746 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13747 ins_cost(1.9 * INSN_COST);
13748 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13749
13750 ins_encode %{
13751 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13752 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13753 %}
13754 ins_pipe(ialu_reg_reg_shift);
13755 %}
13756
13757 // This pattern is automatically generated from aarch64_ad.m4
13758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13759 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13760 %{
13761 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13762 ins_cost(1.9 * INSN_COST);
13763 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13764
13765 ins_encode %{
13766 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13767 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13768 %}
13769 ins_pipe(ialu_reg_reg_shift);
13770 %}
13771
13772 // This pattern is automatically generated from aarch64_ad.m4
13773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13774 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13775 %{
13776 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13777 ins_cost(1.9 * INSN_COST);
13778 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13779
13780 ins_encode %{
13781 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13782 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13783 %}
13784 ins_pipe(ialu_reg_reg_shift);
13785 %}
13786
13787 // This pattern is automatically generated from aarch64_ad.m4
13788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13789 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13790 %{
13791 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13792 ins_cost(1.9 * INSN_COST);
13793 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13794
13795 ins_encode %{
13796 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13797 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13798 %}
13799 ins_pipe(ialu_reg_reg_shift);
13800 %}
13801
13802 // This pattern is automatically generated from aarch64_ad.m4
13803 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13804 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13805 %{
13806 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13807 ins_cost(1.9 * INSN_COST);
13808 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13809
13810 ins_encode %{
13811 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13812 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13813 %}
13814 ins_pipe(ialu_reg_reg_shift);
13815 %}
13816
13817 // This pattern is automatically generated from aarch64_ad.m4
13818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13819 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13820 %{
13821 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13822 ins_cost(1.9 * INSN_COST);
13823 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13824
13825 ins_encode %{
13826 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13827 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13828 %}
13829 ins_pipe(ialu_reg_reg_shift);
13830 %}
13831
13832 // This pattern is automatically generated from aarch64_ad.m4
13833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13834 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13835 %{
13836 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13837 ins_cost(1.9 * INSN_COST);
13838 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13839
13840 ins_encode %{
13841 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13842 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13843 %}
13844 ins_pipe(ialu_reg_reg_shift);
13845 %}
13846
13847 // This pattern is automatically generated from aarch64_ad.m4
13848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13849 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13850 %{
13851 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13852 ins_cost(1.9 * INSN_COST);
13853 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13854
13855 ins_encode %{
13856 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13857 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13858 %}
13859 ins_pipe(ialu_reg_reg_shift);
13860 %}
13861
13862 // This pattern is automatically generated from aarch64_ad.m4
13863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13864 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13865 %{
13866 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13867 ins_cost(1.9 * INSN_COST);
13868 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13869
13870 ins_encode %{
13871 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13872 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13873 %}
13874 ins_pipe(ialu_reg_reg_shift);
13875 %}
13876
13877 // This pattern is automatically generated from aarch64_ad.m4
13878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13879 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13880 %{
13881 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13882 ins_cost(1.9 * INSN_COST);
13883 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13884
13885 ins_encode %{
13886 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13887 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13888 %}
13889 ins_pipe(ialu_reg_reg_shift);
13890 %}
13891
13892 // This pattern is automatically generated from aarch64_ad.m4
13893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13894 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13895 %{
13896 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13897 ins_cost(1.9 * INSN_COST);
13898 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13899
13900 ins_encode %{
13901 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13902 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13903 %}
13904 ins_pipe(ialu_reg_reg_shift);
13905 %}
13906
13907 // This pattern is automatically generated from aarch64_ad.m4
13908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13909 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13910 %{
13911 effect(DEF dst, USE src1, USE src2, USE cr);
13912 ins_cost(INSN_COST * 2);
13913 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13914
13915 ins_encode %{
13916 __ cselw($dst$$Register,
13917 $src1$$Register,
13918 $src2$$Register,
13919 Assembler::LT);
13920 %}
13921 ins_pipe(icond_reg_reg);
13922 %}
13923
13924 // This pattern is automatically generated from aarch64_ad.m4
13925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13926 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13927 %{
13928 effect(DEF dst, USE src1, USE src2, USE cr);
13929 ins_cost(INSN_COST * 2);
13930 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13931
13932 ins_encode %{
13933 __ cselw($dst$$Register,
13934 $src1$$Register,
13935 $src2$$Register,
13936 Assembler::GT);
13937 %}
13938 ins_pipe(icond_reg_reg);
13939 %}
13940
13941 // This pattern is automatically generated from aarch64_ad.m4
13942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13943 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13944 %{
13945 effect(DEF dst, USE src1, USE cr);
13946 ins_cost(INSN_COST * 2);
13947 format %{ "cselw $dst, $src1, zr lt\t" %}
13948
13949 ins_encode %{
13950 __ cselw($dst$$Register,
13951 $src1$$Register,
13952 zr,
13953 Assembler::LT);
13954 %}
13955 ins_pipe(icond_reg);
13956 %}
13957
13958 // This pattern is automatically generated from aarch64_ad.m4
13959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13960 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13961 %{
13962 effect(DEF dst, USE src1, USE cr);
13963 ins_cost(INSN_COST * 2);
13964 format %{ "cselw $dst, $src1, zr gt\t" %}
13965
13966 ins_encode %{
13967 __ cselw($dst$$Register,
13968 $src1$$Register,
13969 zr,
13970 Assembler::GT);
13971 %}
13972 ins_pipe(icond_reg);
13973 %}
13974
13975 // This pattern is automatically generated from aarch64_ad.m4
13976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13977 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13978 %{
13979 effect(DEF dst, USE src1, USE cr);
13980 ins_cost(INSN_COST * 2);
13981 format %{ "csincw $dst, $src1, zr le\t" %}
13982
13983 ins_encode %{
13984 __ csincw($dst$$Register,
13985 $src1$$Register,
13986 zr,
13987 Assembler::LE);
13988 %}
13989 ins_pipe(icond_reg);
13990 %}
13991
13992 // This pattern is automatically generated from aarch64_ad.m4
13993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13994 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13995 %{
13996 effect(DEF dst, USE src1, USE cr);
13997 ins_cost(INSN_COST * 2);
13998 format %{ "csincw $dst, $src1, zr gt\t" %}
13999
14000 ins_encode %{
14001 __ csincw($dst$$Register,
14002 $src1$$Register,
14003 zr,
14004 Assembler::GT);
14005 %}
14006 ins_pipe(icond_reg);
14007 %}
14008
14009 // This pattern is automatically generated from aarch64_ad.m4
14010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14011 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14012 %{
14013 effect(DEF dst, USE src1, USE cr);
14014 ins_cost(INSN_COST * 2);
14015 format %{ "csinvw $dst, $src1, zr lt\t" %}
14016
14017 ins_encode %{
14018 __ csinvw($dst$$Register,
14019 $src1$$Register,
14020 zr,
14021 Assembler::LT);
14022 %}
14023 ins_pipe(icond_reg);
14024 %}
14025
14026 // This pattern is automatically generated from aarch64_ad.m4
14027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14028 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14029 %{
14030 effect(DEF dst, USE src1, USE cr);
14031 ins_cost(INSN_COST * 2);
14032 format %{ "csinvw $dst, $src1, zr ge\t" %}
14033
14034 ins_encode %{
14035 __ csinvw($dst$$Register,
14036 $src1$$Register,
14037 zr,
14038 Assembler::GE);
14039 %}
14040 ins_pipe(icond_reg);
14041 %}
14042
14043 // This pattern is automatically generated from aarch64_ad.m4
14044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14045 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14046 %{
14047 match(Set dst (MinI src imm));
14048 ins_cost(INSN_COST * 3);
14049 expand %{
14050 rFlagsReg cr;
14051 compI_reg_imm0(cr, src);
14052 cmovI_reg_imm0_lt(dst, src, cr);
14053 %}
14054 %}
14055
14056 // This pattern is automatically generated from aarch64_ad.m4
14057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14058 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14059 %{
14060 match(Set dst (MinI imm src));
14061 ins_cost(INSN_COST * 3);
14062 expand %{
14063 rFlagsReg cr;
14064 compI_reg_imm0(cr, src);
14065 cmovI_reg_imm0_lt(dst, src, cr);
14066 %}
14067 %}
14068
14069 // This pattern is automatically generated from aarch64_ad.m4
14070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14071 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14072 %{
14073 match(Set dst (MinI src imm));
14074 ins_cost(INSN_COST * 3);
14075 expand %{
14076 rFlagsReg cr;
14077 compI_reg_imm0(cr, src);
14078 cmovI_reg_imm1_le(dst, src, cr);
14079 %}
14080 %}
14081
14082 // This pattern is automatically generated from aarch64_ad.m4
14083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14084 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14085 %{
14086 match(Set dst (MinI imm src));
14087 ins_cost(INSN_COST * 3);
14088 expand %{
14089 rFlagsReg cr;
14090 compI_reg_imm0(cr, src);
14091 cmovI_reg_imm1_le(dst, src, cr);
14092 %}
14093 %}
14094
14095 // This pattern is automatically generated from aarch64_ad.m4
14096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14097 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14098 %{
14099 match(Set dst (MinI src imm));
14100 ins_cost(INSN_COST * 3);
14101 expand %{
14102 rFlagsReg cr;
14103 compI_reg_imm0(cr, src);
14104 cmovI_reg_immM1_lt(dst, src, cr);
14105 %}
14106 %}
14107
14108 // This pattern is automatically generated from aarch64_ad.m4
14109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14110 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14111 %{
14112 match(Set dst (MinI imm src));
14113 ins_cost(INSN_COST * 3);
14114 expand %{
14115 rFlagsReg cr;
14116 compI_reg_imm0(cr, src);
14117 cmovI_reg_immM1_lt(dst, src, cr);
14118 %}
14119 %}
14120
14121 // This pattern is automatically generated from aarch64_ad.m4
14122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14123 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14124 %{
14125 match(Set dst (MaxI src imm));
14126 ins_cost(INSN_COST * 3);
14127 expand %{
14128 rFlagsReg cr;
14129 compI_reg_imm0(cr, src);
14130 cmovI_reg_imm0_gt(dst, src, cr);
14131 %}
14132 %}
14133
14134 // This pattern is automatically generated from aarch64_ad.m4
14135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14136 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14137 %{
14138 match(Set dst (MaxI imm src));
14139 ins_cost(INSN_COST * 3);
14140 expand %{
14141 rFlagsReg cr;
14142 compI_reg_imm0(cr, src);
14143 cmovI_reg_imm0_gt(dst, src, cr);
14144 %}
14145 %}
14146
14147 // This pattern is automatically generated from aarch64_ad.m4
14148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14149 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14150 %{
14151 match(Set dst (MaxI src imm));
14152 ins_cost(INSN_COST * 3);
14153 expand %{
14154 rFlagsReg cr;
14155 compI_reg_imm0(cr, src);
14156 cmovI_reg_imm1_gt(dst, src, cr);
14157 %}
14158 %}
14159
14160 // This pattern is automatically generated from aarch64_ad.m4
14161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14162 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14163 %{
14164 match(Set dst (MaxI imm src));
14165 ins_cost(INSN_COST * 3);
14166 expand %{
14167 rFlagsReg cr;
14168 compI_reg_imm0(cr, src);
14169 cmovI_reg_imm1_gt(dst, src, cr);
14170 %}
14171 %}
14172
14173 // This pattern is automatically generated from aarch64_ad.m4
14174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14175 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14176 %{
14177 match(Set dst (MaxI src imm));
14178 ins_cost(INSN_COST * 3);
14179 expand %{
14180 rFlagsReg cr;
14181 compI_reg_imm0(cr, src);
14182 cmovI_reg_immM1_ge(dst, src, cr);
14183 %}
14184 %}
14185
14186 // This pattern is automatically generated from aarch64_ad.m4
14187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14188 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14189 %{
14190 match(Set dst (MaxI imm src));
14191 ins_cost(INSN_COST * 3);
14192 expand %{
14193 rFlagsReg cr;
14194 compI_reg_imm0(cr, src);
14195 cmovI_reg_immM1_ge(dst, src, cr);
14196 %}
14197 %}
14198
14199 // This pattern is automatically generated from aarch64_ad.m4
14200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14201 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14202 %{
14203 match(Set dst (ReverseI src));
14204 ins_cost(INSN_COST);
14205 format %{ "rbitw $dst, $src" %}
14206 ins_encode %{
14207 __ rbitw($dst$$Register, $src$$Register);
14208 %}
14209 ins_pipe(ialu_reg);
14210 %}
14211
14212 // This pattern is automatically generated from aarch64_ad.m4
14213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14214 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14215 %{
14216 match(Set dst (ReverseL src));
14217 ins_cost(INSN_COST);
14218 format %{ "rbit $dst, $src" %}
14219 ins_encode %{
14220 __ rbit($dst$$Register, $src$$Register);
14221 %}
14222 ins_pipe(ialu_reg);
14223 %}
14224
14225
14226 // END This section of the file is automatically generated. Do not edit --------------
14227
14228
14229 // ============================================================================
14230 // Floating Point Arithmetic Instructions
14231
14232 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14233 match(Set dst (AddF src1 src2));
14234
14235 ins_cost(INSN_COST * 5);
14236 format %{ "fadds $dst, $src1, $src2" %}
14237
14238 ins_encode %{
14239 __ fadds(as_FloatRegister($dst$$reg),
14240 as_FloatRegister($src1$$reg),
14241 as_FloatRegister($src2$$reg));
14242 %}
14243
14244 ins_pipe(fp_dop_reg_reg_s);
14245 %}
14246
14247 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14248 match(Set dst (AddD src1 src2));
14249
14250 ins_cost(INSN_COST * 5);
14251 format %{ "faddd $dst, $src1, $src2" %}
14252
14253 ins_encode %{
14254 __ faddd(as_FloatRegister($dst$$reg),
14255 as_FloatRegister($src1$$reg),
14256 as_FloatRegister($src2$$reg));
14257 %}
14258
14259 ins_pipe(fp_dop_reg_reg_d);
14260 %}
14261
14262 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14263 match(Set dst (SubF src1 src2));
14264
14265 ins_cost(INSN_COST * 5);
14266 format %{ "fsubs $dst, $src1, $src2" %}
14267
14268 ins_encode %{
14269 __ fsubs(as_FloatRegister($dst$$reg),
14270 as_FloatRegister($src1$$reg),
14271 as_FloatRegister($src2$$reg));
14272 %}
14273
14274 ins_pipe(fp_dop_reg_reg_s);
14275 %}
14276
14277 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14278 match(Set dst (SubD src1 src2));
14279
14280 ins_cost(INSN_COST * 5);
14281 format %{ "fsubd $dst, $src1, $src2" %}
14282
14283 ins_encode %{
14284 __ fsubd(as_FloatRegister($dst$$reg),
14285 as_FloatRegister($src1$$reg),
14286 as_FloatRegister($src2$$reg));
14287 %}
14288
14289 ins_pipe(fp_dop_reg_reg_d);
14290 %}
14291
14292 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14293 match(Set dst (MulF src1 src2));
14294
14295 ins_cost(INSN_COST * 6);
14296 format %{ "fmuls $dst, $src1, $src2" %}
14297
14298 ins_encode %{
14299 __ fmuls(as_FloatRegister($dst$$reg),
14300 as_FloatRegister($src1$$reg),
14301 as_FloatRegister($src2$$reg));
14302 %}
14303
14304 ins_pipe(fp_dop_reg_reg_s);
14305 %}
14306
14307 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14308 match(Set dst (MulD src1 src2));
14309
14310 ins_cost(INSN_COST * 6);
14311 format %{ "fmuld $dst, $src1, $src2" %}
14312
14313 ins_encode %{
14314 __ fmuld(as_FloatRegister($dst$$reg),
14315 as_FloatRegister($src1$$reg),
14316 as_FloatRegister($src2$$reg));
14317 %}
14318
14319 ins_pipe(fp_dop_reg_reg_d);
14320 %}
14321
14322 // src1 * src2 + src3
14323 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14324 predicate(UseFMA);
14325 match(Set dst (FmaF src3 (Binary src1 src2)));
14326
14327 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14328
14329 ins_encode %{
14330 __ fmadds(as_FloatRegister($dst$$reg),
14331 as_FloatRegister($src1$$reg),
14332 as_FloatRegister($src2$$reg),
14333 as_FloatRegister($src3$$reg));
14334 %}
14335
14336 ins_pipe(pipe_class_default);
14337 %}
14338
14339 // src1 * src2 + src3
14340 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14341 predicate(UseFMA);
14342 match(Set dst (FmaD src3 (Binary src1 src2)));
14343
14344 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14345
14346 ins_encode %{
14347 __ fmaddd(as_FloatRegister($dst$$reg),
14348 as_FloatRegister($src1$$reg),
14349 as_FloatRegister($src2$$reg),
14350 as_FloatRegister($src3$$reg));
14351 %}
14352
14353 ins_pipe(pipe_class_default);
14354 %}
14355
14356 // -src1 * src2 + src3
14357 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14358 predicate(UseFMA);
14359 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14360 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14361
14362 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14363
14364 ins_encode %{
14365 __ fmsubs(as_FloatRegister($dst$$reg),
14366 as_FloatRegister($src1$$reg),
14367 as_FloatRegister($src2$$reg),
14368 as_FloatRegister($src3$$reg));
14369 %}
14370
14371 ins_pipe(pipe_class_default);
14372 %}
14373
14374 // -src1 * src2 + src3
14375 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14376 predicate(UseFMA);
14377 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14378 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14379
14380 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14381
14382 ins_encode %{
14383 __ fmsubd(as_FloatRegister($dst$$reg),
14384 as_FloatRegister($src1$$reg),
14385 as_FloatRegister($src2$$reg),
14386 as_FloatRegister($src3$$reg));
14387 %}
14388
14389 ins_pipe(pipe_class_default);
14390 %}
14391
14392 // -src1 * src2 - src3
14393 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14394 predicate(UseFMA);
14395 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14396 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14397
14398 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14399
14400 ins_encode %{
14401 __ fnmadds(as_FloatRegister($dst$$reg),
14402 as_FloatRegister($src1$$reg),
14403 as_FloatRegister($src2$$reg),
14404 as_FloatRegister($src3$$reg));
14405 %}
14406
14407 ins_pipe(pipe_class_default);
14408 %}
14409
14410 // -src1 * src2 - src3
14411 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14412 predicate(UseFMA);
14413 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14414 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14415
14416 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14417
14418 ins_encode %{
14419 __ fnmaddd(as_FloatRegister($dst$$reg),
14420 as_FloatRegister($src1$$reg),
14421 as_FloatRegister($src2$$reg),
14422 as_FloatRegister($src3$$reg));
14423 %}
14424
14425 ins_pipe(pipe_class_default);
14426 %}
14427
14428 // src1 * src2 - src3
14429 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14430 predicate(UseFMA);
14431 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14432
14433 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14434
14435 ins_encode %{
14436 __ fnmsubs(as_FloatRegister($dst$$reg),
14437 as_FloatRegister($src1$$reg),
14438 as_FloatRegister($src2$$reg),
14439 as_FloatRegister($src3$$reg));
14440 %}
14441
14442 ins_pipe(pipe_class_default);
14443 %}
14444
14445 // src1 * src2 - src3
14446 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14447 predicate(UseFMA);
14448 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14449
14450 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14451
14452 ins_encode %{
14453 // n.b. insn name should be fnmsubd
14454 __ fnmsub(as_FloatRegister($dst$$reg),
14455 as_FloatRegister($src1$$reg),
14456 as_FloatRegister($src2$$reg),
14457 as_FloatRegister($src3$$reg));
14458 %}
14459
14460 ins_pipe(pipe_class_default);
14461 %}
14462
14463
14464 // Math.max(FF)F
14465 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14466 match(Set dst (MaxF src1 src2));
14467
14468 format %{ "fmaxs $dst, $src1, $src2" %}
14469 ins_encode %{
14470 __ fmaxs(as_FloatRegister($dst$$reg),
14471 as_FloatRegister($src1$$reg),
14472 as_FloatRegister($src2$$reg));
14473 %}
14474
14475 ins_pipe(fp_dop_reg_reg_s);
14476 %}
14477
14478 // Math.min(FF)F
14479 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14480 match(Set dst (MinF src1 src2));
14481
14482 format %{ "fmins $dst, $src1, $src2" %}
14483 ins_encode %{
14484 __ fmins(as_FloatRegister($dst$$reg),
14485 as_FloatRegister($src1$$reg),
14486 as_FloatRegister($src2$$reg));
14487 %}
14488
14489 ins_pipe(fp_dop_reg_reg_s);
14490 %}
14491
14492 // Math.max(DD)D
14493 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14494 match(Set dst (MaxD src1 src2));
14495
14496 format %{ "fmaxd $dst, $src1, $src2" %}
14497 ins_encode %{
14498 __ fmaxd(as_FloatRegister($dst$$reg),
14499 as_FloatRegister($src1$$reg),
14500 as_FloatRegister($src2$$reg));
14501 %}
14502
14503 ins_pipe(fp_dop_reg_reg_d);
14504 %}
14505
14506 // Math.min(DD)D
14507 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14508 match(Set dst (MinD src1 src2));
14509
14510 format %{ "fmind $dst, $src1, $src2" %}
14511 ins_encode %{
14512 __ fmind(as_FloatRegister($dst$$reg),
14513 as_FloatRegister($src1$$reg),
14514 as_FloatRegister($src2$$reg));
14515 %}
14516
14517 ins_pipe(fp_dop_reg_reg_d);
14518 %}
14519
14520
14521 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14522 match(Set dst (DivF src1 src2));
14523
14524 ins_cost(INSN_COST * 18);
14525 format %{ "fdivs $dst, $src1, $src2" %}
14526
14527 ins_encode %{
14528 __ fdivs(as_FloatRegister($dst$$reg),
14529 as_FloatRegister($src1$$reg),
14530 as_FloatRegister($src2$$reg));
14531 %}
14532
14533 ins_pipe(fp_div_s);
14534 %}
14535
14536 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14537 match(Set dst (DivD src1 src2));
14538
14539 ins_cost(INSN_COST * 32);
14540 format %{ "fdivd $dst, $src1, $src2" %}
14541
14542 ins_encode %{
14543 __ fdivd(as_FloatRegister($dst$$reg),
14544 as_FloatRegister($src1$$reg),
14545 as_FloatRegister($src2$$reg));
14546 %}
14547
14548 ins_pipe(fp_div_d);
14549 %}
14550
14551 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14552 match(Set dst (NegF src));
14553
14554 ins_cost(INSN_COST * 3);
14555 format %{ "fneg $dst, $src" %}
14556
14557 ins_encode %{
14558 __ fnegs(as_FloatRegister($dst$$reg),
14559 as_FloatRegister($src$$reg));
14560 %}
14561
14562 ins_pipe(fp_uop_s);
14563 %}
14564
14565 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14566 match(Set dst (NegD src));
14567
14568 ins_cost(INSN_COST * 3);
14569 format %{ "fnegd $dst, $src" %}
14570
14571 ins_encode %{
14572 __ fnegd(as_FloatRegister($dst$$reg),
14573 as_FloatRegister($src$$reg));
14574 %}
14575
14576 ins_pipe(fp_uop_d);
14577 %}
14578
14579 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14580 %{
14581 match(Set dst (AbsI src));
14582
14583 effect(KILL cr);
14584 ins_cost(INSN_COST * 2);
14585 format %{ "cmpw $src, zr\n\t"
14586 "cnegw $dst, $src, Assembler::LT\t# int abs"
14587 %}
14588
14589 ins_encode %{
14590 __ cmpw(as_Register($src$$reg), zr);
14591 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14592 %}
14593 ins_pipe(pipe_class_default);
14594 %}
14595
14596 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14597 %{
14598 match(Set dst (AbsL src));
14599
14600 effect(KILL cr);
14601 ins_cost(INSN_COST * 2);
14602 format %{ "cmp $src, zr\n\t"
14603 "cneg $dst, $src, Assembler::LT\t# long abs"
14604 %}
14605
14606 ins_encode %{
14607 __ cmp(as_Register($src$$reg), zr);
14608 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14609 %}
14610 ins_pipe(pipe_class_default);
14611 %}
14612
14613 instruct absF_reg(vRegF dst, vRegF src) %{
14614 match(Set dst (AbsF src));
14615
14616 ins_cost(INSN_COST * 3);
14617 format %{ "fabss $dst, $src" %}
14618 ins_encode %{
14619 __ fabss(as_FloatRegister($dst$$reg),
14620 as_FloatRegister($src$$reg));
14621 %}
14622
14623 ins_pipe(fp_uop_s);
14624 %}
14625
14626 instruct absD_reg(vRegD dst, vRegD src) %{
14627 match(Set dst (AbsD src));
14628
14629 ins_cost(INSN_COST * 3);
14630 format %{ "fabsd $dst, $src" %}
14631 ins_encode %{
14632 __ fabsd(as_FloatRegister($dst$$reg),
14633 as_FloatRegister($src$$reg));
14634 %}
14635
14636 ins_pipe(fp_uop_d);
14637 %}
14638
14639 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14640 match(Set dst (AbsF (SubF src1 src2)));
14641
14642 ins_cost(INSN_COST * 3);
14643 format %{ "fabds $dst, $src1, $src2" %}
14644 ins_encode %{
14645 __ fabds(as_FloatRegister($dst$$reg),
14646 as_FloatRegister($src1$$reg),
14647 as_FloatRegister($src2$$reg));
14648 %}
14649
14650 ins_pipe(fp_uop_s);
14651 %}
14652
14653 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14654 match(Set dst (AbsD (SubD src1 src2)));
14655
14656 ins_cost(INSN_COST * 3);
14657 format %{ "fabdd $dst, $src1, $src2" %}
14658 ins_encode %{
14659 __ fabdd(as_FloatRegister($dst$$reg),
14660 as_FloatRegister($src1$$reg),
14661 as_FloatRegister($src2$$reg));
14662 %}
14663
14664 ins_pipe(fp_uop_d);
14665 %}
14666
14667 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14668 match(Set dst (SqrtD src));
14669
14670 ins_cost(INSN_COST * 50);
14671 format %{ "fsqrtd $dst, $src" %}
14672 ins_encode %{
14673 __ fsqrtd(as_FloatRegister($dst$$reg),
14674 as_FloatRegister($src$$reg));
14675 %}
14676
14677 ins_pipe(fp_div_s);
14678 %}
14679
14680 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14681 match(Set dst (SqrtF src));
14682
14683 ins_cost(INSN_COST * 50);
14684 format %{ "fsqrts $dst, $src" %}
14685 ins_encode %{
14686 __ fsqrts(as_FloatRegister($dst$$reg),
14687 as_FloatRegister($src$$reg));
14688 %}
14689
14690 ins_pipe(fp_div_d);
14691 %}
14692
14693 // Math.rint, floor, ceil
14694 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14695 match(Set dst (RoundDoubleMode src rmode));
14696 format %{ "frint $dst, $src, $rmode" %}
14697 ins_encode %{
14698 switch ($rmode$$constant) {
14699 case RoundDoubleModeNode::rmode_rint:
14700 __ frintnd(as_FloatRegister($dst$$reg),
14701 as_FloatRegister($src$$reg));
14702 break;
14703 case RoundDoubleModeNode::rmode_floor:
14704 __ frintmd(as_FloatRegister($dst$$reg),
14705 as_FloatRegister($src$$reg));
14706 break;
14707 case RoundDoubleModeNode::rmode_ceil:
14708 __ frintpd(as_FloatRegister($dst$$reg),
14709 as_FloatRegister($src$$reg));
14710 break;
14711 }
14712 %}
14713 ins_pipe(fp_uop_d);
14714 %}
14715
14716 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14717 match(Set dst (CopySignD src1 (Binary src2 zero)));
14718 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14719 format %{ "CopySignD $dst $src1 $src2" %}
14720 ins_encode %{
14721 FloatRegister dst = as_FloatRegister($dst$$reg),
14722 src1 = as_FloatRegister($src1$$reg),
14723 src2 = as_FloatRegister($src2$$reg),
14724 zero = as_FloatRegister($zero$$reg);
14725 __ fnegd(dst, zero);
14726 __ bsl(dst, __ T8B, src2, src1);
14727 %}
14728 ins_pipe(fp_uop_d);
14729 %}
14730
14731 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14732 match(Set dst (CopySignF src1 src2));
14733 effect(TEMP_DEF dst, USE src1, USE src2);
14734 format %{ "CopySignF $dst $src1 $src2" %}
14735 ins_encode %{
14736 FloatRegister dst = as_FloatRegister($dst$$reg),
14737 src1 = as_FloatRegister($src1$$reg),
14738 src2 = as_FloatRegister($src2$$reg);
14739 __ movi(dst, __ T2S, 0x80, 24);
14740 __ bsl(dst, __ T8B, src2, src1);
14741 %}
14742 ins_pipe(fp_uop_d);
14743 %}
14744
14745 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14746 match(Set dst (SignumD src (Binary zero one)));
14747 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14748 format %{ "signumD $dst, $src" %}
14749 ins_encode %{
14750 FloatRegister src = as_FloatRegister($src$$reg),
14751 dst = as_FloatRegister($dst$$reg),
14752 zero = as_FloatRegister($zero$$reg),
14753 one = as_FloatRegister($one$$reg);
14754 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14755 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14756 // Bit selection instruction gets bit from "one" for each enabled bit in
14757 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14758 // NaN the whole "src" will be copied because "dst" is zero. For all other
14759 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14760 // from "src", and all other bits are copied from 1.0.
14761 __ bsl(dst, __ T8B, one, src);
14762 %}
14763 ins_pipe(fp_uop_d);
14764 %}
14765
14766 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14767 match(Set dst (SignumF src (Binary zero one)));
14768 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14769 format %{ "signumF $dst, $src" %}
14770 ins_encode %{
14771 FloatRegister src = as_FloatRegister($src$$reg),
14772 dst = as_FloatRegister($dst$$reg),
14773 zero = as_FloatRegister($zero$$reg),
14774 one = as_FloatRegister($one$$reg);
14775 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14776 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14777 // Bit selection instruction gets bit from "one" for each enabled bit in
14778 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14779 // NaN the whole "src" will be copied because "dst" is zero. For all other
14780 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14781 // from "src", and all other bits are copied from 1.0.
14782 __ bsl(dst, __ T8B, one, src);
14783 %}
14784 ins_pipe(fp_uop_d);
14785 %}
14786
14787 instruct onspinwait() %{
14788 match(OnSpinWait);
14789 ins_cost(INSN_COST);
14790
14791 format %{ "onspinwait" %}
14792
14793 ins_encode %{
14794 __ spin_wait();
14795 %}
14796 ins_pipe(pipe_class_empty);
14797 %}
14798
14799 // ============================================================================
14800 // Logical Instructions
14801
14802 // Integer Logical Instructions
14803
14804 // And Instructions
14805
14806
14807 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14808 match(Set dst (AndI src1 src2));
14809
14810 format %{ "andw $dst, $src1, $src2\t# int" %}
14811
14812 ins_cost(INSN_COST);
14813 ins_encode %{
14814 __ andw(as_Register($dst$$reg),
14815 as_Register($src1$$reg),
14816 as_Register($src2$$reg));
14817 %}
14818
14819 ins_pipe(ialu_reg_reg);
14820 %}
14821
14822 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14823 match(Set dst (AndI src1 src2));
14824
14825 format %{ "andsw $dst, $src1, $src2\t# int" %}
14826
14827 ins_cost(INSN_COST);
14828 ins_encode %{
14829 __ andw(as_Register($dst$$reg),
14830 as_Register($src1$$reg),
14831 (uint64_t)($src2$$constant));
14832 %}
14833
14834 ins_pipe(ialu_reg_imm);
14835 %}
14836
14837 // Or Instructions
14838
14839 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14840 match(Set dst (OrI src1 src2));
14841
14842 format %{ "orrw $dst, $src1, $src2\t# int" %}
14843
14844 ins_cost(INSN_COST);
14845 ins_encode %{
14846 __ orrw(as_Register($dst$$reg),
14847 as_Register($src1$$reg),
14848 as_Register($src2$$reg));
14849 %}
14850
14851 ins_pipe(ialu_reg_reg);
14852 %}
14853
14854 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14855 match(Set dst (OrI src1 src2));
14856
14857 format %{ "orrw $dst, $src1, $src2\t# int" %}
14858
14859 ins_cost(INSN_COST);
14860 ins_encode %{
14861 __ orrw(as_Register($dst$$reg),
14862 as_Register($src1$$reg),
14863 (uint64_t)($src2$$constant));
14864 %}
14865
14866 ins_pipe(ialu_reg_imm);
14867 %}
14868
14869 // Xor Instructions
14870
14871 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14872 match(Set dst (XorI src1 src2));
14873
14874 format %{ "eorw $dst, $src1, $src2\t# int" %}
14875
14876 ins_cost(INSN_COST);
14877 ins_encode %{
14878 __ eorw(as_Register($dst$$reg),
14879 as_Register($src1$$reg),
14880 as_Register($src2$$reg));
14881 %}
14882
14883 ins_pipe(ialu_reg_reg);
14884 %}
14885
14886 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14887 match(Set dst (XorI src1 src2));
14888
14889 format %{ "eorw $dst, $src1, $src2\t# int" %}
14890
14891 ins_cost(INSN_COST);
14892 ins_encode %{
14893 __ eorw(as_Register($dst$$reg),
14894 as_Register($src1$$reg),
14895 (uint64_t)($src2$$constant));
14896 %}
14897
14898 ins_pipe(ialu_reg_imm);
14899 %}
14900
14901 // Long Logical Instructions
14902 // TODO
14903
14904 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14905 match(Set dst (AndL src1 src2));
14906
14907 format %{ "and $dst, $src1, $src2\t# int" %}
14908
14909 ins_cost(INSN_COST);
14910 ins_encode %{
14911 __ andr(as_Register($dst$$reg),
14912 as_Register($src1$$reg),
14913 as_Register($src2$$reg));
14914 %}
14915
14916 ins_pipe(ialu_reg_reg);
14917 %}
14918
14919 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14920 match(Set dst (AndL src1 src2));
14921
14922 format %{ "and $dst, $src1, $src2\t# int" %}
14923
14924 ins_cost(INSN_COST);
14925 ins_encode %{
14926 __ andr(as_Register($dst$$reg),
14927 as_Register($src1$$reg),
14928 (uint64_t)($src2$$constant));
14929 %}
14930
14931 ins_pipe(ialu_reg_imm);
14932 %}
14933
14934 // Or Instructions
14935
14936 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14937 match(Set dst (OrL src1 src2));
14938
14939 format %{ "orr $dst, $src1, $src2\t# int" %}
14940
14941 ins_cost(INSN_COST);
14942 ins_encode %{
14943 __ orr(as_Register($dst$$reg),
14944 as_Register($src1$$reg),
14945 as_Register($src2$$reg));
14946 %}
14947
14948 ins_pipe(ialu_reg_reg);
14949 %}
14950
14951 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14952 match(Set dst (OrL src1 src2));
14953
14954 format %{ "orr $dst, $src1, $src2\t# int" %}
14955
14956 ins_cost(INSN_COST);
14957 ins_encode %{
14958 __ orr(as_Register($dst$$reg),
14959 as_Register($src1$$reg),
14960 (uint64_t)($src2$$constant));
14961 %}
14962
14963 ins_pipe(ialu_reg_imm);
14964 %}
14965
14966 // Xor Instructions
14967
14968 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14969 match(Set dst (XorL src1 src2));
14970
14971 format %{ "eor $dst, $src1, $src2\t# int" %}
14972
14973 ins_cost(INSN_COST);
14974 ins_encode %{
14975 __ eor(as_Register($dst$$reg),
14976 as_Register($src1$$reg),
14977 as_Register($src2$$reg));
14978 %}
14979
14980 ins_pipe(ialu_reg_reg);
14981 %}
14982
14983 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14984 match(Set dst (XorL src1 src2));
14985
14986 ins_cost(INSN_COST);
14987 format %{ "eor $dst, $src1, $src2\t# int" %}
14988
14989 ins_encode %{
14990 __ eor(as_Register($dst$$reg),
14991 as_Register($src1$$reg),
14992 (uint64_t)($src2$$constant));
14993 %}
14994
14995 ins_pipe(ialu_reg_imm);
14996 %}
14997
14998 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14999 %{
15000 match(Set dst (ConvI2L src));
15001
15002 ins_cost(INSN_COST);
15003 format %{ "sxtw $dst, $src\t# i2l" %}
15004 ins_encode %{
15005 __ sbfm($dst$$Register, $src$$Register, 0, 31);
15006 %}
15007 ins_pipe(ialu_reg_shift);
15008 %}
15009
15010 // this pattern occurs in bigmath arithmetic
15011 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
15012 %{
15013 match(Set dst (AndL (ConvI2L src) mask));
15014
15015 ins_cost(INSN_COST);
15016 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
15017 ins_encode %{
15018 __ ubfm($dst$$Register, $src$$Register, 0, 31);
15019 %}
15020
15021 ins_pipe(ialu_reg_shift);
15022 %}
15023
15024 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15025 match(Set dst (ConvL2I src));
15026
15027 ins_cost(INSN_COST);
15028 format %{ "movw $dst, $src \t// l2i" %}
15029
15030 ins_encode %{
15031 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15032 %}
15033
15034 ins_pipe(ialu_reg);
15035 %}
15036
15037 instruct convD2F_reg(vRegF dst, vRegD src) %{
15038 match(Set dst (ConvD2F src));
15039
15040 ins_cost(INSN_COST * 5);
15041 format %{ "fcvtd $dst, $src \t// d2f" %}
15042
15043 ins_encode %{
15044 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15045 %}
15046
15047 ins_pipe(fp_d2f);
15048 %}
15049
15050 instruct convF2D_reg(vRegD dst, vRegF src) %{
15051 match(Set dst (ConvF2D src));
15052
15053 ins_cost(INSN_COST * 5);
15054 format %{ "fcvts $dst, $src \t// f2d" %}
15055
15056 ins_encode %{
15057 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15058 %}
15059
15060 ins_pipe(fp_f2d);
15061 %}
15062
15063 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15064 match(Set dst (ConvF2I src));
15065
15066 ins_cost(INSN_COST * 5);
15067 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15068
15069 ins_encode %{
15070 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15071 %}
15072
15073 ins_pipe(fp_f2i);
15074 %}
15075
15076 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15077 match(Set dst (ConvF2L src));
15078
15079 ins_cost(INSN_COST * 5);
15080 format %{ "fcvtzs $dst, $src \t// f2l" %}
15081
15082 ins_encode %{
15083 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15084 %}
15085
15086 ins_pipe(fp_f2l);
15087 %}
15088
15089 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
15090 match(Set dst (ConvF2HF src));
15091 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
15092 "smov $dst, $tmp\t# move result from $tmp to $dst"
15093 %}
15094 effect(TEMP tmp);
15095 ins_encode %{
15096 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
15097 %}
15098 ins_pipe(pipe_slow);
15099 %}
15100
15101 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
15102 match(Set dst (ConvHF2F src));
15103 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
15104 "fcvt $dst, $tmp\t# convert half to single precision"
15105 %}
15106 effect(TEMP tmp);
15107 ins_encode %{
15108 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
15109 %}
15110 ins_pipe(pipe_slow);
15111 %}
15112
15113 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15114 match(Set dst (ConvI2F src));
15115
15116 ins_cost(INSN_COST * 5);
15117 format %{ "scvtfws $dst, $src \t// i2f" %}
15118
15119 ins_encode %{
15120 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15121 %}
15122
15123 ins_pipe(fp_i2f);
15124 %}
15125
15126 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15127 match(Set dst (ConvL2F src));
15128
15129 ins_cost(INSN_COST * 5);
15130 format %{ "scvtfs $dst, $src \t// l2f" %}
15131
15132 ins_encode %{
15133 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15134 %}
15135
15136 ins_pipe(fp_l2f);
15137 %}
15138
15139 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15140 match(Set dst (ConvD2I src));
15141
15142 ins_cost(INSN_COST * 5);
15143 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15144
15145 ins_encode %{
15146 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15147 %}
15148
15149 ins_pipe(fp_d2i);
15150 %}
15151
15152 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15153 match(Set dst (ConvD2L src));
15154
15155 ins_cost(INSN_COST * 5);
15156 format %{ "fcvtzd $dst, $src \t// d2l" %}
15157
15158 ins_encode %{
15159 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15160 %}
15161
15162 ins_pipe(fp_d2l);
15163 %}
15164
15165 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15166 match(Set dst (ConvI2D src));
15167
15168 ins_cost(INSN_COST * 5);
15169 format %{ "scvtfwd $dst, $src \t// i2d" %}
15170
15171 ins_encode %{
15172 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15173 %}
15174
15175 ins_pipe(fp_i2d);
15176 %}
15177
15178 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15179 match(Set dst (ConvL2D src));
15180
15181 ins_cost(INSN_COST * 5);
15182 format %{ "scvtfd $dst, $src \t// l2d" %}
15183
15184 ins_encode %{
15185 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15186 %}
15187
15188 ins_pipe(fp_l2d);
15189 %}
15190
15191 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15192 %{
15193 match(Set dst (RoundD src));
15194 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15195 format %{ "java_round_double $dst,$src"%}
15196 ins_encode %{
15197 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15198 as_FloatRegister($ftmp$$reg));
15199 %}
15200 ins_pipe(pipe_slow);
15201 %}
15202
15203 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15204 %{
15205 match(Set dst (RoundF src));
15206 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15207 format %{ "java_round_float $dst,$src"%}
15208 ins_encode %{
15209 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15210 as_FloatRegister($ftmp$$reg));
15211 %}
15212 ins_pipe(pipe_slow);
15213 %}
15214
15215 // stack <-> reg and reg <-> reg shuffles with no conversion
15216
15217 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15218
15219 match(Set dst (MoveF2I src));
15220
15221 effect(DEF dst, USE src);
15222
15223 ins_cost(4 * INSN_COST);
15224
15225 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15226
15227 ins_encode %{
15228 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15229 %}
15230
15231 ins_pipe(iload_reg_reg);
15232
15233 %}
15234
15235 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15236
15237 match(Set dst (MoveI2F src));
15238
15239 effect(DEF dst, USE src);
15240
15241 ins_cost(4 * INSN_COST);
15242
15243 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15244
15245 ins_encode %{
15246 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15247 %}
15248
15249 ins_pipe(pipe_class_memory);
15250
15251 %}
15252
15253 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15254
15255 match(Set dst (MoveD2L src));
15256
15257 effect(DEF dst, USE src);
15258
15259 ins_cost(4 * INSN_COST);
15260
15261 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15262
15263 ins_encode %{
15264 __ ldr($dst$$Register, Address(sp, $src$$disp));
15265 %}
15266
15267 ins_pipe(iload_reg_reg);
15268
15269 %}
15270
15271 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15272
15273 match(Set dst (MoveL2D src));
15274
15275 effect(DEF dst, USE src);
15276
15277 ins_cost(4 * INSN_COST);
15278
15279 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15280
15281 ins_encode %{
15282 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15283 %}
15284
15285 ins_pipe(pipe_class_memory);
15286
15287 %}
15288
15289 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15290
15291 match(Set dst (MoveF2I src));
15292
15293 effect(DEF dst, USE src);
15294
15295 ins_cost(INSN_COST);
15296
15297 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15298
15299 ins_encode %{
15300 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15301 %}
15302
15303 ins_pipe(pipe_class_memory);
15304
15305 %}
15306
15307 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15308
15309 match(Set dst (MoveI2F src));
15310
15311 effect(DEF dst, USE src);
15312
15313 ins_cost(INSN_COST);
15314
15315 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15316
15317 ins_encode %{
15318 __ strw($src$$Register, Address(sp, $dst$$disp));
15319 %}
15320
15321 ins_pipe(istore_reg_reg);
15322
15323 %}
15324
15325 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15326
15327 match(Set dst (MoveD2L src));
15328
15329 effect(DEF dst, USE src);
15330
15331 ins_cost(INSN_COST);
15332
15333 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15334
15335 ins_encode %{
15336 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15337 %}
15338
15339 ins_pipe(pipe_class_memory);
15340
15341 %}
15342
15343 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15344
15345 match(Set dst (MoveL2D src));
15346
15347 effect(DEF dst, USE src);
15348
15349 ins_cost(INSN_COST);
15350
15351 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15352
15353 ins_encode %{
15354 __ str($src$$Register, Address(sp, $dst$$disp));
15355 %}
15356
15357 ins_pipe(istore_reg_reg);
15358
15359 %}
15360
15361 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15362
15363 match(Set dst (MoveF2I src));
15364
15365 effect(DEF dst, USE src);
15366
15367 ins_cost(INSN_COST);
15368
15369 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15370
15371 ins_encode %{
15372 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15373 %}
15374
15375 ins_pipe(fp_f2i);
15376
15377 %}
15378
15379 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15380
15381 match(Set dst (MoveI2F src));
15382
15383 effect(DEF dst, USE src);
15384
15385 ins_cost(INSN_COST);
15386
15387 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15388
15389 ins_encode %{
15390 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15391 %}
15392
15393 ins_pipe(fp_i2f);
15394
15395 %}
15396
15397 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15398
15399 match(Set dst (MoveD2L src));
15400
15401 effect(DEF dst, USE src);
15402
15403 ins_cost(INSN_COST);
15404
15405 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15406
15407 ins_encode %{
15408 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15409 %}
15410
15411 ins_pipe(fp_d2l);
15412
15413 %}
15414
15415 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15416
15417 match(Set dst (MoveL2D src));
15418
15419 effect(DEF dst, USE src);
15420
15421 ins_cost(INSN_COST);
15422
15423 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15424
15425 ins_encode %{
15426 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15427 %}
15428
15429 ins_pipe(fp_l2d);
15430
15431 %}
15432
15433 // ============================================================================
15434 // clearing of an array
15435
15436 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15437 %{
15438 match(Set dummy (ClearArray cnt base));
15439 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15440
15441 ins_cost(4 * INSN_COST);
15442 format %{ "ClearArray $cnt, $base" %}
15443
15444 ins_encode %{
15445 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15446 if (tpc == NULL) {
15447 ciEnv::current()->record_failure("CodeCache is full");
15448 return;
15449 }
15450 %}
15451
15452 ins_pipe(pipe_class_memory);
15453 %}
15454
15455 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15456 %{
15457 predicate((uint64_t)n->in(2)->get_long()
15458 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15459 match(Set dummy (ClearArray cnt base));
15460 effect(TEMP temp, USE_KILL base, KILL cr);
15461
15462 ins_cost(4 * INSN_COST);
15463 format %{ "ClearArray $cnt, $base" %}
15464
15465 ins_encode %{
15466 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15467 if (tpc == NULL) {
15468 ciEnv::current()->record_failure("CodeCache is full");
15469 return;
15470 }
15471 %}
15472
15473 ins_pipe(pipe_class_memory);
15474 %}
15475
15476 // ============================================================================
15477 // Overflow Math Instructions
15478
15479 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15480 %{
15481 match(Set cr (OverflowAddI op1 op2));
15482
15483 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15484 ins_cost(INSN_COST);
15485 ins_encode %{
15486 __ cmnw($op1$$Register, $op2$$Register);
15487 %}
15488
15489 ins_pipe(icmp_reg_reg);
15490 %}
15491
15492 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15493 %{
15494 match(Set cr (OverflowAddI op1 op2));
15495
15496 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15497 ins_cost(INSN_COST);
15498 ins_encode %{
15499 __ cmnw($op1$$Register, $op2$$constant);
15500 %}
15501
15502 ins_pipe(icmp_reg_imm);
15503 %}
15504
15505 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15506 %{
15507 match(Set cr (OverflowAddL op1 op2));
15508
15509 format %{ "cmn $op1, $op2\t# overflow check long" %}
15510 ins_cost(INSN_COST);
15511 ins_encode %{
15512 __ cmn($op1$$Register, $op2$$Register);
15513 %}
15514
15515 ins_pipe(icmp_reg_reg);
15516 %}
15517
15518 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15519 %{
15520 match(Set cr (OverflowAddL op1 op2));
15521
15522 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15523 ins_cost(INSN_COST);
15524 ins_encode %{
15525 __ adds(zr, $op1$$Register, $op2$$constant);
15526 %}
15527
15528 ins_pipe(icmp_reg_imm);
15529 %}
15530
15531 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15532 %{
15533 match(Set cr (OverflowSubI op1 op2));
15534
15535 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15536 ins_cost(INSN_COST);
15537 ins_encode %{
15538 __ cmpw($op1$$Register, $op2$$Register);
15539 %}
15540
15541 ins_pipe(icmp_reg_reg);
15542 %}
15543
15544 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15545 %{
15546 match(Set cr (OverflowSubI op1 op2));
15547
15548 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15549 ins_cost(INSN_COST);
15550 ins_encode %{
15551 __ cmpw($op1$$Register, $op2$$constant);
15552 %}
15553
15554 ins_pipe(icmp_reg_imm);
15555 %}
15556
15557 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15558 %{
15559 match(Set cr (OverflowSubL op1 op2));
15560
15561 format %{ "cmp $op1, $op2\t# overflow check long" %}
15562 ins_cost(INSN_COST);
15563 ins_encode %{
15564 __ cmp($op1$$Register, $op2$$Register);
15565 %}
15566
15567 ins_pipe(icmp_reg_reg);
15568 %}
15569
15570 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15571 %{
15572 match(Set cr (OverflowSubL op1 op2));
15573
15574 format %{ "cmp $op1, $op2\t# overflow check long" %}
15575 ins_cost(INSN_COST);
15576 ins_encode %{
15577 __ subs(zr, $op1$$Register, $op2$$constant);
15578 %}
15579
15580 ins_pipe(icmp_reg_imm);
15581 %}
15582
15583 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15584 %{
15585 match(Set cr (OverflowSubI zero op1));
15586
15587 format %{ "cmpw zr, $op1\t# overflow check int" %}
15588 ins_cost(INSN_COST);
15589 ins_encode %{
15590 __ cmpw(zr, $op1$$Register);
15591 %}
15592
15593 ins_pipe(icmp_reg_imm);
15594 %}
15595
15596 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15597 %{
15598 match(Set cr (OverflowSubL zero op1));
15599
15600 format %{ "cmp zr, $op1\t# overflow check long" %}
15601 ins_cost(INSN_COST);
15602 ins_encode %{
15603 __ cmp(zr, $op1$$Register);
15604 %}
15605
15606 ins_pipe(icmp_reg_imm);
15607 %}
15608
15609 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15610 %{
15611 match(Set cr (OverflowMulI op1 op2));
15612
15613 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15614 "cmp rscratch1, rscratch1, sxtw\n\t"
15615 "movw rscratch1, #0x80000000\n\t"
15616 "cselw rscratch1, rscratch1, zr, NE\n\t"
15617 "cmpw rscratch1, #1" %}
15618 ins_cost(5 * INSN_COST);
15619 ins_encode %{
15620 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15621 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15622 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15623 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15624 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15625 %}
15626
15627 ins_pipe(pipe_slow);
15628 %}
15629
15630 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15631 %{
15632 match(If cmp (OverflowMulI op1 op2));
15633 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15634 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15635 effect(USE labl, KILL cr);
15636
15637 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15638 "cmp rscratch1, rscratch1, sxtw\n\t"
15639 "b$cmp $labl" %}
15640 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15641 ins_encode %{
15642 Label* L = $labl$$label;
15643 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15644 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15645 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15646 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15647 %}
15648
15649 ins_pipe(pipe_serial);
15650 %}
15651
15652 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15653 %{
15654 match(Set cr (OverflowMulL op1 op2));
15655
15656 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15657 "smulh rscratch2, $op1, $op2\n\t"
15658 "cmp rscratch2, rscratch1, ASR #63\n\t"
15659 "movw rscratch1, #0x80000000\n\t"
15660 "cselw rscratch1, rscratch1, zr, NE\n\t"
15661 "cmpw rscratch1, #1" %}
15662 ins_cost(6 * INSN_COST);
15663 ins_encode %{
15664 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15665 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15666 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15667 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15668 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15669 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15670 %}
15671
15672 ins_pipe(pipe_slow);
15673 %}
15674
15675 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15676 %{
15677 match(If cmp (OverflowMulL op1 op2));
15678 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15679 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15680 effect(USE labl, KILL cr);
15681
15682 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15683 "smulh rscratch2, $op1, $op2\n\t"
15684 "cmp rscratch2, rscratch1, ASR #63\n\t"
15685 "b$cmp $labl" %}
15686 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15687 ins_encode %{
15688 Label* L = $labl$$label;
15689 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15690 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15691 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15692 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15693 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15694 %}
15695
15696 ins_pipe(pipe_serial);
15697 %}
15698
15699 // ============================================================================
15700 // Compare Instructions
15701
15702 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15703 %{
15704 match(Set cr (CmpI op1 op2));
15705
15706 effect(DEF cr, USE op1, USE op2);
15707
15708 ins_cost(INSN_COST);
15709 format %{ "cmpw $op1, $op2" %}
15710
15711 ins_encode(aarch64_enc_cmpw(op1, op2));
15712
15713 ins_pipe(icmp_reg_reg);
15714 %}
15715
15716 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15717 %{
15718 match(Set cr (CmpI op1 zero));
15719
15720 effect(DEF cr, USE op1);
15721
15722 ins_cost(INSN_COST);
15723 format %{ "cmpw $op1, 0" %}
15724
15725 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15726
15727 ins_pipe(icmp_reg_imm);
15728 %}
15729
15730 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15731 %{
15732 match(Set cr (CmpI op1 op2));
15733
15734 effect(DEF cr, USE op1);
15735
15736 ins_cost(INSN_COST);
15737 format %{ "cmpw $op1, $op2" %}
15738
15739 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15740
15741 ins_pipe(icmp_reg_imm);
15742 %}
15743
15744 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15745 %{
15746 match(Set cr (CmpI op1 op2));
15747
15748 effect(DEF cr, USE op1);
15749
15750 ins_cost(INSN_COST * 2);
15751 format %{ "cmpw $op1, $op2" %}
15752
15753 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15754
15755 ins_pipe(icmp_reg_imm);
15756 %}
15757
15758 // Unsigned compare Instructions; really, same as signed compare
15759 // except it should only be used to feed an If or a CMovI which takes a
15760 // cmpOpU.
15761
15762 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15763 %{
15764 match(Set cr (CmpU op1 op2));
15765
15766 effect(DEF cr, USE op1, USE op2);
15767
15768 ins_cost(INSN_COST);
15769 format %{ "cmpw $op1, $op2\t# unsigned" %}
15770
15771 ins_encode(aarch64_enc_cmpw(op1, op2));
15772
15773 ins_pipe(icmp_reg_reg);
15774 %}
15775
15776 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15777 %{
15778 match(Set cr (CmpU op1 zero));
15779
15780 effect(DEF cr, USE op1);
15781
15782 ins_cost(INSN_COST);
15783 format %{ "cmpw $op1, #0\t# unsigned" %}
15784
15785 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15786
15787 ins_pipe(icmp_reg_imm);
15788 %}
15789
15790 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15791 %{
15792 match(Set cr (CmpU op1 op2));
15793
15794 effect(DEF cr, USE op1);
15795
15796 ins_cost(INSN_COST);
15797 format %{ "cmpw $op1, $op2\t# unsigned" %}
15798
15799 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15800
15801 ins_pipe(icmp_reg_imm);
15802 %}
15803
15804 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15805 %{
15806 match(Set cr (CmpU op1 op2));
15807
15808 effect(DEF cr, USE op1);
15809
15810 ins_cost(INSN_COST * 2);
15811 format %{ "cmpw $op1, $op2\t# unsigned" %}
15812
15813 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15814
15815 ins_pipe(icmp_reg_imm);
15816 %}
15817
15818 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15819 %{
15820 match(Set cr (CmpL op1 op2));
15821
15822 effect(DEF cr, USE op1, USE op2);
15823
15824 ins_cost(INSN_COST);
15825 format %{ "cmp $op1, $op2" %}
15826
15827 ins_encode(aarch64_enc_cmp(op1, op2));
15828
15829 ins_pipe(icmp_reg_reg);
15830 %}
15831
15832 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15833 %{
15834 match(Set cr (CmpL op1 zero));
15835
15836 effect(DEF cr, USE op1);
15837
15838 ins_cost(INSN_COST);
15839 format %{ "tst $op1" %}
15840
15841 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15842
15843 ins_pipe(icmp_reg_imm);
15844 %}
15845
15846 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15847 %{
15848 match(Set cr (CmpL op1 op2));
15849
15850 effect(DEF cr, USE op1);
15851
15852 ins_cost(INSN_COST);
15853 format %{ "cmp $op1, $op2" %}
15854
15855 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15856
15857 ins_pipe(icmp_reg_imm);
15858 %}
15859
15860 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15861 %{
15862 match(Set cr (CmpL op1 op2));
15863
15864 effect(DEF cr, USE op1);
15865
15866 ins_cost(INSN_COST * 2);
15867 format %{ "cmp $op1, $op2" %}
15868
15869 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15870
15871 ins_pipe(icmp_reg_imm);
15872 %}
15873
15874 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15875 %{
15876 match(Set cr (CmpUL op1 op2));
15877
15878 effect(DEF cr, USE op1, USE op2);
15879
15880 ins_cost(INSN_COST);
15881 format %{ "cmp $op1, $op2" %}
15882
15883 ins_encode(aarch64_enc_cmp(op1, op2));
15884
15885 ins_pipe(icmp_reg_reg);
15886 %}
15887
15888 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15889 %{
15890 match(Set cr (CmpUL op1 zero));
15891
15892 effect(DEF cr, USE op1);
15893
15894 ins_cost(INSN_COST);
15895 format %{ "tst $op1" %}
15896
15897 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15898
15899 ins_pipe(icmp_reg_imm);
15900 %}
15901
15902 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15903 %{
15904 match(Set cr (CmpUL op1 op2));
15905
15906 effect(DEF cr, USE op1);
15907
15908 ins_cost(INSN_COST);
15909 format %{ "cmp $op1, $op2" %}
15910
15911 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15912
15913 ins_pipe(icmp_reg_imm);
15914 %}
15915
15916 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15917 %{
15918 match(Set cr (CmpUL op1 op2));
15919
15920 effect(DEF cr, USE op1);
15921
15922 ins_cost(INSN_COST * 2);
15923 format %{ "cmp $op1, $op2" %}
15924
15925 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15926
15927 ins_pipe(icmp_reg_imm);
15928 %}
15929
15930 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15931 %{
15932 match(Set cr (CmpP op1 op2));
15933
15934 effect(DEF cr, USE op1, USE op2);
15935
15936 ins_cost(INSN_COST);
15937 format %{ "cmp $op1, $op2\t // ptr" %}
15938
15939 ins_encode(aarch64_enc_cmpp(op1, op2));
15940
15941 ins_pipe(icmp_reg_reg);
15942 %}
15943
15944 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15945 %{
15946 match(Set cr (CmpN op1 op2));
15947
15948 effect(DEF cr, USE op1, USE op2);
15949
15950 ins_cost(INSN_COST);
15951 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15952
15953 ins_encode(aarch64_enc_cmpn(op1, op2));
15954
15955 ins_pipe(icmp_reg_reg);
15956 %}
15957
15958 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15959 %{
15960 match(Set cr (CmpP op1 zero));
15961
15962 effect(DEF cr, USE op1, USE zero);
15963
15964 ins_cost(INSN_COST);
15965 format %{ "cmp $op1, 0\t // ptr" %}
15966
15967 ins_encode(aarch64_enc_testp(op1));
15968
15969 ins_pipe(icmp_reg_imm);
15970 %}
15971
15972 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15973 %{
15974 match(Set cr (CmpN op1 zero));
15975
15976 effect(DEF cr, USE op1, USE zero);
15977
15978 ins_cost(INSN_COST);
15979 format %{ "cmp $op1, 0\t // compressed ptr" %}
15980
15981 ins_encode(aarch64_enc_testn(op1));
15982
15983 ins_pipe(icmp_reg_imm);
15984 %}
15985
15986 // FP comparisons
15987 //
15988 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15989 // using normal cmpOp. See declaration of rFlagsReg for details.
15990
15991 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15992 %{
15993 match(Set cr (CmpF src1 src2));
15994
15995 ins_cost(3 * INSN_COST);
15996 format %{ "fcmps $src1, $src2" %}
15997
15998 ins_encode %{
15999 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16000 %}
16001
16002 ins_pipe(pipe_class_compare);
16003 %}
16004
16005 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
16006 %{
16007 match(Set cr (CmpF src1 src2));
16008
16009 ins_cost(3 * INSN_COST);
16010 format %{ "fcmps $src1, 0.0" %}
16011
16012 ins_encode %{
16013 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
16014 %}
16015
16016 ins_pipe(pipe_class_compare);
16017 %}
16018 // FROM HERE
16019
16020 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16021 %{
16022 match(Set cr (CmpD src1 src2));
16023
16024 ins_cost(3 * INSN_COST);
16025 format %{ "fcmpd $src1, $src2" %}
16026
16027 ins_encode %{
16028 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16029 %}
16030
16031 ins_pipe(pipe_class_compare);
16032 %}
16033
16034 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16035 %{
16036 match(Set cr (CmpD src1 src2));
16037
16038 ins_cost(3 * INSN_COST);
16039 format %{ "fcmpd $src1, 0.0" %}
16040
16041 ins_encode %{
16042 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16043 %}
16044
16045 ins_pipe(pipe_class_compare);
16046 %}
16047
16048 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16049 %{
16050 match(Set dst (CmpF3 src1 src2));
16051 effect(KILL cr);
16052
16053 ins_cost(5 * INSN_COST);
16054 format %{ "fcmps $src1, $src2\n\t"
16055 "csinvw($dst, zr, zr, eq\n\t"
16056 "csnegw($dst, $dst, $dst, lt)"
16057 %}
16058
16059 ins_encode %{
16060 Label done;
16061 FloatRegister s1 = as_FloatRegister($src1$$reg);
16062 FloatRegister s2 = as_FloatRegister($src2$$reg);
16063 Register d = as_Register($dst$$reg);
16064 __ fcmps(s1, s2);
16065 // installs 0 if EQ else -1
16066 __ csinvw(d, zr, zr, Assembler::EQ);
16067 // keeps -1 if less or unordered else installs 1
16068 __ csnegw(d, d, d, Assembler::LT);
16069 __ bind(done);
16070 %}
16071
16072 ins_pipe(pipe_class_default);
16073
16074 %}
16075
16076 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16077 %{
16078 match(Set dst (CmpD3 src1 src2));
16079 effect(KILL cr);
16080
16081 ins_cost(5 * INSN_COST);
16082 format %{ "fcmpd $src1, $src2\n\t"
16083 "csinvw($dst, zr, zr, eq\n\t"
16084 "csnegw($dst, $dst, $dst, lt)"
16085 %}
16086
16087 ins_encode %{
16088 Label done;
16089 FloatRegister s1 = as_FloatRegister($src1$$reg);
16090 FloatRegister s2 = as_FloatRegister($src2$$reg);
16091 Register d = as_Register($dst$$reg);
16092 __ fcmpd(s1, s2);
16093 // installs 0 if EQ else -1
16094 __ csinvw(d, zr, zr, Assembler::EQ);
16095 // keeps -1 if less or unordered else installs 1
16096 __ csnegw(d, d, d, Assembler::LT);
16097 __ bind(done);
16098 %}
16099 ins_pipe(pipe_class_default);
16100
16101 %}
16102
16103 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16104 %{
16105 match(Set dst (CmpF3 src1 zero));
16106 effect(KILL cr);
16107
16108 ins_cost(5 * INSN_COST);
16109 format %{ "fcmps $src1, 0.0\n\t"
16110 "csinvw($dst, zr, zr, eq\n\t"
16111 "csnegw($dst, $dst, $dst, lt)"
16112 %}
16113
16114 ins_encode %{
16115 Label done;
16116 FloatRegister s1 = as_FloatRegister($src1$$reg);
16117 Register d = as_Register($dst$$reg);
16118 __ fcmps(s1, 0.0);
16119 // installs 0 if EQ else -1
16120 __ csinvw(d, zr, zr, Assembler::EQ);
16121 // keeps -1 if less or unordered else installs 1
16122 __ csnegw(d, d, d, Assembler::LT);
16123 __ bind(done);
16124 %}
16125
16126 ins_pipe(pipe_class_default);
16127
16128 %}
16129
16130 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16131 %{
16132 match(Set dst (CmpD3 src1 zero));
16133 effect(KILL cr);
16134
16135 ins_cost(5 * INSN_COST);
16136 format %{ "fcmpd $src1, 0.0\n\t"
16137 "csinvw($dst, zr, zr, eq\n\t"
16138 "csnegw($dst, $dst, $dst, lt)"
16139 %}
16140
16141 ins_encode %{
16142 Label done;
16143 FloatRegister s1 = as_FloatRegister($src1$$reg);
16144 Register d = as_Register($dst$$reg);
16145 __ fcmpd(s1, 0.0);
16146 // installs 0 if EQ else -1
16147 __ csinvw(d, zr, zr, Assembler::EQ);
16148 // keeps -1 if less or unordered else installs 1
16149 __ csnegw(d, d, d, Assembler::LT);
16150 __ bind(done);
16151 %}
16152 ins_pipe(pipe_class_default);
16153
16154 %}
16155
16156 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16157 %{
16158 match(Set dst (CmpLTMask p q));
16159 effect(KILL cr);
16160
16161 ins_cost(3 * INSN_COST);
16162
16163 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16164 "csetw $dst, lt\n\t"
16165 "subw $dst, zr, $dst"
16166 %}
16167
16168 ins_encode %{
16169 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16170 __ csetw(as_Register($dst$$reg), Assembler::LT);
16171 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16172 %}
16173
16174 ins_pipe(ialu_reg_reg);
16175 %}
16176
16177 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16178 %{
16179 match(Set dst (CmpLTMask src zero));
16180 effect(KILL cr);
16181
16182 ins_cost(INSN_COST);
16183
16184 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16185
16186 ins_encode %{
16187 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16188 %}
16189
16190 ins_pipe(ialu_reg_shift);
16191 %}
16192
16193 // ============================================================================
16194 // Max and Min
16195
16196 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16197
16198 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16199 %{
16200 effect(DEF cr, USE src);
16201 ins_cost(INSN_COST);
16202 format %{ "cmpw $src, 0" %}
16203
16204 ins_encode %{
16205 __ cmpw($src$$Register, 0);
16206 %}
16207 ins_pipe(icmp_reg_imm);
16208 %}
16209
16210 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16211 %{
16212 match(Set dst (MinI src1 src2));
16213 ins_cost(INSN_COST * 3);
16214
16215 expand %{
16216 rFlagsReg cr;
16217 compI_reg_reg(cr, src1, src2);
16218 cmovI_reg_reg_lt(dst, src1, src2, cr);
16219 %}
16220 %}
16221
16222 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16223 %{
16224 match(Set dst (MaxI src1 src2));
16225 ins_cost(INSN_COST * 3);
16226
16227 expand %{
16228 rFlagsReg cr;
16229 compI_reg_reg(cr, src1, src2);
16230 cmovI_reg_reg_gt(dst, src1, src2, cr);
16231 %}
16232 %}
16233
16234
16235 // ============================================================================
16236 // Branch Instructions
16237
16238 // Direct Branch.
16239 instruct branch(label lbl)
16240 %{
16241 match(Goto);
16242
16243 effect(USE lbl);
16244
16245 ins_cost(BRANCH_COST);
16246 format %{ "b $lbl" %}
16247
16248 ins_encode(aarch64_enc_b(lbl));
16249
16250 ins_pipe(pipe_branch);
16251 %}
16252
16253 // Conditional Near Branch
16254 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16255 %{
16256 // Same match rule as `branchConFar'.
16257 match(If cmp cr);
16258
16259 effect(USE lbl);
16260
16261 ins_cost(BRANCH_COST);
16262 // If set to 1 this indicates that the current instruction is a
16263 // short variant of a long branch. This avoids using this
16264 // instruction in first-pass matching. It will then only be used in
16265 // the `Shorten_branches' pass.
16266 // ins_short_branch(1);
16267 format %{ "b$cmp $lbl" %}
16268
16269 ins_encode(aarch64_enc_br_con(cmp, lbl));
16270
16271 ins_pipe(pipe_branch_cond);
16272 %}
16273
16274 // Conditional Near Branch Unsigned
16275 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16276 %{
16277 // Same match rule as `branchConFar'.
16278 match(If cmp cr);
16279
16280 effect(USE lbl);
16281
16282 ins_cost(BRANCH_COST);
16283 // If set to 1 this indicates that the current instruction is a
16284 // short variant of a long branch. This avoids using this
16285 // instruction in first-pass matching. It will then only be used in
16286 // the `Shorten_branches' pass.
16287 // ins_short_branch(1);
16288 format %{ "b$cmp $lbl\t# unsigned" %}
16289
16290 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16291
16292 ins_pipe(pipe_branch_cond);
16293 %}
16294
16295 // Make use of CBZ and CBNZ. These instructions, as well as being
16296 // shorter than (cmp; branch), have the additional benefit of not
16297 // killing the flags.
16298
16299 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16300 match(If cmp (CmpI op1 op2));
16301 effect(USE labl);
16302
16303 ins_cost(BRANCH_COST);
16304 format %{ "cbw$cmp $op1, $labl" %}
16305 ins_encode %{
16306 Label* L = $labl$$label;
16307 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16308 if (cond == Assembler::EQ)
16309 __ cbzw($op1$$Register, *L);
16310 else
16311 __ cbnzw($op1$$Register, *L);
16312 %}
16313 ins_pipe(pipe_cmp_branch);
16314 %}
16315
16316 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16317 match(If cmp (CmpL op1 op2));
16318 effect(USE labl);
16319
16320 ins_cost(BRANCH_COST);
16321 format %{ "cb$cmp $op1, $labl" %}
16322 ins_encode %{
16323 Label* L = $labl$$label;
16324 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16325 if (cond == Assembler::EQ)
16326 __ cbz($op1$$Register, *L);
16327 else
16328 __ cbnz($op1$$Register, *L);
16329 %}
16330 ins_pipe(pipe_cmp_branch);
16331 %}
16332
16333 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16334 match(If cmp (CmpP op1 op2));
16335 effect(USE labl);
16336
16337 ins_cost(BRANCH_COST);
16338 format %{ "cb$cmp $op1, $labl" %}
16339 ins_encode %{
16340 Label* L = $labl$$label;
16341 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16342 if (cond == Assembler::EQ)
16343 __ cbz($op1$$Register, *L);
16344 else
16345 __ cbnz($op1$$Register, *L);
16346 %}
16347 ins_pipe(pipe_cmp_branch);
16348 %}
16349
16350 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16351 match(If cmp (CmpN op1 op2));
16352 effect(USE labl);
16353
16354 ins_cost(BRANCH_COST);
16355 format %{ "cbw$cmp $op1, $labl" %}
16356 ins_encode %{
16357 Label* L = $labl$$label;
16358 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16359 if (cond == Assembler::EQ)
16360 __ cbzw($op1$$Register, *L);
16361 else
16362 __ cbnzw($op1$$Register, *L);
16363 %}
16364 ins_pipe(pipe_cmp_branch);
16365 %}
16366
16367 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16368 match(If cmp (CmpP (DecodeN oop) zero));
16369 effect(USE labl);
16370
16371 ins_cost(BRANCH_COST);
16372 format %{ "cb$cmp $oop, $labl" %}
16373 ins_encode %{
16374 Label* L = $labl$$label;
16375 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16376 if (cond == Assembler::EQ)
16377 __ cbzw($oop$$Register, *L);
16378 else
16379 __ cbnzw($oop$$Register, *L);
16380 %}
16381 ins_pipe(pipe_cmp_branch);
16382 %}
16383
16384 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16385 match(If cmp (CmpU op1 op2));
16386 effect(USE labl);
16387
16388 ins_cost(BRANCH_COST);
16389 format %{ "cbw$cmp $op1, $labl" %}
16390 ins_encode %{
16391 Label* L = $labl$$label;
16392 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16393 if (cond == Assembler::EQ || cond == Assembler::LS)
16394 __ cbzw($op1$$Register, *L);
16395 else
16396 __ cbnzw($op1$$Register, *L);
16397 %}
16398 ins_pipe(pipe_cmp_branch);
16399 %}
16400
16401 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16402 match(If cmp (CmpUL op1 op2));
16403 effect(USE labl);
16404
16405 ins_cost(BRANCH_COST);
16406 format %{ "cb$cmp $op1, $labl" %}
16407 ins_encode %{
16408 Label* L = $labl$$label;
16409 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16410 if (cond == Assembler::EQ || cond == Assembler::LS)
16411 __ cbz($op1$$Register, *L);
16412 else
16413 __ cbnz($op1$$Register, *L);
16414 %}
16415 ins_pipe(pipe_cmp_branch);
16416 %}
16417
16418 // Test bit and Branch
16419
16420 // Patterns for short (< 32KiB) variants
16421 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16422 match(If cmp (CmpL op1 op2));
16423 effect(USE labl);
16424
16425 ins_cost(BRANCH_COST);
16426 format %{ "cb$cmp $op1, $labl # long" %}
16427 ins_encode %{
16428 Label* L = $labl$$label;
16429 Assembler::Condition cond =
16430 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16431 __ tbr(cond, $op1$$Register, 63, *L);
16432 %}
16433 ins_pipe(pipe_cmp_branch);
16434 ins_short_branch(1);
16435 %}
16436
16437 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16438 match(If cmp (CmpI op1 op2));
16439 effect(USE labl);
16440
16441 ins_cost(BRANCH_COST);
16442 format %{ "cb$cmp $op1, $labl # int" %}
16443 ins_encode %{
16444 Label* L = $labl$$label;
16445 Assembler::Condition cond =
16446 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16447 __ tbr(cond, $op1$$Register, 31, *L);
16448 %}
16449 ins_pipe(pipe_cmp_branch);
16450 ins_short_branch(1);
16451 %}
16452
16453 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16454 match(If cmp (CmpL (AndL op1 op2) op3));
16455 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16456 effect(USE labl);
16457
16458 ins_cost(BRANCH_COST);
16459 format %{ "tb$cmp $op1, $op2, $labl" %}
16460 ins_encode %{
16461 Label* L = $labl$$label;
16462 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16463 int bit = exact_log2_long($op2$$constant);
16464 __ tbr(cond, $op1$$Register, bit, *L);
16465 %}
16466 ins_pipe(pipe_cmp_branch);
16467 ins_short_branch(1);
16468 %}
16469
16470 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16471 match(If cmp (CmpI (AndI op1 op2) op3));
16472 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16473 effect(USE labl);
16474
16475 ins_cost(BRANCH_COST);
16476 format %{ "tb$cmp $op1, $op2, $labl" %}
16477 ins_encode %{
16478 Label* L = $labl$$label;
16479 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16480 int bit = exact_log2((juint)$op2$$constant);
16481 __ tbr(cond, $op1$$Register, bit, *L);
16482 %}
16483 ins_pipe(pipe_cmp_branch);
16484 ins_short_branch(1);
16485 %}
16486
16487 // And far variants
16488 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16489 match(If cmp (CmpL op1 op2));
16490 effect(USE labl);
16491
16492 ins_cost(BRANCH_COST);
16493 format %{ "cb$cmp $op1, $labl # long" %}
16494 ins_encode %{
16495 Label* L = $labl$$label;
16496 Assembler::Condition cond =
16497 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16498 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16499 %}
16500 ins_pipe(pipe_cmp_branch);
16501 %}
16502
16503 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16504 match(If cmp (CmpI op1 op2));
16505 effect(USE labl);
16506
16507 ins_cost(BRANCH_COST);
16508 format %{ "cb$cmp $op1, $labl # int" %}
16509 ins_encode %{
16510 Label* L = $labl$$label;
16511 Assembler::Condition cond =
16512 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16513 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16514 %}
16515 ins_pipe(pipe_cmp_branch);
16516 %}
16517
16518 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16519 match(If cmp (CmpL (AndL op1 op2) op3));
16520 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16521 effect(USE labl);
16522
16523 ins_cost(BRANCH_COST);
16524 format %{ "tb$cmp $op1, $op2, $labl" %}
16525 ins_encode %{
16526 Label* L = $labl$$label;
16527 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16528 int bit = exact_log2_long($op2$$constant);
16529 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16530 %}
16531 ins_pipe(pipe_cmp_branch);
16532 %}
16533
16534 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16535 match(If cmp (CmpI (AndI op1 op2) op3));
16536 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16537 effect(USE labl);
16538
16539 ins_cost(BRANCH_COST);
16540 format %{ "tb$cmp $op1, $op2, $labl" %}
16541 ins_encode %{
16542 Label* L = $labl$$label;
16543 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16544 int bit = exact_log2((juint)$op2$$constant);
16545 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16546 %}
16547 ins_pipe(pipe_cmp_branch);
16548 %}
16549
16550 // Test bits
16551
16552 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16553 match(Set cr (CmpL (AndL op1 op2) op3));
16554 predicate(Assembler::operand_valid_for_logical_immediate
16555 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16556
16557 ins_cost(INSN_COST);
16558 format %{ "tst $op1, $op2 # long" %}
16559 ins_encode %{
16560 __ tst($op1$$Register, $op2$$constant);
16561 %}
16562 ins_pipe(ialu_reg_reg);
16563 %}
16564
16565 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16566 match(Set cr (CmpI (AndI op1 op2) op3));
16567 predicate(Assembler::operand_valid_for_logical_immediate
16568 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16569
16570 ins_cost(INSN_COST);
16571 format %{ "tst $op1, $op2 # int" %}
16572 ins_encode %{
16573 __ tstw($op1$$Register, $op2$$constant);
16574 %}
16575 ins_pipe(ialu_reg_reg);
16576 %}
16577
16578 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16579 match(Set cr (CmpL (AndL op1 op2) op3));
16580
16581 ins_cost(INSN_COST);
16582 format %{ "tst $op1, $op2 # long" %}
16583 ins_encode %{
16584 __ tst($op1$$Register, $op2$$Register);
16585 %}
16586 ins_pipe(ialu_reg_reg);
16587 %}
16588
16589 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16590 match(Set cr (CmpI (AndI op1 op2) op3));
16591
16592 ins_cost(INSN_COST);
16593 format %{ "tstw $op1, $op2 # int" %}
16594 ins_encode %{
16595 __ tstw($op1$$Register, $op2$$Register);
16596 %}
16597 ins_pipe(ialu_reg_reg);
16598 %}
16599
16600
16601 // Conditional Far Branch
16602 // Conditional Far Branch Unsigned
16603 // TODO: fixme
16604
16605 // counted loop end branch near
16606 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16607 %{
16608 match(CountedLoopEnd cmp cr);
16609
16610 effect(USE lbl);
16611
16612 ins_cost(BRANCH_COST);
16613 // short variant.
16614 // ins_short_branch(1);
16615 format %{ "b$cmp $lbl \t// counted loop end" %}
16616
16617 ins_encode(aarch64_enc_br_con(cmp, lbl));
16618
16619 ins_pipe(pipe_branch);
16620 %}
16621
16622 // counted loop end branch far
16623 // TODO: fixme
16624
16625 // ============================================================================
16626 // inlined locking and unlocking
16627
16628 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16629 %{
16630 match(Set cr (FastLock object box));
16631 effect(TEMP tmp, TEMP tmp2);
16632
16633 // TODO
16634 // identify correct cost
16635 ins_cost(5 * INSN_COST);
16636 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16637
16638 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16639
16640 ins_pipe(pipe_serial);
16641 %}
16642
16643 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16644 %{
16645 match(Set cr (FastUnlock object box));
16646 effect(TEMP tmp, TEMP tmp2);
16647
16648 ins_cost(5 * INSN_COST);
16649 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16650
16651 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16652
16653 ins_pipe(pipe_serial);
16654 %}
16655
16656
16657 // ============================================================================
16658 // Safepoint Instructions
16659
16660 // TODO
16661 // provide a near and far version of this code
16662
16663 instruct safePoint(rFlagsReg cr, iRegP poll)
16664 %{
16665 match(SafePoint poll);
16666 effect(KILL cr);
16667
16668 format %{
16669 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16670 %}
16671 ins_encode %{
16672 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16673 %}
16674 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16675 %}
16676
16677
16678 // ============================================================================
16679 // Procedure Call/Return Instructions
16680
16681 // Call Java Static Instruction
16682
16683 instruct CallStaticJavaDirect(method meth)
16684 %{
16685 match(CallStaticJava);
16686
16687 effect(USE meth);
16688
16689 ins_cost(CALL_COST);
16690
16691 format %{ "call,static $meth \t// ==> " %}
16692
16693 ins_encode(aarch64_enc_java_static_call(meth),
16694 aarch64_enc_call_epilog);
16695
16696 ins_pipe(pipe_class_call);
16697 %}
16698
16699 // TO HERE
16700
16701 // Call Java Dynamic Instruction
16702 instruct CallDynamicJavaDirect(method meth)
16703 %{
16704 match(CallDynamicJava);
16705
16706 effect(USE meth);
16707
16708 ins_cost(CALL_COST);
16709
16710 format %{ "CALL,dynamic $meth \t// ==> " %}
16711
16712 ins_encode(aarch64_enc_java_dynamic_call(meth),
16713 aarch64_enc_call_epilog);
16714
16715 ins_pipe(pipe_class_call);
16716 %}
16717
16718 // Call Runtime Instruction
16719
16720 instruct CallRuntimeDirect(method meth)
16721 %{
16722 match(CallRuntime);
16723
16724 effect(USE meth);
16725
16726 ins_cost(CALL_COST);
16727
16728 format %{ "CALL, runtime $meth" %}
16729
16730 ins_encode( aarch64_enc_java_to_runtime(meth) );
16731
16732 ins_pipe(pipe_class_call);
16733 %}
16734
16735 // Call Runtime Instruction
16736
16737 instruct CallLeafDirect(method meth)
16738 %{
16739 match(CallLeaf);
16740
16741 effect(USE meth);
16742
16743 ins_cost(CALL_COST);
16744
16745 format %{ "CALL, runtime leaf $meth" %}
16746
16747 ins_encode( aarch64_enc_java_to_runtime(meth) );
16748
16749 ins_pipe(pipe_class_call);
16750 %}
16751
16752 // Call Runtime Instruction
16753
16754 instruct CallLeafNoFPDirect(method meth)
16755 %{
16756 match(CallLeafNoFP);
16757
16758 effect(USE meth);
16759
16760 ins_cost(CALL_COST);
16761
16762 format %{ "CALL, runtime leaf nofp $meth" %}
16763
16764 ins_encode( aarch64_enc_java_to_runtime(meth) );
16765
16766 ins_pipe(pipe_class_call);
16767 %}
16768
16769 // Tail Call; Jump from runtime stub to Java code.
16770 // Also known as an 'interprocedural jump'.
16771 // Target of jump will eventually return to caller.
16772 // TailJump below removes the return address.
16773 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16774 %{
16775 match(TailCall jump_target method_ptr);
16776
16777 ins_cost(CALL_COST);
16778
16779 format %{ "br $jump_target\t# $method_ptr holds method" %}
16780
16781 ins_encode(aarch64_enc_tail_call(jump_target));
16782
16783 ins_pipe(pipe_class_call);
16784 %}
16785
16786 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16787 %{
16788 match(TailJump jump_target ex_oop);
16789
16790 ins_cost(CALL_COST);
16791
16792 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16793
16794 ins_encode(aarch64_enc_tail_jmp(jump_target));
16795
16796 ins_pipe(pipe_class_call);
16797 %}
16798
16799 // Create exception oop: created by stack-crawling runtime code.
16800 // Created exception is now available to this handler, and is setup
16801 // just prior to jumping to this handler. No code emitted.
16802 // TODO check
16803 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16804 instruct CreateException(iRegP_R0 ex_oop)
16805 %{
16806 match(Set ex_oop (CreateEx));
16807
16808 format %{ " -- \t// exception oop; no code emitted" %}
16809
16810 size(0);
16811
16812 ins_encode( /*empty*/ );
16813
16814 ins_pipe(pipe_class_empty);
16815 %}
16816
16817 // Rethrow exception: The exception oop will come in the first
16818 // argument position. Then JUMP (not call) to the rethrow stub code.
16819 instruct RethrowException() %{
16820 match(Rethrow);
16821 ins_cost(CALL_COST);
16822
16823 format %{ "b rethrow_stub" %}
16824
16825 ins_encode( aarch64_enc_rethrow() );
16826
16827 ins_pipe(pipe_class_call);
16828 %}
16829
16830
16831 // Return Instruction
16832 // epilog node loads ret address into lr as part of frame pop
16833 instruct Ret()
16834 %{
16835 match(Return);
16836
16837 format %{ "ret\t// return register" %}
16838
16839 ins_encode( aarch64_enc_ret() );
16840
16841 ins_pipe(pipe_branch);
16842 %}
16843
16844 // Die now.
16845 instruct ShouldNotReachHere() %{
16846 match(Halt);
16847
16848 ins_cost(CALL_COST);
16849 format %{ "ShouldNotReachHere" %}
16850
16851 ins_encode %{
16852 if (is_reachable()) {
16853 __ stop(_halt_reason);
16854 }
16855 %}
16856
16857 ins_pipe(pipe_class_default);
16858 %}
16859
16860 // ============================================================================
16861 // Partial Subtype Check
16862 //
16863 // superklass array for an instance of the superklass. Set a hidden
16864 // internal cache on a hit (cache is checked with exposed code in
16865 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16866 // encoding ALSO sets flags.
16867
16868 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16869 %{
16870 match(Set result (PartialSubtypeCheck sub super));
16871 effect(KILL cr, KILL temp);
16872
16873 ins_cost(1100); // slightly larger than the next version
16874 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16875
16876 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16877
16878 opcode(0x1); // Force zero of result reg on hit
16879
16880 ins_pipe(pipe_class_memory);
16881 %}
16882
16883 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16884 %{
16885 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16886 effect(KILL temp, KILL result);
16887
16888 ins_cost(1100); // slightly larger than the next version
16889 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16890
16891 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16892
16893 opcode(0x0); // Don't zero result reg on hit
16894
16895 ins_pipe(pipe_class_memory);
16896 %}
16897
16898 // Intrisics for String.compareTo()
16899
16900 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16901 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16902 %{
16903 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16904 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16905 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16906
16907 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16908 ins_encode %{
16909 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16910 __ string_compare($str1$$Register, $str2$$Register,
16911 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16912 $tmp1$$Register, $tmp2$$Register,
16913 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16914 %}
16915 ins_pipe(pipe_class_memory);
16916 %}
16917
16918 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16919 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16920 %{
16921 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16922 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16923 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16924
16925 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16926 ins_encode %{
16927 __ string_compare($str1$$Register, $str2$$Register,
16928 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16929 $tmp1$$Register, $tmp2$$Register,
16930 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16931 %}
16932 ins_pipe(pipe_class_memory);
16933 %}
16934
16935 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16936 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16937 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16938 %{
16939 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16940 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16941 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16942 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16943
16944 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16945 ins_encode %{
16946 __ string_compare($str1$$Register, $str2$$Register,
16947 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16948 $tmp1$$Register, $tmp2$$Register,
16949 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16950 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16951 %}
16952 ins_pipe(pipe_class_memory);
16953 %}
16954
16955 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16956 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16957 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16958 %{
16959 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16960 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16961 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16962 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16963
16964 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16965 ins_encode %{
16966 __ string_compare($str1$$Register, $str2$$Register,
16967 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16968 $tmp1$$Register, $tmp2$$Register,
16969 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16970 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16971 %}
16972 ins_pipe(pipe_class_memory);
16973 %}
16974
16975 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16976 // these string_compare variants as NEON register type for convenience so that the prototype of
16977 // string_compare can be shared with all variants.
16978
16979 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16980 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16981 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16982 pRegGov_P1 pgtmp2, rFlagsReg cr)
16983 %{
16984 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16985 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16986 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16987 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16988
16989 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16990 ins_encode %{
16991 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16992 __ string_compare($str1$$Register, $str2$$Register,
16993 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16994 $tmp1$$Register, $tmp2$$Register,
16995 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16996 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16997 StrIntrinsicNode::LL);
16998 %}
16999 ins_pipe(pipe_class_memory);
17000 %}
17001
17002 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17003 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17004 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17005 pRegGov_P1 pgtmp2, rFlagsReg cr)
17006 %{
17007 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
17008 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17009 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17010 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17011
17012 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17013 ins_encode %{
17014 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17015 __ string_compare($str1$$Register, $str2$$Register,
17016 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17017 $tmp1$$Register, $tmp2$$Register,
17018 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17019 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17020 StrIntrinsicNode::LU);
17021 %}
17022 ins_pipe(pipe_class_memory);
17023 %}
17024
17025 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17026 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17027 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17028 pRegGov_P1 pgtmp2, rFlagsReg cr)
17029 %{
17030 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
17031 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17032 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17033 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17034
17035 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17036 ins_encode %{
17037 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17038 __ string_compare($str1$$Register, $str2$$Register,
17039 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17040 $tmp1$$Register, $tmp2$$Register,
17041 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17042 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17043 StrIntrinsicNode::UL);
17044 %}
17045 ins_pipe(pipe_class_memory);
17046 %}
17047
17048 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17049 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17050 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
17051 pRegGov_P1 pgtmp2, rFlagsReg cr)
17052 %{
17053 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
17054 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17055 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
17056 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17057
17058 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
17059 ins_encode %{
17060 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17061 __ string_compare($str1$$Register, $str2$$Register,
17062 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17063 $tmp1$$Register, $tmp2$$Register,
17064 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
17065 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
17066 StrIntrinsicNode::UU);
17067 %}
17068 ins_pipe(pipe_class_memory);
17069 %}
17070
17071 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17072 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17073 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17074 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17075 %{
17076 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17077 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17078 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17079 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17080 TEMP vtmp0, TEMP vtmp1, KILL cr);
17081 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17082 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17083
17084 ins_encode %{
17085 __ string_indexof($str1$$Register, $str2$$Register,
17086 $cnt1$$Register, $cnt2$$Register,
17087 $tmp1$$Register, $tmp2$$Register,
17088 $tmp3$$Register, $tmp4$$Register,
17089 $tmp5$$Register, $tmp6$$Register,
17090 -1, $result$$Register, StrIntrinsicNode::UU);
17091 %}
17092 ins_pipe(pipe_class_memory);
17093 %}
17094
17095 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17096 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17097 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17098 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17099 %{
17100 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17101 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17102 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17103 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17104 TEMP vtmp0, TEMP vtmp1, KILL cr);
17105 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17106 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17107
17108 ins_encode %{
17109 __ string_indexof($str1$$Register, $str2$$Register,
17110 $cnt1$$Register, $cnt2$$Register,
17111 $tmp1$$Register, $tmp2$$Register,
17112 $tmp3$$Register, $tmp4$$Register,
17113 $tmp5$$Register, $tmp6$$Register,
17114 -1, $result$$Register, StrIntrinsicNode::LL);
17115 %}
17116 ins_pipe(pipe_class_memory);
17117 %}
17118
17119 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17120 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17121 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17122 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17123 %{
17124 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17125 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17126 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17127 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17128 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17129 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17130 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17131
17132 ins_encode %{
17133 __ string_indexof($str1$$Register, $str2$$Register,
17134 $cnt1$$Register, $cnt2$$Register,
17135 $tmp1$$Register, $tmp2$$Register,
17136 $tmp3$$Register, $tmp4$$Register,
17137 $tmp5$$Register, $tmp6$$Register,
17138 -1, $result$$Register, StrIntrinsicNode::UL);
17139 %}
17140 ins_pipe(pipe_class_memory);
17141 %}
17142
17143 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17144 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17145 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17146 %{
17147 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17148 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17149 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17150 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17151 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17152 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17153
17154 ins_encode %{
17155 int icnt2 = (int)$int_cnt2$$constant;
17156 __ string_indexof($str1$$Register, $str2$$Register,
17157 $cnt1$$Register, zr,
17158 $tmp1$$Register, $tmp2$$Register,
17159 $tmp3$$Register, $tmp4$$Register, zr, zr,
17160 icnt2, $result$$Register, StrIntrinsicNode::UU);
17161 %}
17162 ins_pipe(pipe_class_memory);
17163 %}
17164
17165 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17166 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17167 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17168 %{
17169 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17170 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17171 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17172 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17173 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17174 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17175
17176 ins_encode %{
17177 int icnt2 = (int)$int_cnt2$$constant;
17178 __ string_indexof($str1$$Register, $str2$$Register,
17179 $cnt1$$Register, zr,
17180 $tmp1$$Register, $tmp2$$Register,
17181 $tmp3$$Register, $tmp4$$Register, zr, zr,
17182 icnt2, $result$$Register, StrIntrinsicNode::LL);
17183 %}
17184 ins_pipe(pipe_class_memory);
17185 %}
17186
17187 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17188 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17189 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17190 %{
17191 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17192 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17193 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17194 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17195 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17196 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17197
17198 ins_encode %{
17199 int icnt2 = (int)$int_cnt2$$constant;
17200 __ string_indexof($str1$$Register, $str2$$Register,
17201 $cnt1$$Register, zr,
17202 $tmp1$$Register, $tmp2$$Register,
17203 $tmp3$$Register, $tmp4$$Register, zr, zr,
17204 icnt2, $result$$Register, StrIntrinsicNode::UL);
17205 %}
17206 ins_pipe(pipe_class_memory);
17207 %}
17208
17209 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17210 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17211 iRegINoSp tmp3, rFlagsReg cr)
17212 %{
17213 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17214 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17215 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17216 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17217
17218 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17219
17220 ins_encode %{
17221 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17222 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17223 $tmp3$$Register);
17224 %}
17225 ins_pipe(pipe_class_memory);
17226 %}
17227
17228 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17229 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17230 iRegINoSp tmp3, rFlagsReg cr)
17231 %{
17232 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17233 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17234 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17235 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17236
17237 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17238
17239 ins_encode %{
17240 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17241 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17242 $tmp3$$Register);
17243 %}
17244 ins_pipe(pipe_class_memory);
17245 %}
17246
17247 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17248 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17249 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17250 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17251 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17252 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17253 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17254 ins_encode %{
17255 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17256 $result$$Register, $ztmp1$$FloatRegister,
17257 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17258 $ptmp$$PRegister, true /* isL */);
17259 %}
17260 ins_pipe(pipe_class_memory);
17261 %}
17262
17263 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17264 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17265 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17266 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17267 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17268 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17269 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17270 ins_encode %{
17271 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17272 $result$$Register, $ztmp1$$FloatRegister,
17273 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17274 $ptmp$$PRegister, false /* isL */);
17275 %}
17276 ins_pipe(pipe_class_memory);
17277 %}
17278
17279 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17280 iRegI_R0 result, rFlagsReg cr)
17281 %{
17282 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17283 match(Set result (StrEquals (Binary str1 str2) cnt));
17284 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17285
17286 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17287 ins_encode %{
17288 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17289 __ string_equals($str1$$Register, $str2$$Register,
17290 $result$$Register, $cnt$$Register, 1);
17291 %}
17292 ins_pipe(pipe_class_memory);
17293 %}
17294
17295 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17296 iRegI_R0 result, rFlagsReg cr)
17297 %{
17298 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17299 match(Set result (StrEquals (Binary str1 str2) cnt));
17300 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17301
17302 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17303 ins_encode %{
17304 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17305 __ string_equals($str1$$Register, $str2$$Register,
17306 $result$$Register, $cnt$$Register, 2);
17307 %}
17308 ins_pipe(pipe_class_memory);
17309 %}
17310
17311 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17312 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17313 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17314 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17315 iRegP_R10 tmp, rFlagsReg cr)
17316 %{
17317 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17318 match(Set result (AryEq ary1 ary2));
17319 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17320 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17321 TEMP vtmp6, TEMP vtmp7, KILL cr);
17322
17323 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17324 ins_encode %{
17325 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17326 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17327 $result$$Register, $tmp$$Register, 1);
17328 if (tpc == NULL) {
17329 ciEnv::current()->record_failure("CodeCache is full");
17330 return;
17331 }
17332 %}
17333 ins_pipe(pipe_class_memory);
17334 %}
17335
17336 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17337 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17338 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17339 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17340 iRegP_R10 tmp, rFlagsReg cr)
17341 %{
17342 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17343 match(Set result (AryEq ary1 ary2));
17344 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17345 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17346 TEMP vtmp6, TEMP vtmp7, KILL cr);
17347
17348 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17349 ins_encode %{
17350 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17351 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17352 $result$$Register, $tmp$$Register, 2);
17353 if (tpc == NULL) {
17354 ciEnv::current()->record_failure("CodeCache is full");
17355 return;
17356 }
17357 %}
17358 ins_pipe(pipe_class_memory);
17359 %}
17360
17361 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17362 %{
17363 match(Set result (CountPositives ary1 len));
17364 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17365 format %{ "count positives byte[] $ary1,$len -> $result" %}
17366 ins_encode %{
17367 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17368 if (tpc == NULL) {
17369 ciEnv::current()->record_failure("CodeCache is full");
17370 return;
17371 }
17372 %}
17373 ins_pipe( pipe_slow );
17374 %}
17375
17376 // fast char[] to byte[] compression
17377 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17378 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17379 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17380 iRegI_R0 result, rFlagsReg cr)
17381 %{
17382 match(Set result (StrCompressedCopy src (Binary dst len)));
17383 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17384 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17385
17386 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17387 ins_encode %{
17388 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17389 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17390 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17391 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17392 %}
17393 ins_pipe(pipe_slow);
17394 %}
17395
17396 // fast byte[] to char[] inflation
17397 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17398 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17399 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17400 %{
17401 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17402 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17403 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17404 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17405
17406 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17407 ins_encode %{
17408 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17409 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17410 $vtmp2$$FloatRegister, $tmp$$Register);
17411 if (tpc == NULL) {
17412 ciEnv::current()->record_failure("CodeCache is full");
17413 return;
17414 }
17415 %}
17416 ins_pipe(pipe_class_memory);
17417 %}
17418
17419 // encode char[] to byte[] in ISO_8859_1
17420 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17421 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17422 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17423 iRegI_R0 result, rFlagsReg cr)
17424 %{
17425 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17426 match(Set result (EncodeISOArray src (Binary dst len)));
17427 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17428 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17429
17430 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17431 ins_encode %{
17432 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17433 $result$$Register, false,
17434 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17435 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17436 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17437 %}
17438 ins_pipe(pipe_class_memory);
17439 %}
17440
17441 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17442 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17443 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17444 iRegI_R0 result, rFlagsReg cr)
17445 %{
17446 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17447 match(Set result (EncodeISOArray src (Binary dst len)));
17448 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17449 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17450
17451 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17452 ins_encode %{
17453 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17454 $result$$Register, true,
17455 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17456 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17457 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17458 %}
17459 ins_pipe(pipe_class_memory);
17460 %}
17461
17462 //----------------------------- CompressBits/ExpandBits ------------------------
17463
17464 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17465 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17466 match(Set dst (CompressBits src mask));
17467 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17468 format %{ "mov $tsrc, $src\n\t"
17469 "mov $tmask, $mask\n\t"
17470 "bext $tdst, $tsrc, $tmask\n\t"
17471 "mov $dst, $tdst"
17472 %}
17473 ins_encode %{
17474 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17475 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17476 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17477 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17478 %}
17479 ins_pipe(pipe_slow);
17480 %}
17481
17482 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17483 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17484 match(Set dst (CompressBits (LoadI mem) mask));
17485 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17486 format %{ "ldrs $tsrc, $mem\n\t"
17487 "ldrs $tmask, $mask\n\t"
17488 "bext $tdst, $tsrc, $tmask\n\t"
17489 "mov $dst, $tdst"
17490 %}
17491 ins_encode %{
17492 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17493 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17494 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17495 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17496 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17497 %}
17498 ins_pipe(pipe_slow);
17499 %}
17500
17501 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17502 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17503 match(Set dst (CompressBits src mask));
17504 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17505 format %{ "mov $tsrc, $src\n\t"
17506 "mov $tmask, $mask\n\t"
17507 "bext $tdst, $tsrc, $tmask\n\t"
17508 "mov $dst, $tdst"
17509 %}
17510 ins_encode %{
17511 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17512 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17513 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17514 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17515 %}
17516 ins_pipe(pipe_slow);
17517 %}
17518
17519 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17520 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17521 match(Set dst (CompressBits (LoadL mem) mask));
17522 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17523 format %{ "ldrd $tsrc, $mem\n\t"
17524 "ldrd $tmask, $mask\n\t"
17525 "bext $tdst, $tsrc, $tmask\n\t"
17526 "mov $dst, $tdst"
17527 %}
17528 ins_encode %{
17529 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17530 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17531 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17532 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17533 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17534 %}
17535 ins_pipe(pipe_slow);
17536 %}
17537
17538 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17539 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17540 match(Set dst (ExpandBits src mask));
17541 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17542 format %{ "mov $tsrc, $src\n\t"
17543 "mov $tmask, $mask\n\t"
17544 "bdep $tdst, $tsrc, $tmask\n\t"
17545 "mov $dst, $tdst"
17546 %}
17547 ins_encode %{
17548 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17549 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17550 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17551 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17552 %}
17553 ins_pipe(pipe_slow);
17554 %}
17555
17556 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17557 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17558 match(Set dst (ExpandBits (LoadI mem) mask));
17559 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17560 format %{ "ldrs $tsrc, $mem\n\t"
17561 "ldrs $tmask, $mask\n\t"
17562 "bdep $tdst, $tsrc, $tmask\n\t"
17563 "mov $dst, $tdst"
17564 %}
17565 ins_encode %{
17566 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17567 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17568 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17569 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17570 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17571 %}
17572 ins_pipe(pipe_slow);
17573 %}
17574
17575 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17576 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17577 match(Set dst (ExpandBits src mask));
17578 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17579 format %{ "mov $tsrc, $src\n\t"
17580 "mov $tmask, $mask\n\t"
17581 "bdep $tdst, $tsrc, $tmask\n\t"
17582 "mov $dst, $tdst"
17583 %}
17584 ins_encode %{
17585 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17586 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17587 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17588 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17589 %}
17590 ins_pipe(pipe_slow);
17591 %}
17592
17593
17594 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17595 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17596 match(Set dst (ExpandBits (LoadL mem) mask));
17597 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17598 format %{ "ldrd $tsrc, $mem\n\t"
17599 "ldrd $tmask, $mask\n\t"
17600 "bdep $tdst, $tsrc, $tmask\n\t"
17601 "mov $dst, $tdst"
17602 %}
17603 ins_encode %{
17604 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17605 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17606 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17607 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17608 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17609 %}
17610 ins_pipe(pipe_slow);
17611 %}
17612
17613 // ============================================================================
17614 // This name is KNOWN by the ADLC and cannot be changed.
17615 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17616 // for this guy.
17617 instruct tlsLoadP(thread_RegP dst)
17618 %{
17619 match(Set dst (ThreadLocal));
17620
17621 ins_cost(0);
17622
17623 format %{ " -- \t// $dst=Thread::current(), empty" %}
17624
17625 size(0);
17626
17627 ins_encode( /*empty*/ );
17628
17629 ins_pipe(pipe_class_empty);
17630 %}
17631
17632 //----------PEEPHOLE RULES-----------------------------------------------------
17633 // These must follow all instruction definitions as they use the names
17634 // defined in the instructions definitions.
17635 //
17636 // peepmatch ( root_instr_name [preceding_instruction]* );
17637 //
17638 // peepconstraint %{
17639 // (instruction_number.operand_name relational_op instruction_number.operand_name
17640 // [, ...] );
17641 // // instruction numbers are zero-based using left to right order in peepmatch
17642 //
17643 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17644 // // provide an instruction_number.operand_name for each operand that appears
17645 // // in the replacement instruction's match rule
17646 //
17647 // ---------VM FLAGS---------------------------------------------------------
17648 //
17649 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17650 //
17651 // Each peephole rule is given an identifying number starting with zero and
17652 // increasing by one in the order seen by the parser. An individual peephole
17653 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17654 // on the command-line.
17655 //
17656 // ---------CURRENT LIMITATIONS----------------------------------------------
17657 //
17658 // Only match adjacent instructions in same basic block
17659 // Only equality constraints
17660 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17661 // Only one replacement instruction
17662 //
17663 // ---------EXAMPLE----------------------------------------------------------
17664 //
17665 // // pertinent parts of existing instructions in architecture description
17666 // instruct movI(iRegINoSp dst, iRegI src)
17667 // %{
17668 // match(Set dst (CopyI src));
17669 // %}
17670 //
17671 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17672 // %{
17673 // match(Set dst (AddI dst src));
17674 // effect(KILL cr);
17675 // %}
17676 //
17677 // // Change (inc mov) to lea
17678 // peephole %{
17679 // // increment preceded by register-register move
17680 // peepmatch ( incI_iReg movI );
17681 // // require that the destination register of the increment
17682 // // match the destination register of the move
17683 // peepconstraint ( 0.dst == 1.dst );
17684 // // construct a replacement instruction that sets
17685 // // the destination to ( move's source register + one )
17686 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17687 // %}
17688 //
17689
17690 // Implementation no longer uses movX instructions since
17691 // machine-independent system no longer uses CopyX nodes.
17692 //
17693 // peephole
17694 // %{
17695 // peepmatch (incI_iReg movI);
17696 // peepconstraint (0.dst == 1.dst);
17697 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17698 // %}
17699
17700 // peephole
17701 // %{
17702 // peepmatch (decI_iReg movI);
17703 // peepconstraint (0.dst == 1.dst);
17704 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17705 // %}
17706
17707 // peephole
17708 // %{
17709 // peepmatch (addI_iReg_imm movI);
17710 // peepconstraint (0.dst == 1.dst);
17711 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17712 // %}
17713
17714 // peephole
17715 // %{
17716 // peepmatch (incL_iReg movL);
17717 // peepconstraint (0.dst == 1.dst);
17718 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17719 // %}
17720
17721 // peephole
17722 // %{
17723 // peepmatch (decL_iReg movL);
17724 // peepconstraint (0.dst == 1.dst);
17725 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17726 // %}
17727
17728 // peephole
17729 // %{
17730 // peepmatch (addL_iReg_imm movL);
17731 // peepconstraint (0.dst == 1.dst);
17732 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17733 // %}
17734
17735 // peephole
17736 // %{
17737 // peepmatch (addP_iReg_imm movP);
17738 // peepconstraint (0.dst == 1.dst);
17739 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17740 // %}
17741
17742 // // Change load of spilled value to only a spill
17743 // instruct storeI(memory mem, iRegI src)
17744 // %{
17745 // match(Set mem (StoreI mem src));
17746 // %}
17747 //
17748 // instruct loadI(iRegINoSp dst, memory mem)
17749 // %{
17750 // match(Set dst (LoadI mem));
17751 // %}
17752 //
17753
17754 //----------SMARTSPILL RULES---------------------------------------------------
17755 // These must follow all instruction definitions as they use the names
17756 // defined in the instructions definitions.
17757
17758 // Local Variables:
17759 // mode: c++
17760 // End: