1 //
2 // Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all float registers
698 reg_class float_reg(
699 V0,
700 V1,
701 V2,
702 V3,
703 V4,
704 V5,
705 V6,
706 V7,
707 V8,
708 V9,
709 V10,
710 V11,
711 V12,
712 V13,
713 V14,
714 V15,
715 V16,
716 V17,
717 V18,
718 V19,
719 V20,
720 V21,
721 V22,
722 V23,
723 V24,
724 V25,
725 V26,
726 V27,
727 V28,
728 V29,
729 V30,
730 V31
731 );
732
733 // Double precision float registers have virtual `high halves' that
734 // are needed by the allocator.
735 // Class for all double registers
736 reg_class double_reg(
737 V0, V0_H,
738 V1, V1_H,
739 V2, V2_H,
740 V3, V3_H,
741 V4, V4_H,
742 V5, V5_H,
743 V6, V6_H,
744 V7, V7_H,
745 V8, V8_H,
746 V9, V9_H,
747 V10, V10_H,
748 V11, V11_H,
749 V12, V12_H,
750 V13, V13_H,
751 V14, V14_H,
752 V15, V15_H,
753 V16, V16_H,
754 V17, V17_H,
755 V18, V18_H,
756 V19, V19_H,
757 V20, V20_H,
758 V21, V21_H,
759 V22, V22_H,
760 V23, V23_H,
761 V24, V24_H,
762 V25, V25_H,
763 V26, V26_H,
764 V27, V27_H,
765 V28, V28_H,
766 V29, V29_H,
767 V30, V30_H,
768 V31, V31_H
769 );
770
771 // Class for all SVE vector registers.
772 reg_class vectora_reg (
773 V0, V0_H, V0_J, V0_K,
774 V1, V1_H, V1_J, V1_K,
775 V2, V2_H, V2_J, V2_K,
776 V3, V3_H, V3_J, V3_K,
777 V4, V4_H, V4_J, V4_K,
778 V5, V5_H, V5_J, V5_K,
779 V6, V6_H, V6_J, V6_K,
780 V7, V7_H, V7_J, V7_K,
781 V8, V8_H, V8_J, V8_K,
782 V9, V9_H, V9_J, V9_K,
783 V10, V10_H, V10_J, V10_K,
784 V11, V11_H, V11_J, V11_K,
785 V12, V12_H, V12_J, V12_K,
786 V13, V13_H, V13_J, V13_K,
787 V14, V14_H, V14_J, V14_K,
788 V15, V15_H, V15_J, V15_K,
789 V16, V16_H, V16_J, V16_K,
790 V17, V17_H, V17_J, V17_K,
791 V18, V18_H, V18_J, V18_K,
792 V19, V19_H, V19_J, V19_K,
793 V20, V20_H, V20_J, V20_K,
794 V21, V21_H, V21_J, V21_K,
795 V22, V22_H, V22_J, V22_K,
796 V23, V23_H, V23_J, V23_K,
797 V24, V24_H, V24_J, V24_K,
798 V25, V25_H, V25_J, V25_K,
799 V26, V26_H, V26_J, V26_K,
800 V27, V27_H, V27_J, V27_K,
801 V28, V28_H, V28_J, V28_K,
802 V29, V29_H, V29_J, V29_K,
803 V30, V30_H, V30_J, V30_K,
804 V31, V31_H, V31_J, V31_K,
805 );
806
807 // Class for all 64bit vector registers
808 reg_class vectord_reg(
809 V0, V0_H,
810 V1, V1_H,
811 V2, V2_H,
812 V3, V3_H,
813 V4, V4_H,
814 V5, V5_H,
815 V6, V6_H,
816 V7, V7_H,
817 V8, V8_H,
818 V9, V9_H,
819 V10, V10_H,
820 V11, V11_H,
821 V12, V12_H,
822 V13, V13_H,
823 V14, V14_H,
824 V15, V15_H,
825 V16, V16_H,
826 V17, V17_H,
827 V18, V18_H,
828 V19, V19_H,
829 V20, V20_H,
830 V21, V21_H,
831 V22, V22_H,
832 V23, V23_H,
833 V24, V24_H,
834 V25, V25_H,
835 V26, V26_H,
836 V27, V27_H,
837 V28, V28_H,
838 V29, V29_H,
839 V30, V30_H,
840 V31, V31_H
841 );
842
843 // Class for all 128bit vector registers
844 reg_class vectorx_reg(
845 V0, V0_H, V0_J, V0_K,
846 V1, V1_H, V1_J, V1_K,
847 V2, V2_H, V2_J, V2_K,
848 V3, V3_H, V3_J, V3_K,
849 V4, V4_H, V4_J, V4_K,
850 V5, V5_H, V5_J, V5_K,
851 V6, V6_H, V6_J, V6_K,
852 V7, V7_H, V7_J, V7_K,
853 V8, V8_H, V8_J, V8_K,
854 V9, V9_H, V9_J, V9_K,
855 V10, V10_H, V10_J, V10_K,
856 V11, V11_H, V11_J, V11_K,
857 V12, V12_H, V12_J, V12_K,
858 V13, V13_H, V13_J, V13_K,
859 V14, V14_H, V14_J, V14_K,
860 V15, V15_H, V15_J, V15_K,
861 V16, V16_H, V16_J, V16_K,
862 V17, V17_H, V17_J, V17_K,
863 V18, V18_H, V18_J, V18_K,
864 V19, V19_H, V19_J, V19_K,
865 V20, V20_H, V20_J, V20_K,
866 V21, V21_H, V21_J, V21_K,
867 V22, V22_H, V22_J, V22_K,
868 V23, V23_H, V23_J, V23_K,
869 V24, V24_H, V24_J, V24_K,
870 V25, V25_H, V25_J, V25_K,
871 V26, V26_H, V26_J, V26_K,
872 V27, V27_H, V27_J, V27_K,
873 V28, V28_H, V28_J, V28_K,
874 V29, V29_H, V29_J, V29_K,
875 V30, V30_H, V30_J, V30_K,
876 V31, V31_H, V31_J, V31_K
877 );
878
879 // Class for 128 bit register v0
880 reg_class v0_reg(
881 V0, V0_H
882 );
883
884 // Class for 128 bit register v1
885 reg_class v1_reg(
886 V1, V1_H
887 );
888
889 // Class for 128 bit register v2
890 reg_class v2_reg(
891 V2, V2_H
892 );
893
894 // Class for 128 bit register v3
895 reg_class v3_reg(
896 V3, V3_H
897 );
898
899 // Class for 128 bit register v4
900 reg_class v4_reg(
901 V4, V4_H
902 );
903
904 // Class for 128 bit register v5
905 reg_class v5_reg(
906 V5, V5_H
907 );
908
909 // Class for 128 bit register v6
910 reg_class v6_reg(
911 V6, V6_H
912 );
913
914 // Class for 128 bit register v7
915 reg_class v7_reg(
916 V7, V7_H
917 );
918
919 // Class for 128 bit register v8
920 reg_class v8_reg(
921 V8, V8_H
922 );
923
924 // Class for 128 bit register v9
925 reg_class v9_reg(
926 V9, V9_H
927 );
928
929 // Class for 128 bit register v10
930 reg_class v10_reg(
931 V10, V10_H
932 );
933
934 // Class for 128 bit register v11
935 reg_class v11_reg(
936 V11, V11_H
937 );
938
939 // Class for 128 bit register v12
940 reg_class v12_reg(
941 V12, V12_H
942 );
943
944 // Class for 128 bit register v13
945 reg_class v13_reg(
946 V13, V13_H
947 );
948
949 // Class for 128 bit register v14
950 reg_class v14_reg(
951 V14, V14_H
952 );
953
954 // Class for 128 bit register v15
955 reg_class v15_reg(
956 V15, V15_H
957 );
958
959 // Class for 128 bit register v16
960 reg_class v16_reg(
961 V16, V16_H
962 );
963
964 // Class for 128 bit register v17
965 reg_class v17_reg(
966 V17, V17_H
967 );
968
969 // Class for 128 bit register v18
970 reg_class v18_reg(
971 V18, V18_H
972 );
973
974 // Class for 128 bit register v19
975 reg_class v19_reg(
976 V19, V19_H
977 );
978
979 // Class for 128 bit register v20
980 reg_class v20_reg(
981 V20, V20_H
982 );
983
984 // Class for 128 bit register v21
985 reg_class v21_reg(
986 V21, V21_H
987 );
988
989 // Class for 128 bit register v22
990 reg_class v22_reg(
991 V22, V22_H
992 );
993
994 // Class for 128 bit register v23
995 reg_class v23_reg(
996 V23, V23_H
997 );
998
999 // Class for 128 bit register v24
1000 reg_class v24_reg(
1001 V24, V24_H
1002 );
1003
1004 // Class for 128 bit register v25
1005 reg_class v25_reg(
1006 V25, V25_H
1007 );
1008
1009 // Class for 128 bit register v26
1010 reg_class v26_reg(
1011 V26, V26_H
1012 );
1013
1014 // Class for 128 bit register v27
1015 reg_class v27_reg(
1016 V27, V27_H
1017 );
1018
1019 // Class for 128 bit register v28
1020 reg_class v28_reg(
1021 V28, V28_H
1022 );
1023
1024 // Class for 128 bit register v29
1025 reg_class v29_reg(
1026 V29, V29_H
1027 );
1028
1029 // Class for 128 bit register v30
1030 reg_class v30_reg(
1031 V30, V30_H
1032 );
1033
1034 // Class for 128 bit register v31
1035 reg_class v31_reg(
1036 V31, V31_H
1037 );
1038
1039 // Class for all SVE predicate registers.
1040 reg_class pr_reg (
1041 P0,
1042 P1,
1043 P2,
1044 P3,
1045 P4,
1046 P5,
1047 P6,
1048 // P7, non-allocatable, preserved with all elements preset to TRUE.
1049 P8,
1050 P9,
1051 P10,
1052 P11,
1053 P12,
1054 P13,
1055 P14,
1056 P15
1057 );
1058
1059 // Class for SVE governing predicate registers, which are used
1060 // to determine the active elements of a predicated instruction.
1061 reg_class gov_pr (
1062 P0,
1063 P1,
1064 P2,
1065 P3,
1066 P4,
1067 P5,
1068 P6,
1069 // P7, non-allocatable, preserved with all elements preset to TRUE.
1070 );
1071
1072 reg_class p0_reg(P0);
1073 reg_class p1_reg(P1);
1074
1075 // Singleton class for condition codes
1076 reg_class int_flags(RFLAGS);
1077
1078 %}
1079
1080 //----------DEFINITION BLOCK---------------------------------------------------
1081 // Define name --> value mappings to inform the ADLC of an integer valued name
1082 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1083 // Format:
1084 // int_def <name> ( <int_value>, <expression>);
1085 // Generated Code in ad_<arch>.hpp
1086 // #define <name> (<expression>)
1087 // // value == <int_value>
1088 // Generated code in ad_<arch>.cpp adlc_verification()
1089 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1090 //
1091
1092 // we follow the ppc-aix port in using a simple cost model which ranks
1093 // register operations as cheap, memory ops as more expensive and
1094 // branches as most expensive. the first two have a low as well as a
1095 // normal cost. huge cost appears to be a way of saying don't do
1096 // something
1097
1098 definitions %{
1099 // The default cost (of a register move instruction).
1100 int_def INSN_COST ( 100, 100);
1101 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1102 int_def CALL_COST ( 200, 2 * INSN_COST);
1103 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1104 %}
1105
1106
1107 //----------SOURCE BLOCK-------------------------------------------------------
1108 // This is a block of C++ code which provides values, functions, and
1109 // definitions necessary in the rest of the architecture description
1110
1111 source_hpp %{
1112
1113 #include "asm/macroAssembler.hpp"
1114 #include "gc/shared/barrierSetAssembler.hpp"
1115 #include "gc/shared/cardTable.hpp"
1116 #include "gc/shared/cardTableBarrierSet.hpp"
1117 #include "gc/shared/collectedHeap.hpp"
1118 #include "opto/addnode.hpp"
1119 #include "opto/convertnode.hpp"
1120 #include "runtime/objectMonitor.hpp"
1121
1122 extern RegMask _ANY_REG32_mask;
1123 extern RegMask _ANY_REG_mask;
1124 extern RegMask _PTR_REG_mask;
1125 extern RegMask _NO_SPECIAL_REG32_mask;
1126 extern RegMask _NO_SPECIAL_REG_mask;
1127 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1128
1129 class CallStubImpl {
1130
1131 //--------------------------------------------------------------
1132 //---< Used for optimization in Compile::shorten_branches >---
1133 //--------------------------------------------------------------
1134
1135 public:
1136 // Size of call trampoline stub.
1137 static uint size_call_trampoline() {
1138 return 0; // no call trampolines on this platform
1139 }
1140
1141 // number of relocations needed by a call trampoline stub
1142 static uint reloc_call_trampoline() {
1143 return 0; // no call trampolines on this platform
1144 }
1145 };
1146
1147 class HandlerImpl {
1148
1149 public:
1150
1151 static int emit_exception_handler(CodeBuffer &cbuf);
1152 static int emit_deopt_handler(CodeBuffer& cbuf);
1153
1154 static uint size_exception_handler() {
1155 return MacroAssembler::far_codestub_branch_size();
1156 }
1157
1158 static uint size_deopt_handler() {
1159 // count one adr and one far branch instruction
1160 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1161 }
1162 };
1163
1164 class Node::PD {
1165 public:
1166 enum NodeFlags {
1167 _last_flag = Node::_last_flag
1168 };
1169 };
1170
1171 bool is_CAS(int opcode, bool maybe_volatile);
1172
1173 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1174
1175 bool unnecessary_acquire(const Node *barrier);
1176 bool needs_acquiring_load(const Node *load);
1177
1178 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1179
1180 bool unnecessary_release(const Node *barrier);
1181 bool unnecessary_volatile(const Node *barrier);
1182 bool needs_releasing_store(const Node *store);
1183
1184 // predicate controlling translation of CompareAndSwapX
1185 bool needs_acquiring_load_exclusive(const Node *load);
1186
1187 // predicate controlling addressing modes
1188 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1189
1190 // Convert BootTest condition to Assembler condition.
1191 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1192 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1193 %}
1194
1195 source %{
1196
1197 // Derived RegMask with conditionally allocatable registers
1198
1199 void PhaseOutput::pd_perform_mach_node_analysis() {
1200 }
1201
1202 int MachNode::pd_alignment_required() const {
1203 return 1;
1204 }
1205
1206 int MachNode::compute_padding(int current_offset) const {
1207 return 0;
1208 }
1209
1210 RegMask _ANY_REG32_mask;
1211 RegMask _ANY_REG_mask;
1212 RegMask _PTR_REG_mask;
1213 RegMask _NO_SPECIAL_REG32_mask;
1214 RegMask _NO_SPECIAL_REG_mask;
1215 RegMask _NO_SPECIAL_PTR_REG_mask;
1216
1217 void reg_mask_init() {
1218 // We derive below RegMask(s) from the ones which are auto-generated from
1219 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1220 // registers conditionally reserved.
1221
1222 _ANY_REG32_mask = _ALL_REG32_mask;
1223 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1224
1225 _ANY_REG_mask = _ALL_REG_mask;
1226
1227 _PTR_REG_mask = _ALL_REG_mask;
1228
1229 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1230 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1231
1232 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1236 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1237
1238 // r27 is not allocatable when compressed oops is on and heapbase is not
1239 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1240 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1241 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1242 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1243 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1244 }
1245
1246 // r29 is not allocatable when PreserveFramePointer is on
1247 if (PreserveFramePointer) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1251 }
1252 }
1253
1254 // Optimizaton of volatile gets and puts
1255 // -------------------------------------
1256 //
1257 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1258 // use to implement volatile reads and writes. For a volatile read
1259 // we simply need
1260 //
1261 // ldar<x>
1262 //
1263 // and for a volatile write we need
1264 //
1265 // stlr<x>
1266 //
1267 // Alternatively, we can implement them by pairing a normal
1268 // load/store with a memory barrier. For a volatile read we need
1269 //
1270 // ldr<x>
1271 // dmb ishld
1272 //
1273 // for a volatile write
1274 //
1275 // dmb ish
1276 // str<x>
1277 // dmb ish
1278 //
1279 // We can also use ldaxr and stlxr to implement compare and swap CAS
1280 // sequences. These are normally translated to an instruction
1281 // sequence like the following
1282 //
1283 // dmb ish
1284 // retry:
1285 // ldxr<x> rval raddr
1286 // cmp rval rold
1287 // b.ne done
1288 // stlxr<x> rval, rnew, rold
1289 // cbnz rval retry
1290 // done:
1291 // cset r0, eq
1292 // dmb ishld
1293 //
1294 // Note that the exclusive store is already using an stlxr
1295 // instruction. That is required to ensure visibility to other
1296 // threads of the exclusive write (assuming it succeeds) before that
1297 // of any subsequent writes.
1298 //
1299 // The following instruction sequence is an improvement on the above
1300 //
1301 // retry:
1302 // ldaxr<x> rval raddr
1303 // cmp rval rold
1304 // b.ne done
1305 // stlxr<x> rval, rnew, rold
1306 // cbnz rval retry
1307 // done:
1308 // cset r0, eq
1309 //
1310 // We don't need the leading dmb ish since the stlxr guarantees
1311 // visibility of prior writes in the case that the swap is
1312 // successful. Crucially we don't have to worry about the case where
1313 // the swap is not successful since no valid program should be
1314 // relying on visibility of prior changes by the attempting thread
1315 // in the case where the CAS fails.
1316 //
1317 // Similarly, we don't need the trailing dmb ishld if we substitute
1318 // an ldaxr instruction since that will provide all the guarantees we
1319 // require regarding observation of changes made by other threads
1320 // before any change to the CAS address observed by the load.
1321 //
1322 // In order to generate the desired instruction sequence we need to
1323 // be able to identify specific 'signature' ideal graph node
1324 // sequences which i) occur as a translation of a volatile reads or
1325 // writes or CAS operations and ii) do not occur through any other
1326 // translation or graph transformation. We can then provide
1327 // alternative aldc matching rules which translate these node
1328 // sequences to the desired machine code sequences. Selection of the
1329 // alternative rules can be implemented by predicates which identify
1330 // the relevant node sequences.
1331 //
1332 // The ideal graph generator translates a volatile read to the node
1333 // sequence
1334 //
1335 // LoadX[mo_acquire]
1336 // MemBarAcquire
1337 //
1338 // As a special case when using the compressed oops optimization we
1339 // may also see this variant
1340 //
1341 // LoadN[mo_acquire]
1342 // DecodeN
1343 // MemBarAcquire
1344 //
1345 // A volatile write is translated to the node sequence
1346 //
1347 // MemBarRelease
1348 // StoreX[mo_release] {CardMark}-optional
1349 // MemBarVolatile
1350 //
1351 // n.b. the above node patterns are generated with a strict
1352 // 'signature' configuration of input and output dependencies (see
1353 // the predicates below for exact details). The card mark may be as
1354 // simple as a few extra nodes or, in a few GC configurations, may
1355 // include more complex control flow between the leading and
1356 // trailing memory barriers. However, whatever the card mark
1357 // configuration these signatures are unique to translated volatile
1358 // reads/stores -- they will not appear as a result of any other
1359 // bytecode translation or inlining nor as a consequence of
1360 // optimizing transforms.
1361 //
1362 // We also want to catch inlined unsafe volatile gets and puts and
1363 // be able to implement them using either ldar<x>/stlr<x> or some
1364 // combination of ldr<x>/stlr<x> and dmb instructions.
1365 //
1366 // Inlined unsafe volatiles puts manifest as a minor variant of the
1367 // normal volatile put node sequence containing an extra cpuorder
1368 // membar
1369 //
1370 // MemBarRelease
1371 // MemBarCPUOrder
1372 // StoreX[mo_release] {CardMark}-optional
1373 // MemBarCPUOrder
1374 // MemBarVolatile
1375 //
1376 // n.b. as an aside, a cpuorder membar is not itself subject to
1377 // matching and translation by adlc rules. However, the rule
1378 // predicates need to detect its presence in order to correctly
1379 // select the desired adlc rules.
1380 //
1381 // Inlined unsafe volatile gets manifest as a slightly different
1382 // node sequence to a normal volatile get because of the
1383 // introduction of some CPUOrder memory barriers to bracket the
1384 // Load. However, but the same basic skeleton of a LoadX feeding a
1385 // MemBarAcquire, possibly through an optional DecodeN, is still
1386 // present
1387 //
1388 // MemBarCPUOrder
1389 // || \\
1390 // MemBarCPUOrder LoadX[mo_acquire]
1391 // || |
1392 // || {DecodeN} optional
1393 // || /
1394 // MemBarAcquire
1395 //
1396 // In this case the acquire membar does not directly depend on the
1397 // load. However, we can be sure that the load is generated from an
1398 // inlined unsafe volatile get if we see it dependent on this unique
1399 // sequence of membar nodes. Similarly, given an acquire membar we
1400 // can know that it was added because of an inlined unsafe volatile
1401 // get if it is fed and feeds a cpuorder membar and if its feed
1402 // membar also feeds an acquiring load.
1403 //
1404 // Finally an inlined (Unsafe) CAS operation is translated to the
1405 // following ideal graph
1406 //
1407 // MemBarRelease
1408 // MemBarCPUOrder
1409 // CompareAndSwapX {CardMark}-optional
1410 // MemBarCPUOrder
1411 // MemBarAcquire
1412 //
1413 // So, where we can identify these volatile read and write
1414 // signatures we can choose to plant either of the above two code
1415 // sequences. For a volatile read we can simply plant a normal
1416 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1417 // also choose to inhibit translation of the MemBarAcquire and
1418 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1419 //
1420 // When we recognise a volatile store signature we can choose to
1421 // plant at a dmb ish as a translation for the MemBarRelease, a
1422 // normal str<x> and then a dmb ish for the MemBarVolatile.
1423 // Alternatively, we can inhibit translation of the MemBarRelease
1424 // and MemBarVolatile and instead plant a simple stlr<x>
1425 // instruction.
1426 //
1427 // when we recognise a CAS signature we can choose to plant a dmb
1428 // ish as a translation for the MemBarRelease, the conventional
1429 // macro-instruction sequence for the CompareAndSwap node (which
1430 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1431 // Alternatively, we can elide generation of the dmb instructions
1432 // and plant the alternative CompareAndSwap macro-instruction
1433 // sequence (which uses ldaxr<x>).
1434 //
1435 // Of course, the above only applies when we see these signature
1436 // configurations. We still want to plant dmb instructions in any
1437 // other cases where we may see a MemBarAcquire, MemBarRelease or
1438 // MemBarVolatile. For example, at the end of a constructor which
1439 // writes final/volatile fields we will see a MemBarRelease
1440 // instruction and this needs a 'dmb ish' lest we risk the
1441 // constructed object being visible without making the
1442 // final/volatile field writes visible.
1443 //
1444 // n.b. the translation rules below which rely on detection of the
1445 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1446 // If we see anything other than the signature configurations we
1447 // always just translate the loads and stores to ldr<x> and str<x>
1448 // and translate acquire, release and volatile membars to the
1449 // relevant dmb instructions.
1450 //
1451
1452 // is_CAS(int opcode, bool maybe_volatile)
1453 //
1454 // return true if opcode is one of the possible CompareAndSwapX
1455 // values otherwise false.
1456
1457 bool is_CAS(int opcode, bool maybe_volatile)
1458 {
1459 switch(opcode) {
1460 // We handle these
1461 case Op_CompareAndSwapI:
1462 case Op_CompareAndSwapL:
1463 case Op_CompareAndSwapP:
1464 case Op_CompareAndSwapN:
1465 case Op_ShenandoahCompareAndSwapP:
1466 case Op_ShenandoahCompareAndSwapN:
1467 case Op_CompareAndSwapB:
1468 case Op_CompareAndSwapS:
1469 case Op_GetAndSetI:
1470 case Op_GetAndSetL:
1471 case Op_GetAndSetP:
1472 case Op_GetAndSetN:
1473 case Op_GetAndAddI:
1474 case Op_GetAndAddL:
1475 return true;
1476 case Op_CompareAndExchangeI:
1477 case Op_CompareAndExchangeN:
1478 case Op_CompareAndExchangeB:
1479 case Op_CompareAndExchangeS:
1480 case Op_CompareAndExchangeL:
1481 case Op_CompareAndExchangeP:
1482 case Op_WeakCompareAndSwapB:
1483 case Op_WeakCompareAndSwapS:
1484 case Op_WeakCompareAndSwapI:
1485 case Op_WeakCompareAndSwapL:
1486 case Op_WeakCompareAndSwapP:
1487 case Op_WeakCompareAndSwapN:
1488 case Op_ShenandoahWeakCompareAndSwapP:
1489 case Op_ShenandoahWeakCompareAndSwapN:
1490 case Op_ShenandoahCompareAndExchangeP:
1491 case Op_ShenandoahCompareAndExchangeN:
1492 return maybe_volatile;
1493 default:
1494 return false;
1495 }
1496 }
1497
1498 // helper to determine the maximum number of Phi nodes we may need to
1499 // traverse when searching from a card mark membar for the merge mem
1500 // feeding a trailing membar or vice versa
1501
1502 // predicates controlling emit of ldr<x>/ldar<x>
1503
1504 bool unnecessary_acquire(const Node *barrier)
1505 {
1506 assert(barrier->is_MemBar(), "expecting a membar");
1507
1508 MemBarNode* mb = barrier->as_MemBar();
1509
1510 if (mb->trailing_load()) {
1511 return true;
1512 }
1513
1514 if (mb->trailing_load_store()) {
1515 Node* load_store = mb->in(MemBarNode::Precedent);
1516 assert(load_store->is_LoadStore(), "unexpected graph shape");
1517 return is_CAS(load_store->Opcode(), true);
1518 }
1519
1520 return false;
1521 }
1522
1523 bool needs_acquiring_load(const Node *n)
1524 {
1525 assert(n->is_Load(), "expecting a load");
1526 LoadNode *ld = n->as_Load();
1527 return ld->is_acquire();
1528 }
1529
1530 bool unnecessary_release(const Node *n)
1531 {
1532 assert((n->is_MemBar() &&
1533 n->Opcode() == Op_MemBarRelease),
1534 "expecting a release membar");
1535
1536 MemBarNode *barrier = n->as_MemBar();
1537 if (!barrier->leading()) {
1538 return false;
1539 } else {
1540 Node* trailing = barrier->trailing_membar();
1541 MemBarNode* trailing_mb = trailing->as_MemBar();
1542 assert(trailing_mb->trailing(), "Not a trailing membar?");
1543 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1544
1545 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1546 if (mem->is_Store()) {
1547 assert(mem->as_Store()->is_release(), "");
1548 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1549 return true;
1550 } else {
1551 assert(mem->is_LoadStore(), "");
1552 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1553 return is_CAS(mem->Opcode(), true);
1554 }
1555 }
1556 return false;
1557 }
1558
1559 bool unnecessary_volatile(const Node *n)
1560 {
1561 // assert n->is_MemBar();
1562 MemBarNode *mbvol = n->as_MemBar();
1563
1564 bool release = mbvol->trailing_store();
1565 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1566 #ifdef ASSERT
1567 if (release) {
1568 Node* leading = mbvol->leading_membar();
1569 assert(leading->Opcode() == Op_MemBarRelease, "");
1570 assert(leading->as_MemBar()->leading_store(), "");
1571 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1572 }
1573 #endif
1574
1575 return release;
1576 }
1577
1578 // predicates controlling emit of str<x>/stlr<x>
1579
1580 bool needs_releasing_store(const Node *n)
1581 {
1582 // assert n->is_Store();
1583 StoreNode *st = n->as_Store();
1584 return st->trailing_membar() != NULL;
1585 }
1586
1587 // predicate controlling translation of CAS
1588 //
1589 // returns true if CAS needs to use an acquiring load otherwise false
1590
1591 bool needs_acquiring_load_exclusive(const Node *n)
1592 {
1593 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1594 LoadStoreNode* ldst = n->as_LoadStore();
1595 if (is_CAS(n->Opcode(), false)) {
1596 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1597 } else {
1598 return ldst->trailing_membar() != NULL;
1599 }
1600
1601 // so we can just return true here
1602 return true;
1603 }
1604
1605 #define __ _masm.
1606
1607 // advance declarations for helper functions to convert register
1608 // indices to register objects
1609
1610 // the ad file has to provide implementations of certain methods
1611 // expected by the generic code
1612 //
1613 // REQUIRED FUNCTIONALITY
1614
1615 //=============================================================================
1616
1617 // !!!!! Special hack to get all types of calls to specify the byte offset
1618 // from the start of the call to the point where the return address
1619 // will point.
1620
1621 int MachCallStaticJavaNode::ret_addr_offset()
1622 {
1623 // call should be a simple bl
1624 int off = 4;
1625 return off;
1626 }
1627
1628 int MachCallDynamicJavaNode::ret_addr_offset()
1629 {
1630 return 16; // movz, movk, movk, bl
1631 }
1632
1633 int MachCallRuntimeNode::ret_addr_offset() {
1634 // for generated stubs the call will be
1635 // bl(addr)
1636 // or with far branches
1637 // bl(trampoline_stub)
1638 // for real runtime callouts it will be six instructions
1639 // see aarch64_enc_java_to_runtime
1640 // adr(rscratch2, retaddr)
1641 // lea(rscratch1, RuntimeAddress(addr)
1642 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1643 // blr(rscratch1)
1644 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1645 if (cb) {
1646 return 1 * NativeInstruction::instruction_size;
1647 } else {
1648 return 6 * NativeInstruction::instruction_size;
1649 }
1650 }
1651
1652 //=============================================================================
1653
1654 #ifndef PRODUCT
1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1656 st->print("BREAKPOINT");
1657 }
1658 #endif
1659
1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1661 C2_MacroAssembler _masm(&cbuf);
1662 __ brk(0);
1663 }
1664
1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1666 return MachNode::size(ra_);
1667 }
1668
1669 //=============================================================================
1670
1671 #ifndef PRODUCT
1672 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1673 st->print("nop \t# %d bytes pad for loops and calls", _count);
1674 }
1675 #endif
1676
1677 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1678 C2_MacroAssembler _masm(&cbuf);
1679 for (int i = 0; i < _count; i++) {
1680 __ nop();
1681 }
1682 }
1683
1684 uint MachNopNode::size(PhaseRegAlloc*) const {
1685 return _count * NativeInstruction::instruction_size;
1686 }
1687
1688 //=============================================================================
1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1690
1691 int ConstantTable::calculate_table_base_offset() const {
1692 return 0; // absolute addressing, no offset
1693 }
1694
1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1697 ShouldNotReachHere();
1698 }
1699
1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1701 // Empty encoding
1702 }
1703
1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1705 return 0;
1706 }
1707
1708 #ifndef PRODUCT
1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1710 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1711 }
1712 #endif
1713
1714 #ifndef PRODUCT
1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1716 Compile* C = ra_->C;
1717
1718 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1719
1720 if (C->output()->need_stack_bang(framesize))
1721 st->print("# stack bang size=%d\n\t", framesize);
1722
1723 if (VM_Version::use_rop_protection()) {
1724 st->print("ldr zr, [lr]\n\t");
1725 st->print("pacia lr, rfp\n\t");
1726 }
1727 if (framesize < ((1 << 9) + 2 * wordSize)) {
1728 st->print("sub sp, sp, #%d\n\t", framesize);
1729 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1730 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1731 } else {
1732 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1733 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1734 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1735 st->print("sub sp, sp, rscratch1");
1736 }
1737 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1738 st->print("\n\t");
1739 st->print("ldr rscratch1, [guard]\n\t");
1740 st->print("dmb ishld\n\t");
1741 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1742 st->print("cmp rscratch1, rscratch2\n\t");
1743 st->print("b.eq skip");
1744 st->print("\n\t");
1745 st->print("blr #nmethod_entry_barrier_stub\n\t");
1746 st->print("b skip\n\t");
1747 st->print("guard: int\n\t");
1748 st->print("\n\t");
1749 st->print("skip:\n\t");
1750 }
1751 }
1752 #endif
1753
1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1755 Compile* C = ra_->C;
1756 C2_MacroAssembler _masm(&cbuf);
1757
1758 // n.b. frame size includes space for return pc and rfp
1759 const int framesize = C->output()->frame_size_in_bytes();
1760
1761 // insert a nop at the start of the prolog so we can patch in a
1762 // branch if we need to invalidate the method later
1763 __ nop();
1764
1765 if (C->clinit_barrier_on_entry()) {
1766 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1767
1768 Label L_skip_barrier;
1769
1770 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1771 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1772 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1773 __ bind(L_skip_barrier);
1774 }
1775
1776 if (C->max_vector_size() > 0) {
1777 __ reinitialize_ptrue();
1778 }
1779
1780 int bangsize = C->output()->bang_size_in_bytes();
1781 if (C->output()->need_stack_bang(bangsize))
1782 __ generate_stack_overflow_check(bangsize);
1783
1784 __ build_frame(framesize);
1785
1786 if (C->stub_function() == NULL) {
1787 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1788 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1789 // Dummy labels for just measuring the code size
1790 Label dummy_slow_path;
1791 Label dummy_continuation;
1792 Label dummy_guard;
1793 Label* slow_path = &dummy_slow_path;
1794 Label* continuation = &dummy_continuation;
1795 Label* guard = &dummy_guard;
1796 if (!Compile::current()->output()->in_scratch_emit_size()) {
1797 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1798 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1799 Compile::current()->output()->add_stub(stub);
1800 slow_path = &stub->entry();
1801 continuation = &stub->continuation();
1802 guard = &stub->guard();
1803 }
1804 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1805 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1806 }
1807 }
1808
1809 if (VerifyStackAtCalls) {
1810 Unimplemented();
1811 }
1812
1813 C->output()->set_frame_complete(cbuf.insts_size());
1814
1815 if (C->has_mach_constant_base_node()) {
1816 // NOTE: We set the table base offset here because users might be
1817 // emitted before MachConstantBaseNode.
1818 ConstantTable& constant_table = C->output()->constant_table();
1819 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1820 }
1821 }
1822
1823 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1824 {
1825 return MachNode::size(ra_); // too many variables; just compute it
1826 // the hard way
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autia lr, rfp\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 C2_MacroAssembler _masm(&cbuf);
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 __ remove_frame(framesize);
1873
1874 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1875 __ reserved_stack_check();
1876 }
1877
1878 if (do_polling() && C->is_method_compilation()) {
1879 Label dummy_label;
1880 Label* code_stub = &dummy_label;
1881 if (!C->output()->in_scratch_emit_size()) {
1882 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1883 C->output()->add_stub(stub);
1884 code_stub = &stub->entry();
1885 }
1886 __ relocate(relocInfo::poll_return_type);
1887 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1888 }
1889 }
1890
1891 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1892 // Variable size. Determine dynamically.
1893 return MachNode::size(ra_);
1894 }
1895
1896 int MachEpilogNode::reloc() const {
1897 // Return number of relocatable values contained in this instruction.
1898 return 1; // 1 for polling page.
1899 }
1900
1901 const Pipeline * MachEpilogNode::pipeline() const {
1902 return MachNode::pipeline_class();
1903 }
1904
1905 //=============================================================================
1906
1907 // Figure out which register class each belongs in: rc_int, rc_float or
1908 // rc_stack.
1909 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1910
1911 static enum RC rc_class(OptoReg::Name reg) {
1912
1913 if (reg == OptoReg::Bad) {
1914 return rc_bad;
1915 }
1916
1917 // we have 32 int registers * 2 halves
1918 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1919
1920 if (reg < slots_of_int_registers) {
1921 return rc_int;
1922 }
1923
1924 // we have 32 float register * 8 halves
1925 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1926 if (reg < slots_of_int_registers + slots_of_float_registers) {
1927 return rc_float;
1928 }
1929
1930 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1931 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1932 return rc_predicate;
1933 }
1934
1935 // Between predicate regs & stack is the flags.
1936 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1937
1938 return rc_stack;
1939 }
1940
1941 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1942 Compile* C = ra_->C;
1943
1944 // Get registers to move.
1945 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1946 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1947 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1948 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1949
1950 enum RC src_hi_rc = rc_class(src_hi);
1951 enum RC src_lo_rc = rc_class(src_lo);
1952 enum RC dst_hi_rc = rc_class(dst_hi);
1953 enum RC dst_lo_rc = rc_class(dst_lo);
1954
1955 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1956
1957 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1958 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1959 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1960 "expected aligned-adjacent pairs");
1961 }
1962
1963 if (src_lo == dst_lo && src_hi == dst_hi) {
1964 return 0; // Self copy, no move.
1965 }
1966
1967 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1968 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1969 int src_offset = ra_->reg2offset(src_lo);
1970 int dst_offset = ra_->reg2offset(dst_lo);
1971
1972 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1973 uint ireg = ideal_reg();
1974 if (ireg == Op_VecA && cbuf) {
1975 C2_MacroAssembler _masm(cbuf);
1976 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1977 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1978 // stack->stack
1979 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1980 sve_vector_reg_size_in_bytes);
1981 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1982 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1983 sve_vector_reg_size_in_bytes);
1984 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1985 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1986 sve_vector_reg_size_in_bytes);
1987 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1988 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1989 as_FloatRegister(Matcher::_regEncode[src_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]));
1991 } else {
1992 ShouldNotReachHere();
1993 }
1994 } else if (cbuf) {
1995 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1996 C2_MacroAssembler _masm(cbuf);
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (cbuf) {
2024 C2_MacroAssembler _masm(cbuf);
2025 switch (src_lo_rc) {
2026 case rc_int:
2027 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2028 if (is64) {
2029 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2030 as_Register(Matcher::_regEncode[src_lo]));
2031 } else {
2032 C2_MacroAssembler _masm(cbuf);
2033 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2034 as_Register(Matcher::_regEncode[src_lo]));
2035 }
2036 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2037 if (is64) {
2038 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 as_Register(Matcher::_regEncode[src_lo]));
2040 } else {
2041 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2042 as_Register(Matcher::_regEncode[src_lo]));
2043 }
2044 } else { // gpr --> stack spill
2045 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2046 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2047 }
2048 break;
2049 case rc_float:
2050 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2051 if (is64) {
2052 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 } else {
2055 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_FloatRegister(Matcher::_regEncode[src_lo]));
2057 }
2058 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2059 if (is64) {
2060 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2061 as_FloatRegister(Matcher::_regEncode[src_lo]));
2062 } else {
2063 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_FloatRegister(Matcher::_regEncode[src_lo]));
2065 }
2066 } else { // fpr --> stack spill
2067 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2068 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2069 is64 ? __ D : __ S, dst_offset);
2070 }
2071 break;
2072 case rc_stack:
2073 if (dst_lo_rc == rc_int) { // stack --> gpr load
2074 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2075 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2076 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2077 is64 ? __ D : __ S, src_offset);
2078 } else if (dst_lo_rc == rc_predicate) {
2079 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2080 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2081 } else { // stack --> stack copy
2082 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2083 if (ideal_reg() == Op_RegVectMask) {
2084 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2085 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2086 } else {
2087 __ unspill(rscratch1, is64, src_offset);
2088 __ spill(rscratch1, is64, dst_offset);
2089 }
2090 }
2091 break;
2092 case rc_predicate:
2093 if (dst_lo_rc == rc_predicate) {
2094 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2095 } else if (dst_lo_rc == rc_stack) {
2096 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2097 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2098 } else {
2099 assert(false, "bad src and dst rc_class combination.");
2100 ShouldNotReachHere();
2101 }
2102 break;
2103 default:
2104 assert(false, "bad rc_class for spill");
2105 ShouldNotReachHere();
2106 }
2107 }
2108
2109 if (st) {
2110 st->print("spill ");
2111 if (src_lo_rc == rc_stack) {
2112 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2113 } else {
2114 st->print("%s -> ", Matcher::regName[src_lo]);
2115 }
2116 if (dst_lo_rc == rc_stack) {
2117 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2118 } else {
2119 st->print("%s", Matcher::regName[dst_lo]);
2120 }
2121 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2122 int vsize = 0;
2123 switch (ideal_reg()) {
2124 case Op_VecD:
2125 vsize = 64;
2126 break;
2127 case Op_VecX:
2128 vsize = 128;
2129 break;
2130 case Op_VecA:
2131 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2132 break;
2133 default:
2134 assert(false, "bad register type for spill");
2135 ShouldNotReachHere();
2136 }
2137 st->print("\t# vector spill size = %d", vsize);
2138 } else if (ideal_reg() == Op_RegVectMask) {
2139 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2140 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2141 st->print("\t# predicate spill size = %d", vsize);
2142 } else {
2143 st->print("\t# spill size = %d", is64 ? 64 : 32);
2144 }
2145 }
2146
2147 return 0;
2148
2149 }
2150
2151 #ifndef PRODUCT
2152 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2153 if (!ra_)
2154 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2155 else
2156 implementation(NULL, ra_, false, st);
2157 }
2158 #endif
2159
2160 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2161 implementation(&cbuf, ra_, false, NULL);
2162 }
2163
2164 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2165 return MachNode::size(ra_);
2166 }
2167
2168 //=============================================================================
2169
2170 #ifndef PRODUCT
2171 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2172 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2173 int reg = ra_->get_reg_first(this);
2174 st->print("add %s, rsp, #%d]\t# box lock",
2175 Matcher::regName[reg], offset);
2176 }
2177 #endif
2178
2179 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2180 C2_MacroAssembler _masm(&cbuf);
2181
2182 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2183 int reg = ra_->get_encode(this);
2184
2185 // This add will handle any 24-bit signed offset. 24 bits allows an
2186 // 8 megabyte stack frame.
2187 __ add(as_Register(reg), sp, offset);
2188 }
2189
2190 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2191 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2192 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2193
2194 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2195 return NativeInstruction::instruction_size;
2196 } else {
2197 return 2 * NativeInstruction::instruction_size;
2198 }
2199 }
2200
2201 //=============================================================================
2202
2203 #ifndef PRODUCT
2204 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2205 {
2206 st->print_cr("# MachUEPNode");
2207 if (UseCompressedClassPointers) {
2208 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 if (CompressedKlassPointers::shift() != 0) {
2210 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2211 }
2212 } else {
2213 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2214 }
2215 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2216 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2217 }
2218 #endif
2219
2220 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2221 {
2222 // This is the unverified entry point.
2223 C2_MacroAssembler _masm(&cbuf);
2224
2225 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2226 Label skip;
2227 // TODO
2228 // can we avoid this skip and still use a reloc?
2229 __ br(Assembler::EQ, skip);
2230 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2231 __ bind(skip);
2232 }
2233
2234 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2235 {
2236 return MachNode::size(ra_);
2237 }
2238
2239 // REQUIRED EMIT CODE
2240
2241 //=============================================================================
2242
2243 // Emit exception handler code.
2244 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2245 {
2246 // mov rscratch1 #exception_blob_entry_point
2247 // br rscratch1
2248 // Note that the code buffer's insts_mark is always relative to insts.
2249 // That's why we must use the macroassembler to generate a handler.
2250 C2_MacroAssembler _masm(&cbuf);
2251 address base = __ start_a_stub(size_exception_handler());
2252 if (base == NULL) {
2253 ciEnv::current()->record_failure("CodeCache is full");
2254 return 0; // CodeBuffer::expand failed
2255 }
2256 int offset = __ offset();
2257 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2258 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2259 __ end_a_stub();
2260 return offset;
2261 }
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 C2_MacroAssembler _masm(&cbuf);
2269 address base = __ start_a_stub(size_deopt_handler());
2270 if (base == NULL) {
2271 ciEnv::current()->record_failure("CodeCache is full");
2272 return 0; // CodeBuffer::expand failed
2273 }
2274 int offset = __ offset();
2275
2276 __ adr(lr, __ pc());
2277 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 __ end_a_stub();
2281 return offset;
2282 }
2283
2284 // REQUIRED MATCHER CODE
2285
2286 //=============================================================================
2287
2288 const bool Matcher::match_rule_supported(int opcode) {
2289 if (!has_match_rule(opcode))
2290 return false;
2291
2292 bool ret_value = true;
2293 switch (opcode) {
2294 case Op_OnSpinWait:
2295 return VM_Version::supports_on_spin_wait();
2296 case Op_CacheWB:
2297 case Op_CacheWBPreSync:
2298 case Op_CacheWBPostSync:
2299 if (!VM_Version::supports_data_cache_line_flush()) {
2300 ret_value = false;
2301 }
2302 break;
2303 case Op_ExpandBits:
2304 case Op_CompressBits:
2305 if (!(UseSVE > 1 && VM_Version::supports_svebitperm())) {
2306 ret_value = false;
2307 }
2308 break;
2309 }
2310
2311 return ret_value; // Per default match rules are supported.
2312 }
2313
2314 const RegMask* Matcher::predicate_reg_mask(void) {
2315 return &_PR_REG_mask;
2316 }
2317
2318 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2319 return new TypeVectMask(elemTy, length);
2320 }
2321
2322 // Vector calling convention not yet implemented.
2323 const bool Matcher::supports_vector_calling_convention(void) {
2324 return false;
2325 }
2326
2327 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2328 Unimplemented();
2329 return OptoRegPair(0, 0);
2330 }
2331
2332 // Is this branch offset short enough that a short branch can be used?
2333 //
2334 // NOTE: If the platform does not provide any short branch variants, then
2335 // this method should return false for offset 0.
2336 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2337 // The passed offset is relative to address of the branch.
2338
2339 return (-32768 <= offset && offset < 32768);
2340 }
2341
2342 // Vector width in bytes.
2343 const int Matcher::vector_width_in_bytes(BasicType bt) {
2344 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2345 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2346 // Minimum 2 values in vector
2347 if (size < 2*type2aelembytes(bt)) size = 0;
2348 // But never < 4
2349 if (size < 4) size = 0;
2350 return size;
2351 }
2352
2353 // Limits on vector size (number of elements) loaded into vector.
2354 const int Matcher::max_vector_size(const BasicType bt) {
2355 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2356 }
2357
2358 const int Matcher::min_vector_size(const BasicType bt) {
2359 int max_size = max_vector_size(bt);
2360 // Limit the min vector size to 8 bytes.
2361 int size = 8 / type2aelembytes(bt);
2362 if (bt == T_BYTE) {
2363 // To support vector api shuffle/rearrange.
2364 size = 4;
2365 } else if (bt == T_BOOLEAN) {
2366 // To support vector api load/store mask.
2367 size = 2;
2368 }
2369 if (size < 2) size = 2;
2370 return MIN2(size, max_size);
2371 }
2372
2373 const int Matcher::superword_max_vector_size(const BasicType bt) {
2374 return Matcher::max_vector_size(bt);
2375 }
2376
2377 // Actual max scalable vector register length.
2378 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2379 return Matcher::max_vector_size(bt);
2380 }
2381
2382 // Vector ideal reg.
2383 const uint Matcher::vector_ideal_reg(int len) {
2384 if (UseSVE > 0 && 16 < len && len <= 256) {
2385 return Op_VecA;
2386 }
2387 switch(len) {
2388 // For 16-bit/32-bit mask vector, reuse VecD.
2389 case 2:
2390 case 4:
2391 case 8: return Op_VecD;
2392 case 16: return Op_VecX;
2393 }
2394 ShouldNotReachHere();
2395 return 0;
2396 }
2397
2398 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2399 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2400 switch (ideal_reg) {
2401 case Op_VecA: return new vecAOper();
2402 case Op_VecD: return new vecDOper();
2403 case Op_VecX: return new vecXOper();
2404 }
2405 ShouldNotReachHere();
2406 return NULL;
2407 }
2408
2409 bool Matcher::is_reg2reg_move(MachNode* m) {
2410 return false;
2411 }
2412
2413 bool Matcher::is_generic_vector(MachOper* opnd) {
2414 return opnd->opcode() == VREG;
2415 }
2416
2417 // Return whether or not this register is ever used as an argument.
2418 // This function is used on startup to build the trampoline stubs in
2419 // generateOptoStub. Registers not mentioned will be killed by the VM
2420 // call in the trampoline, and arguments in those registers not be
2421 // available to the callee.
2422 bool Matcher::can_be_java_arg(int reg)
2423 {
2424 return
2425 reg == R0_num || reg == R0_H_num ||
2426 reg == R1_num || reg == R1_H_num ||
2427 reg == R2_num || reg == R2_H_num ||
2428 reg == R3_num || reg == R3_H_num ||
2429 reg == R4_num || reg == R4_H_num ||
2430 reg == R5_num || reg == R5_H_num ||
2431 reg == R6_num || reg == R6_H_num ||
2432 reg == R7_num || reg == R7_H_num ||
2433 reg == V0_num || reg == V0_H_num ||
2434 reg == V1_num || reg == V1_H_num ||
2435 reg == V2_num || reg == V2_H_num ||
2436 reg == V3_num || reg == V3_H_num ||
2437 reg == V4_num || reg == V4_H_num ||
2438 reg == V5_num || reg == V5_H_num ||
2439 reg == V6_num || reg == V6_H_num ||
2440 reg == V7_num || reg == V7_H_num;
2441 }
2442
2443 bool Matcher::is_spillable_arg(int reg)
2444 {
2445 return can_be_java_arg(reg);
2446 }
2447
2448 uint Matcher::int_pressure_limit()
2449 {
2450 // JDK-8183543: When taking the number of available registers as int
2451 // register pressure threshold, the jtreg test:
2452 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2453 // failed due to C2 compilation failure with
2454 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2455 //
2456 // A derived pointer is live at CallNode and then is flagged by RA
2457 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2458 // derived pointers and lastly fail to spill after reaching maximum
2459 // number of iterations. Lowering the default pressure threshold to
2460 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2461 // a high register pressure area of the code so that split_DEF can
2462 // generate DefinitionSpillCopy for the derived pointer.
2463 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2464 if (!PreserveFramePointer) {
2465 // When PreserveFramePointer is off, frame pointer is allocatable,
2466 // but different from other SOC registers, it is excluded from
2467 // fatproj's mask because its save type is No-Save. Decrease 1 to
2468 // ensure high pressure at fatproj when PreserveFramePointer is off.
2469 // See check_pressure_at_fatproj().
2470 default_int_pressure_threshold--;
2471 }
2472 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2473 }
2474
2475 uint Matcher::float_pressure_limit()
2476 {
2477 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2478 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2479 }
2480
2481 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2482 return false;
2483 }
2484
2485 RegMask Matcher::divI_proj_mask() {
2486 ShouldNotReachHere();
2487 return RegMask();
2488 }
2489
2490 // Register for MODI projection of divmodI.
2491 RegMask Matcher::modI_proj_mask() {
2492 ShouldNotReachHere();
2493 return RegMask();
2494 }
2495
2496 // Register for DIVL projection of divmodL.
2497 RegMask Matcher::divL_proj_mask() {
2498 ShouldNotReachHere();
2499 return RegMask();
2500 }
2501
2502 // Register for MODL projection of divmodL.
2503 RegMask Matcher::modL_proj_mask() {
2504 ShouldNotReachHere();
2505 return RegMask();
2506 }
2507
2508 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2509 return FP_REG_mask();
2510 }
2511
2512 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2513 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2514 Node* u = addp->fast_out(i);
2515 if (u->is_LoadStore()) {
2516 // On AArch64, LoadStoreNodes (i.e. compare and swap
2517 // instructions) only take register indirect as an operand, so
2518 // any attempt to use an AddPNode as an input to a LoadStoreNode
2519 // must fail.
2520 return false;
2521 }
2522 if (u->is_Mem()) {
2523 int opsize = u->as_Mem()->memory_size();
2524 assert(opsize > 0, "unexpected memory operand size");
2525 if (u->as_Mem()->memory_size() != (1<<shift)) {
2526 return false;
2527 }
2528 }
2529 }
2530 return true;
2531 }
2532
2533 // Convert BootTest condition to Assembler condition.
2534 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2535 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2536 Assembler::Condition result;
2537 switch(cond) {
2538 case BoolTest::eq:
2539 result = Assembler::EQ; break;
2540 case BoolTest::ne:
2541 result = Assembler::NE; break;
2542 case BoolTest::le:
2543 result = Assembler::LE; break;
2544 case BoolTest::ge:
2545 result = Assembler::GE; break;
2546 case BoolTest::lt:
2547 result = Assembler::LT; break;
2548 case BoolTest::gt:
2549 result = Assembler::GT; break;
2550 case BoolTest::ule:
2551 result = Assembler::LS; break;
2552 case BoolTest::uge:
2553 result = Assembler::HS; break;
2554 case BoolTest::ult:
2555 result = Assembler::LO; break;
2556 case BoolTest::ugt:
2557 result = Assembler::HI; break;
2558 case BoolTest::overflow:
2559 result = Assembler::VS; break;
2560 case BoolTest::no_overflow:
2561 result = Assembler::VC; break;
2562 default:
2563 ShouldNotReachHere();
2564 return Assembler::Condition(-1);
2565 }
2566
2567 // Check conversion
2568 if (cond & BoolTest::unsigned_compare) {
2569 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2570 } else {
2571 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2572 }
2573
2574 return result;
2575 }
2576
2577 // Binary src (Replicate con)
2578 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2579 if (n == NULL || m == NULL) {
2580 return false;
2581 }
2582
2583 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2584 return false;
2585 }
2586
2587 Node* imm_node = m->in(1);
2588 if (!imm_node->is_Con()) {
2589 return false;
2590 }
2591
2592 const Type* t = imm_node->bottom_type();
2593 if (!(t->isa_int() || t->isa_long())) {
2594 return false;
2595 }
2596
2597 switch (n->Opcode()) {
2598 case Op_AndV:
2599 case Op_OrV:
2600 case Op_XorV: {
2601 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2602 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2603 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2604 }
2605 case Op_AddVB:
2606 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2607 case Op_AddVS:
2608 case Op_AddVI:
2609 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2610 case Op_AddVL:
2611 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2612 default:
2613 return false;
2614 }
2615 }
2616
2617 // (XorV src (Replicate m1))
2618 // (XorVMask src (MaskAll m1))
2619 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2620 if (n != NULL && m != NULL) {
2621 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2622 VectorNode::is_all_ones_vector(m);
2623 }
2624 return false;
2625 }
2626
2627 // Should the matcher clone input 'm' of node 'n'?
2628 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2629 if (is_vshift_con_pattern(n, m) ||
2630 is_vector_bitwise_not_pattern(n, m) ||
2631 is_valid_sve_arith_imm_pattern(n, m)) {
2632 mstack.push(m, Visit);
2633 return true;
2634 }
2635 return false;
2636 }
2637
2638 // Should the Matcher clone shifts on addressing modes, expecting them
2639 // to be subsumed into complex addressing expressions or compute them
2640 // into registers?
2641 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2642 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2643 return true;
2644 }
2645
2646 Node *off = m->in(AddPNode::Offset);
2647 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2648 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2649 // Are there other uses besides address expressions?
2650 !is_visited(off)) {
2651 address_visited.set(off->_idx); // Flag as address_visited
2652 mstack.push(off->in(2), Visit);
2653 Node *conv = off->in(1);
2654 if (conv->Opcode() == Op_ConvI2L &&
2655 // Are there other uses besides address expressions?
2656 !is_visited(conv)) {
2657 address_visited.set(conv->_idx); // Flag as address_visited
2658 mstack.push(conv->in(1), Pre_Visit);
2659 } else {
2660 mstack.push(conv, Pre_Visit);
2661 }
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2664 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2665 return true;
2666 } else if (off->Opcode() == Op_ConvI2L &&
2667 // Are there other uses besides address expressions?
2668 !is_visited(off)) {
2669 address_visited.test_set(m->_idx); // Flag as address_visited
2670 address_visited.set(off->_idx); // Flag as address_visited
2671 mstack.push(off->in(1), Pre_Visit);
2672 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2673 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2674 return true;
2675 }
2676 return false;
2677 }
2678
2679 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2680 C2_MacroAssembler _masm(&cbuf); \
2681 { \
2682 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2683 guarantee(DISP == 0, "mode not permitted for volatile"); \
2684 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2685 __ INSN(REG, as_Register(BASE)); \
2686 }
2687
2688
2689 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2690 {
2691 Address::extend scale;
2692
2693 // Hooboy, this is fugly. We need a way to communicate to the
2694 // encoder that the index needs to be sign extended, so we have to
2695 // enumerate all the cases.
2696 switch (opcode) {
2697 case INDINDEXSCALEDI2L:
2698 case INDINDEXSCALEDI2LN:
2699 case INDINDEXI2L:
2700 case INDINDEXI2LN:
2701 scale = Address::sxtw(size);
2702 break;
2703 default:
2704 scale = Address::lsl(size);
2705 }
2706
2707 if (index == -1) {
2708 return Address(base, disp);
2709 } else {
2710 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2711 return Address(base, as_Register(index), scale);
2712 }
2713 }
2714
2715
2716 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2717 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2718 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2719 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2720 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2721
2722 // Used for all non-volatile memory accesses. The use of
2723 // $mem->opcode() to discover whether this pattern uses sign-extended
2724 // offsets is something of a kludge.
2725 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2726 Register reg, int opcode,
2727 Register base, int index, int scale, int disp,
2728 int size_in_memory)
2729 {
2730 Address addr = mem2address(opcode, base, index, scale, disp);
2731 if (addr.getMode() == Address::base_plus_offset) {
2732 /* If we get an out-of-range offset it is a bug in the compiler,
2733 so we assert here. */
2734 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2735 "c2 compiler bug");
2736 /* Fix up any out-of-range offsets. */
2737 assert_different_registers(rscratch1, base);
2738 assert_different_registers(rscratch1, reg);
2739 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2740 }
2741 (masm.*insn)(reg, addr);
2742 }
2743
2744 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2745 FloatRegister reg, int opcode,
2746 Register base, int index, int size, int disp,
2747 int size_in_memory)
2748 {
2749 Address::extend scale;
2750
2751 switch (opcode) {
2752 case INDINDEXSCALEDI2L:
2753 case INDINDEXSCALEDI2LN:
2754 scale = Address::sxtw(size);
2755 break;
2756 default:
2757 scale = Address::lsl(size);
2758 }
2759
2760 if (index == -1) {
2761 /* If we get an out-of-range offset it is a bug in the compiler,
2762 so we assert here. */
2763 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2764 /* Fix up any out-of-range offsets. */
2765 assert_different_registers(rscratch1, base);
2766 Address addr = Address(base, disp);
2767 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2768 (masm.*insn)(reg, addr);
2769 } else {
2770 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2771 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2772 }
2773 }
2774
2775 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2776 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2777 int opcode, Register base, int index, int size, int disp)
2778 {
2779 if (index == -1) {
2780 (masm.*insn)(reg, T, Address(base, disp));
2781 } else {
2782 assert(disp == 0, "unsupported address mode");
2783 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2784 }
2785 }
2786
2787 %}
2788
2789
2790
2791 //----------ENCODING BLOCK-----------------------------------------------------
2792 // This block specifies the encoding classes used by the compiler to
2793 // output byte streams. Encoding classes are parameterized macros
2794 // used by Machine Instruction Nodes in order to generate the bit
2795 // encoding of the instruction. Operands specify their base encoding
2796 // interface with the interface keyword. There are currently
2797 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2798 // COND_INTER. REG_INTER causes an operand to generate a function
2799 // which returns its register number when queried. CONST_INTER causes
2800 // an operand to generate a function which returns the value of the
2801 // constant when queried. MEMORY_INTER causes an operand to generate
2802 // four functions which return the Base Register, the Index Register,
2803 // the Scale Value, and the Offset Value of the operand when queried.
2804 // COND_INTER causes an operand to generate six functions which return
2805 // the encoding code (ie - encoding bits for the instruction)
2806 // associated with each basic boolean condition for a conditional
2807 // instruction.
2808 //
2809 // Instructions specify two basic values for encoding. Again, a
2810 // function is available to check if the constant displacement is an
2811 // oop. They use the ins_encode keyword to specify their encoding
2812 // classes (which must be a sequence of enc_class names, and their
2813 // parameters, specified in the encoding block), and they use the
2814 // opcode keyword to specify, in order, their primary, secondary, and
2815 // tertiary opcode. Only the opcode sections which a particular
2816 // instruction needs for encoding need to be specified.
2817 encode %{
2818 // Build emit functions for each basic byte or larger field in the
2819 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2820 // from C++ code in the enc_class source block. Emit functions will
2821 // live in the main source block for now. In future, we can
2822 // generalize this by adding a syntax that specifies the sizes of
2823 // fields in an order, so that the adlc can build the emit functions
2824 // automagically
2825
2826 // catch all for unimplemented encodings
2827 enc_class enc_unimplemented %{
2828 C2_MacroAssembler _masm(&cbuf);
2829 __ unimplemented("C2 catch all");
2830 %}
2831
2832 // BEGIN Non-volatile memory access
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2933 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2934 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2941 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2942 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2949 Register src_reg = as_Register($src$$reg);
2950 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_strb0(memory1 mem) %{
2957 C2_MacroAssembler _masm(&cbuf);
2958 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strh0(memory2 mem) %{
2973 C2_MacroAssembler _masm(&cbuf);
2974 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2981 Register src_reg = as_Register($src$$reg);
2982 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_strw0(memory4 mem) %{
2989 C2_MacroAssembler _masm(&cbuf);
2990 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2997 Register src_reg = as_Register($src$$reg);
2998 // we sometimes get asked to store the stack pointer into the
2999 // current thread -- we cannot do that directly on AArch64
3000 if (src_reg == r31_sp) {
3001 C2_MacroAssembler _masm(&cbuf);
3002 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3003 __ mov(rscratch2, sp);
3004 src_reg = rscratch2;
3005 }
3006 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_str0(memory8 mem) %{
3013 C2_MacroAssembler _masm(&cbuf);
3014 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3021 FloatRegister src_reg = as_FloatRegister($src$$reg);
3022 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3029 FloatRegister src_reg = as_FloatRegister($src$$reg);
3030 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3037 C2_MacroAssembler _masm(&cbuf);
3038 __ membar(Assembler::StoreStore);
3039 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3041 %}
3042
3043 // END Non-volatile memory access
3044
3045 // Vector loads and stores
3046 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3047 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3048 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3049 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3050 %}
3051
3052 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3053 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3054 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3055 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3056 %}
3057
3058 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3059 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3060 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3061 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3062 %}
3063
3064 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3065 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3066 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3067 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3068 %}
3069
3070 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3071 FloatRegister src_reg = as_FloatRegister($src$$reg);
3072 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3073 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3074 %}
3075
3076 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3079 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3080 %}
3081
3082 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3083 FloatRegister src_reg = as_FloatRegister($src$$reg);
3084 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3085 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3086 %}
3087
3088 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3089 FloatRegister src_reg = as_FloatRegister($src$$reg);
3090 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3091 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3092 %}
3093
3094 // volatile loads and stores
3095
3096 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3097 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrb);
3099 %}
3100
3101 enc_class aarch64_enc_stlrb0(memory mem) %{
3102 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrb);
3104 %}
3105
3106 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3107 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, stlrh);
3109 %}
3110
3111 enc_class aarch64_enc_stlrh0(memory mem) %{
3112 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, stlrh);
3114 %}
3115
3116 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3117 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, stlrw);
3119 %}
3120
3121 enc_class aarch64_enc_stlrw0(memory mem) %{
3122 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, stlrw);
3124 %}
3125
3126 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3127 Register dst_reg = as_Register($dst$$reg);
3128 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3129 rscratch1, ldarb);
3130 __ sxtbw(dst_reg, dst_reg);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtb(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3141 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, ldarb);
3143 %}
3144
3145 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3146 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, ldarb);
3148 %}
3149
3150 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3151 Register dst_reg = as_Register($dst$$reg);
3152 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, ldarh);
3154 __ sxthw(dst_reg, dst_reg);
3155 %}
3156
3157 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxth(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarh);
3167 %}
3168
3169 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3170 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarh);
3172 %}
3173
3174 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarw);
3177 %}
3178
3179 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarw);
3182 %}
3183
3184 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3185 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3186 rscratch1, ldar);
3187 %}
3188
3189 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3190 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarw);
3192 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3193 %}
3194
3195 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3196 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldar);
3198 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3199 %}
3200
3201 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3202 Register src_reg = as_Register($src$$reg);
3203 // we sometimes get asked to store the stack pointer into the
3204 // current thread -- we cannot do that directly on AArch64
3205 if (src_reg == r31_sp) {
3206 C2_MacroAssembler _masm(&cbuf);
3207 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3208 __ mov(rscratch2, sp);
3209 src_reg = rscratch2;
3210 }
3211 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 enc_class aarch64_enc_stlr0(memory mem) %{
3216 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3217 rscratch1, stlr);
3218 %}
3219
3220 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3221 {
3222 C2_MacroAssembler _masm(&cbuf);
3223 FloatRegister src_reg = as_FloatRegister($src$$reg);
3224 __ fmovs(rscratch2, src_reg);
3225 }
3226 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, stlrw);
3228 %}
3229
3230 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3231 {
3232 C2_MacroAssembler _masm(&cbuf);
3233 FloatRegister src_reg = as_FloatRegister($src$$reg);
3234 __ fmovd(rscratch2, src_reg);
3235 }
3236 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, stlr);
3238 %}
3239
3240 // synchronized read/update encodings
3241
3242 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3243 C2_MacroAssembler _masm(&cbuf);
3244 Register dst_reg = as_Register($dst$$reg);
3245 Register base = as_Register($mem$$base);
3246 int index = $mem$$index;
3247 int scale = $mem$$scale;
3248 int disp = $mem$$disp;
3249 if (index == -1) {
3250 if (disp != 0) {
3251 __ lea(rscratch1, Address(base, disp));
3252 __ ldaxr(dst_reg, rscratch1);
3253 } else {
3254 // TODO
3255 // should we ever get anything other than this case?
3256 __ ldaxr(dst_reg, base);
3257 }
3258 } else {
3259 Register index_reg = as_Register(index);
3260 if (disp == 0) {
3261 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3262 __ ldaxr(dst_reg, rscratch1);
3263 } else {
3264 __ lea(rscratch1, Address(base, disp));
3265 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3266 __ ldaxr(dst_reg, rscratch1);
3267 }
3268 }
3269 %}
3270
3271 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3272 C2_MacroAssembler _masm(&cbuf);
3273 Register src_reg = as_Register($src$$reg);
3274 Register base = as_Register($mem$$base);
3275 int index = $mem$$index;
3276 int scale = $mem$$scale;
3277 int disp = $mem$$disp;
3278 if (index == -1) {
3279 if (disp != 0) {
3280 __ lea(rscratch2, Address(base, disp));
3281 __ stlxr(rscratch1, src_reg, rscratch2);
3282 } else {
3283 // TODO
3284 // should we ever get anything other than this case?
3285 __ stlxr(rscratch1, src_reg, base);
3286 }
3287 } else {
3288 Register index_reg = as_Register(index);
3289 if (disp == 0) {
3290 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3291 __ stlxr(rscratch1, src_reg, rscratch2);
3292 } else {
3293 __ lea(rscratch2, Address(base, disp));
3294 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3295 __ stlxr(rscratch1, src_reg, rscratch2);
3296 }
3297 }
3298 __ cmpw(rscratch1, zr);
3299 %}
3300
3301 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3302 C2_MacroAssembler _masm(&cbuf);
3303 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3304 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3305 Assembler::xword, /*acquire*/ false, /*release*/ true,
3306 /*weak*/ false, noreg);
3307 %}
3308
3309 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3310 C2_MacroAssembler _masm(&cbuf);
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::word, /*acquire*/ false, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 C2_MacroAssembler _masm(&cbuf);
3319 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3320 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3321 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3322 /*weak*/ false, noreg);
3323 %}
3324
3325 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3326 C2_MacroAssembler _masm(&cbuf);
3327 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3328 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3329 Assembler::byte, /*acquire*/ false, /*release*/ true,
3330 /*weak*/ false, noreg);
3331 %}
3332
3333
3334 // The only difference between aarch64_enc_cmpxchg and
3335 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3336 // CompareAndSwap sequence to serve as a barrier on acquiring a
3337 // lock.
3338 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3339 C2_MacroAssembler _masm(&cbuf);
3340 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3341 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3342 Assembler::xword, /*acquire*/ true, /*release*/ true,
3343 /*weak*/ false, noreg);
3344 %}
3345
3346 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3347 C2_MacroAssembler _masm(&cbuf);
3348 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3349 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3350 Assembler::word, /*acquire*/ true, /*release*/ true,
3351 /*weak*/ false, noreg);
3352 %}
3353
3354 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3355 C2_MacroAssembler _masm(&cbuf);
3356 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3357 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3358 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3359 /*weak*/ false, noreg);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3363 C2_MacroAssembler _masm(&cbuf);
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::byte, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 // auxiliary used for CompareAndSwapX to set result register
3371 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3372 C2_MacroAssembler _masm(&cbuf);
3373 Register res_reg = as_Register($res$$reg);
3374 __ cset(res_reg, Assembler::EQ);
3375 %}
3376
3377 // prefetch encodings
3378
3379 enc_class aarch64_enc_prefetchw(memory mem) %{
3380 C2_MacroAssembler _masm(&cbuf);
3381 Register base = as_Register($mem$$base);
3382 int index = $mem$$index;
3383 int scale = $mem$$scale;
3384 int disp = $mem$$disp;
3385 if (index == -1) {
3386 __ prfm(Address(base, disp), PSTL1KEEP);
3387 } else {
3388 Register index_reg = as_Register(index);
3389 if (disp == 0) {
3390 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3391 } else {
3392 __ lea(rscratch1, Address(base, disp));
3393 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3394 }
3395 }
3396 %}
3397
3398 /// mov envcodings
3399
3400 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3401 C2_MacroAssembler _masm(&cbuf);
3402 uint32_t con = (uint32_t)$src$$constant;
3403 Register dst_reg = as_Register($dst$$reg);
3404 if (con == 0) {
3405 __ movw(dst_reg, zr);
3406 } else {
3407 __ movw(dst_reg, con);
3408 }
3409 %}
3410
3411 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3412 C2_MacroAssembler _masm(&cbuf);
3413 Register dst_reg = as_Register($dst$$reg);
3414 uint64_t con = (uint64_t)$src$$constant;
3415 if (con == 0) {
3416 __ mov(dst_reg, zr);
3417 } else {
3418 __ mov(dst_reg, con);
3419 }
3420 %}
3421
3422 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3423 C2_MacroAssembler _masm(&cbuf);
3424 Register dst_reg = as_Register($dst$$reg);
3425 address con = (address)$src$$constant;
3426 if (con == NULL || con == (address)1) {
3427 ShouldNotReachHere();
3428 } else {
3429 relocInfo::relocType rtype = $src->constant_reloc();
3430 if (rtype == relocInfo::oop_type) {
3431 __ movoop(dst_reg, (jobject)con);
3432 } else if (rtype == relocInfo::metadata_type) {
3433 __ mov_metadata(dst_reg, (Metadata*)con);
3434 } else {
3435 assert(rtype == relocInfo::none, "unexpected reloc type");
3436 if (! __ is_valid_AArch64_address(con) ||
3437 con < (address)(uintptr_t)os::vm_page_size()) {
3438 __ mov(dst_reg, con);
3439 } else {
3440 uint64_t offset;
3441 __ adrp(dst_reg, con, offset);
3442 __ add(dst_reg, dst_reg, offset);
3443 }
3444 }
3445 }
3446 %}
3447
3448 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3449 C2_MacroAssembler _masm(&cbuf);
3450 Register dst_reg = as_Register($dst$$reg);
3451 __ mov(dst_reg, zr);
3452 %}
3453
3454 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3455 C2_MacroAssembler _masm(&cbuf);
3456 Register dst_reg = as_Register($dst$$reg);
3457 __ mov(dst_reg, (uint64_t)1);
3458 %}
3459
3460 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3461 C2_MacroAssembler _masm(&cbuf);
3462 __ load_byte_map_base($dst$$Register);
3463 %}
3464
3465 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3466 C2_MacroAssembler _masm(&cbuf);
3467 Register dst_reg = as_Register($dst$$reg);
3468 address con = (address)$src$$constant;
3469 if (con == NULL) {
3470 ShouldNotReachHere();
3471 } else {
3472 relocInfo::relocType rtype = $src->constant_reloc();
3473 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3474 __ set_narrow_oop(dst_reg, (jobject)con);
3475 }
3476 %}
3477
3478 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3479 C2_MacroAssembler _masm(&cbuf);
3480 Register dst_reg = as_Register($dst$$reg);
3481 __ mov(dst_reg, zr);
3482 %}
3483
3484 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3485 C2_MacroAssembler _masm(&cbuf);
3486 Register dst_reg = as_Register($dst$$reg);
3487 address con = (address)$src$$constant;
3488 if (con == NULL) {
3489 ShouldNotReachHere();
3490 } else {
3491 relocInfo::relocType rtype = $src->constant_reloc();
3492 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3493 __ set_narrow_klass(dst_reg, (Klass *)con);
3494 }
3495 %}
3496
3497 // arithmetic encodings
3498
3499 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3500 C2_MacroAssembler _masm(&cbuf);
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src_reg = as_Register($src1$$reg);
3503 int32_t con = (int32_t)$src2$$constant;
3504 // add has primary == 0, subtract has primary == 1
3505 if ($primary) { con = -con; }
3506 if (con < 0) {
3507 __ subw(dst_reg, src_reg, -con);
3508 } else {
3509 __ addw(dst_reg, src_reg, con);
3510 }
3511 %}
3512
3513 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3514 C2_MacroAssembler _masm(&cbuf);
3515 Register dst_reg = as_Register($dst$$reg);
3516 Register src_reg = as_Register($src1$$reg);
3517 int32_t con = (int32_t)$src2$$constant;
3518 // add has primary == 0, subtract has primary == 1
3519 if ($primary) { con = -con; }
3520 if (con < 0) {
3521 __ sub(dst_reg, src_reg, -con);
3522 } else {
3523 __ add(dst_reg, src_reg, con);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3528 C2_MacroAssembler _masm(&cbuf);
3529 Register dst_reg = as_Register($dst$$reg);
3530 Register src1_reg = as_Register($src1$$reg);
3531 Register src2_reg = as_Register($src2$$reg);
3532 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3533 %}
3534
3535 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3536 C2_MacroAssembler _masm(&cbuf);
3537 Register dst_reg = as_Register($dst$$reg);
3538 Register src1_reg = as_Register($src1$$reg);
3539 Register src2_reg = as_Register($src2$$reg);
3540 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3541 %}
3542
3543 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3544 C2_MacroAssembler _masm(&cbuf);
3545 Register dst_reg = as_Register($dst$$reg);
3546 Register src1_reg = as_Register($src1$$reg);
3547 Register src2_reg = as_Register($src2$$reg);
3548 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3552 C2_MacroAssembler _masm(&cbuf);
3553 Register dst_reg = as_Register($dst$$reg);
3554 Register src1_reg = as_Register($src1$$reg);
3555 Register src2_reg = as_Register($src2$$reg);
3556 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3557 %}
3558
3559 // compare instruction encodings
3560
3561 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3562 C2_MacroAssembler _masm(&cbuf);
3563 Register reg1 = as_Register($src1$$reg);
3564 Register reg2 = as_Register($src2$$reg);
3565 __ cmpw(reg1, reg2);
3566 %}
3567
3568 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3569 C2_MacroAssembler _masm(&cbuf);
3570 Register reg = as_Register($src1$$reg);
3571 int32_t val = $src2$$constant;
3572 if (val >= 0) {
3573 __ subsw(zr, reg, val);
3574 } else {
3575 __ addsw(zr, reg, -val);
3576 }
3577 %}
3578
3579 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3580 C2_MacroAssembler _masm(&cbuf);
3581 Register reg1 = as_Register($src1$$reg);
3582 uint32_t val = (uint32_t)$src2$$constant;
3583 __ movw(rscratch1, val);
3584 __ cmpw(reg1, rscratch1);
3585 %}
3586
3587 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3588 C2_MacroAssembler _masm(&cbuf);
3589 Register reg1 = as_Register($src1$$reg);
3590 Register reg2 = as_Register($src2$$reg);
3591 __ cmp(reg1, reg2);
3592 %}
3593
3594 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3595 C2_MacroAssembler _masm(&cbuf);
3596 Register reg = as_Register($src1$$reg);
3597 int64_t val = $src2$$constant;
3598 if (val >= 0) {
3599 __ subs(zr, reg, val);
3600 } else if (val != -val) {
3601 __ adds(zr, reg, -val);
3602 } else {
3603 // aargh, Long.MIN_VALUE is a special case
3604 __ orr(rscratch1, zr, (uint64_t)val);
3605 __ subs(zr, reg, rscratch1);
3606 }
3607 %}
3608
3609 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3610 C2_MacroAssembler _masm(&cbuf);
3611 Register reg1 = as_Register($src1$$reg);
3612 uint64_t val = (uint64_t)$src2$$constant;
3613 __ mov(rscratch1, val);
3614 __ cmp(reg1, rscratch1);
3615 %}
3616
3617 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3618 C2_MacroAssembler _masm(&cbuf);
3619 Register reg1 = as_Register($src1$$reg);
3620 Register reg2 = as_Register($src2$$reg);
3621 __ cmp(reg1, reg2);
3622 %}
3623
3624 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3625 C2_MacroAssembler _masm(&cbuf);
3626 Register reg1 = as_Register($src1$$reg);
3627 Register reg2 = as_Register($src2$$reg);
3628 __ cmpw(reg1, reg2);
3629 %}
3630
3631 enc_class aarch64_enc_testp(iRegP src) %{
3632 C2_MacroAssembler _masm(&cbuf);
3633 Register reg = as_Register($src$$reg);
3634 __ cmp(reg, zr);
3635 %}
3636
3637 enc_class aarch64_enc_testn(iRegN src) %{
3638 C2_MacroAssembler _masm(&cbuf);
3639 Register reg = as_Register($src$$reg);
3640 __ cmpw(reg, zr);
3641 %}
3642
3643 enc_class aarch64_enc_b(label lbl) %{
3644 C2_MacroAssembler _masm(&cbuf);
3645 Label *L = $lbl$$label;
3646 __ b(*L);
3647 %}
3648
3649 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3650 C2_MacroAssembler _masm(&cbuf);
3651 Label *L = $lbl$$label;
3652 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3653 %}
3654
3655 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3656 C2_MacroAssembler _masm(&cbuf);
3657 Label *L = $lbl$$label;
3658 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3659 %}
3660
3661 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3662 %{
3663 Register sub_reg = as_Register($sub$$reg);
3664 Register super_reg = as_Register($super$$reg);
3665 Register temp_reg = as_Register($temp$$reg);
3666 Register result_reg = as_Register($result$$reg);
3667
3668 Label miss;
3669 C2_MacroAssembler _masm(&cbuf);
3670 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3671 NULL, &miss,
3672 /*set_cond_codes:*/ true);
3673 if ($primary) {
3674 __ mov(result_reg, zr);
3675 }
3676 __ bind(miss);
3677 %}
3678
3679 enc_class aarch64_enc_java_static_call(method meth) %{
3680 C2_MacroAssembler _masm(&cbuf);
3681
3682 address addr = (address)$meth$$method;
3683 address call;
3684 if (!_method) {
3685 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3686 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3687 if (call == NULL) {
3688 ciEnv::current()->record_failure("CodeCache is full");
3689 return;
3690 }
3691 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3692 // The NOP here is purely to ensure that eliding a call to
3693 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3694 __ nop();
3695 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3696 } else {
3697 int method_index = resolved_method_index(cbuf);
3698 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3699 : static_call_Relocation::spec(method_index);
3700 call = __ trampoline_call(Address(addr, rspec));
3701 if (call == NULL) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3706 // Calls of the same statically bound method can share
3707 // a stub to the interpreter.
3708 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3709 } else {
3710 // Emit stub for static call
3711 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3712 if (stub == NULL) {
3713 ciEnv::current()->record_failure("CodeCache is full");
3714 return;
3715 }
3716 }
3717 }
3718
3719 __ post_call_nop();
3720
3721 // Only non uncommon_trap calls need to reinitialize ptrue.
3722 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3723 __ reinitialize_ptrue();
3724 }
3725 %}
3726
3727 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3728 C2_MacroAssembler _masm(&cbuf);
3729 int method_index = resolved_method_index(cbuf);
3730 address call = __ ic_call((address)$meth$$method, method_index);
3731 if (call == NULL) {
3732 ciEnv::current()->record_failure("CodeCache is full");
3733 return;
3734 }
3735 __ post_call_nop();
3736 if (Compile::current()->max_vector_size() > 0) {
3737 __ reinitialize_ptrue();
3738 }
3739 %}
3740
3741 enc_class aarch64_enc_call_epilog() %{
3742 C2_MacroAssembler _masm(&cbuf);
3743 if (VerifyStackAtCalls) {
3744 // Check that stack depth is unchanged: find majik cookie on stack
3745 __ call_Unimplemented();
3746 }
3747 %}
3748
3749 enc_class aarch64_enc_java_to_runtime(method meth) %{
3750 C2_MacroAssembler _masm(&cbuf);
3751
3752 // some calls to generated routines (arraycopy code) are scheduled
3753 // by C2 as runtime calls. if so we can call them using a br (they
3754 // will be in a reachable segment) otherwise we have to use a blr
3755 // which loads the absolute address into a register.
3756 address entry = (address)$meth$$method;
3757 CodeBlob *cb = CodeCache::find_blob(entry);
3758 if (cb) {
3759 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3760 if (call == NULL) {
3761 ciEnv::current()->record_failure("CodeCache is full");
3762 return;
3763 }
3764 __ post_call_nop();
3765 } else {
3766 Label retaddr;
3767 __ adr(rscratch2, retaddr);
3768 __ lea(rscratch1, RuntimeAddress(entry));
3769 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3770 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3771 __ blr(rscratch1);
3772 __ bind(retaddr);
3773 __ post_call_nop();
3774 __ add(sp, sp, 2 * wordSize);
3775 }
3776 if (Compile::current()->max_vector_size() > 0) {
3777 __ reinitialize_ptrue();
3778 }
3779 %}
3780
3781 enc_class aarch64_enc_rethrow() %{
3782 C2_MacroAssembler _masm(&cbuf);
3783 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3784 %}
3785
3786 enc_class aarch64_enc_ret() %{
3787 C2_MacroAssembler _masm(&cbuf);
3788 #ifdef ASSERT
3789 if (Compile::current()->max_vector_size() > 0) {
3790 __ verify_ptrue();
3791 }
3792 #endif
3793 __ ret(lr);
3794 %}
3795
3796 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3797 C2_MacroAssembler _masm(&cbuf);
3798 Register target_reg = as_Register($jump_target$$reg);
3799 __ br(target_reg);
3800 %}
3801
3802 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3803 C2_MacroAssembler _masm(&cbuf);
3804 Register target_reg = as_Register($jump_target$$reg);
3805 // exception oop should be in r0
3806 // ret addr has been popped into lr
3807 // callee expects it in r3
3808 __ mov(r3, lr);
3809 __ br(target_reg);
3810 %}
3811
3812 %}
3813
3814 //----------FRAME--------------------------------------------------------------
3815 // Definition of frame structure and management information.
3816 //
3817 // S T A C K L A Y O U T Allocators stack-slot number
3818 // | (to get allocators register number
3819 // G Owned by | | v add OptoReg::stack0())
3820 // r CALLER | |
3821 // o | +--------+ pad to even-align allocators stack-slot
3822 // w V | pad0 | numbers; owned by CALLER
3823 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3824 // h ^ | in | 5
3825 // | | args | 4 Holes in incoming args owned by SELF
3826 // | | | | 3
3827 // | | +--------+
3828 // V | | old out| Empty on Intel, window on Sparc
3829 // | old |preserve| Must be even aligned.
3830 // | SP-+--------+----> Matcher::_old_SP, even aligned
3831 // | | in | 3 area for Intel ret address
3832 // Owned by |preserve| Empty on Sparc.
3833 // SELF +--------+
3834 // | | pad2 | 2 pad to align old SP
3835 // | +--------+ 1
3836 // | | locks | 0
3837 // | +--------+----> OptoReg::stack0(), even aligned
3838 // | | pad1 | 11 pad to align new SP
3839 // | +--------+
3840 // | | | 10
3841 // | | spills | 9 spills
3842 // V | | 8 (pad0 slot for callee)
3843 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3844 // ^ | out | 7
3845 // | | args | 6 Holes in outgoing args owned by CALLEE
3846 // Owned by +--------+
3847 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3848 // | new |preserve| Must be even-aligned.
3849 // | SP-+--------+----> Matcher::_new_SP, even aligned
3850 // | | |
3851 //
3852 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3853 // known from SELF's arguments and the Java calling convention.
3854 // Region 6-7 is determined per call site.
3855 // Note 2: If the calling convention leaves holes in the incoming argument
3856 // area, those holes are owned by SELF. Holes in the outgoing area
3857 // are owned by the CALLEE. Holes should not be necessary in the
3858 // incoming area, as the Java calling convention is completely under
3859 // the control of the AD file. Doubles can be sorted and packed to
3860 // avoid holes. Holes in the outgoing arguments may be necessary for
3861 // varargs C calling conventions.
3862 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3863 // even aligned with pad0 as needed.
3864 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3865 // (the latter is true on Intel but is it false on AArch64?)
3866 // region 6-11 is even aligned; it may be padded out more so that
3867 // the region from SP to FP meets the minimum stack alignment.
3868 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3869 // alignment. Region 11, pad1, may be dynamically extended so that
3870 // SP meets the minimum alignment.
3871
3872 frame %{
3873 // These three registers define part of the calling convention
3874 // between compiled code and the interpreter.
3875
3876 // Inline Cache Register or Method for I2C.
3877 inline_cache_reg(R12);
3878
3879 // Number of stack slots consumed by locking an object
3880 sync_stack_slots(2);
3881
3882 // Compiled code's Frame Pointer
3883 frame_pointer(R31);
3884
3885 // Interpreter stores its frame pointer in a register which is
3886 // stored to the stack by I2CAdaptors.
3887 // I2CAdaptors convert from interpreted java to compiled java.
3888 interpreter_frame_pointer(R29);
3889
3890 // Stack alignment requirement
3891 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3892
3893 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3894 // for calls to C. Supports the var-args backing area for register parms.
3895 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3896
3897 // The after-PROLOG location of the return address. Location of
3898 // return address specifies a type (REG or STACK) and a number
3899 // representing the register number (i.e. - use a register name) or
3900 // stack slot.
3901 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3902 // Otherwise, it is above the locks and verification slot and alignment word
3903 // TODO this may well be correct but need to check why that - 2 is there
3904 // ppc port uses 0 but we definitely need to allow for fixed_slots
3905 // which folds in the space used for monitors
3906 return_addr(STACK - 2 +
3907 align_up((Compile::current()->in_preserve_stack_slots() +
3908 Compile::current()->fixed_slots()),
3909 stack_alignment_in_slots()));
3910
3911 // Location of compiled Java return values. Same as C for now.
3912 return_value
3913 %{
3914 // TODO do we allow ideal_reg == Op_RegN???
3915 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3916 "only return normal values");
3917
3918 static const int lo[Op_RegL + 1] = { // enum name
3919 0, // Op_Node
3920 0, // Op_Set
3921 R0_num, // Op_RegN
3922 R0_num, // Op_RegI
3923 R0_num, // Op_RegP
3924 V0_num, // Op_RegF
3925 V0_num, // Op_RegD
3926 R0_num // Op_RegL
3927 };
3928
3929 static const int hi[Op_RegL + 1] = { // enum name
3930 0, // Op_Node
3931 0, // Op_Set
3932 OptoReg::Bad, // Op_RegN
3933 OptoReg::Bad, // Op_RegI
3934 R0_H_num, // Op_RegP
3935 OptoReg::Bad, // Op_RegF
3936 V0_H_num, // Op_RegD
3937 R0_H_num // Op_RegL
3938 };
3939
3940 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3941 %}
3942 %}
3943
3944 //----------ATTRIBUTES---------------------------------------------------------
3945 //----------Operand Attributes-------------------------------------------------
3946 op_attrib op_cost(1); // Required cost attribute
3947
3948 //----------Instruction Attributes---------------------------------------------
3949 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3950 ins_attrib ins_size(32); // Required size attribute (in bits)
3951 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3952 // a non-matching short branch variant
3953 // of some long branch?
3954 ins_attrib ins_alignment(4); // Required alignment attribute (must
3955 // be a power of 2) specifies the
3956 // alignment that some part of the
3957 // instruction (not necessarily the
3958 // start) requires. If > 1, a
3959 // compute_padding() function must be
3960 // provided for the instruction
3961
3962 //----------OPERANDS-----------------------------------------------------------
3963 // Operand definitions must precede instruction definitions for correct parsing
3964 // in the ADLC because operands constitute user defined types which are used in
3965 // instruction definitions.
3966
3967 //----------Simple Operands----------------------------------------------------
3968
3969 // Integer operands 32 bit
3970 // 32 bit immediate
3971 operand immI()
3972 %{
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 // 32 bit zero
3981 operand immI0()
3982 %{
3983 predicate(n->get_int() == 0);
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 // 32 bit unit increment
3992 operand immI_1()
3993 %{
3994 predicate(n->get_int() == 1);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 // 32 bit unit decrement
4003 operand immI_M1()
4004 %{
4005 predicate(n->get_int() == -1);
4006 match(ConI);
4007
4008 op_cost(0);
4009 format %{ %}
4010 interface(CONST_INTER);
4011 %}
4012
4013 // Shift values for add/sub extension shift
4014 operand immIExt()
4015 %{
4016 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4017 match(ConI);
4018
4019 op_cost(0);
4020 format %{ %}
4021 interface(CONST_INTER);
4022 %}
4023
4024 operand immI_gt_1()
4025 %{
4026 predicate(n->get_int() > 1);
4027 match(ConI);
4028
4029 op_cost(0);
4030 format %{ %}
4031 interface(CONST_INTER);
4032 %}
4033
4034 operand immI_le_4()
4035 %{
4036 predicate(n->get_int() <= 4);
4037 match(ConI);
4038
4039 op_cost(0);
4040 format %{ %}
4041 interface(CONST_INTER);
4042 %}
4043
4044 operand immI_16()
4045 %{
4046 predicate(n->get_int() == 16);
4047 match(ConI);
4048
4049 op_cost(0);
4050 format %{ %}
4051 interface(CONST_INTER);
4052 %}
4053
4054 operand immI_24()
4055 %{
4056 predicate(n->get_int() == 24);
4057 match(ConI);
4058
4059 op_cost(0);
4060 format %{ %}
4061 interface(CONST_INTER);
4062 %}
4063
4064 operand immI_32()
4065 %{
4066 predicate(n->get_int() == 32);
4067 match(ConI);
4068
4069 op_cost(0);
4070 format %{ %}
4071 interface(CONST_INTER);
4072 %}
4073
4074 operand immI_48()
4075 %{
4076 predicate(n->get_int() == 48);
4077 match(ConI);
4078
4079 op_cost(0);
4080 format %{ %}
4081 interface(CONST_INTER);
4082 %}
4083
4084 operand immI_56()
4085 %{
4086 predicate(n->get_int() == 56);
4087 match(ConI);
4088
4089 op_cost(0);
4090 format %{ %}
4091 interface(CONST_INTER);
4092 %}
4093
4094 operand immI_63()
4095 %{
4096 predicate(n->get_int() == 63);
4097 match(ConI);
4098
4099 op_cost(0);
4100 format %{ %}
4101 interface(CONST_INTER);
4102 %}
4103
4104 operand immI_64()
4105 %{
4106 predicate(n->get_int() == 64);
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immI_255()
4115 %{
4116 predicate(n->get_int() == 255);
4117 match(ConI);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immI_65535()
4125 %{
4126 predicate(n->get_int() == 65535);
4127 match(ConI);
4128
4129 op_cost(0);
4130 format %{ %}
4131 interface(CONST_INTER);
4132 %}
4133
4134 operand immI_positive()
4135 %{
4136 predicate(n->get_int() > 0);
4137 match(ConI);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 // BoolTest condition for signed compare
4145 operand immI_cmp_cond()
4146 %{
4147 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4148 match(ConI);
4149
4150 op_cost(0);
4151 format %{ %}
4152 interface(CONST_INTER);
4153 %}
4154
4155 // BoolTest condition for unsigned compare
4156 operand immI_cmpU_cond()
4157 %{
4158 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4159 match(ConI);
4160
4161 op_cost(0);
4162 format %{ %}
4163 interface(CONST_INTER);
4164 %}
4165
4166 operand immL_255()
4167 %{
4168 predicate(n->get_long() == 255L);
4169 match(ConL);
4170
4171 op_cost(0);
4172 format %{ %}
4173 interface(CONST_INTER);
4174 %}
4175
4176 operand immL_65535()
4177 %{
4178 predicate(n->get_long() == 65535L);
4179 match(ConL);
4180
4181 op_cost(0);
4182 format %{ %}
4183 interface(CONST_INTER);
4184 %}
4185
4186 operand immL_4294967295()
4187 %{
4188 predicate(n->get_long() == 4294967295L);
4189 match(ConL);
4190
4191 op_cost(0);
4192 format %{ %}
4193 interface(CONST_INTER);
4194 %}
4195
4196 operand immL_bitmask()
4197 %{
4198 predicate((n->get_long() != 0)
4199 && ((n->get_long() & 0xc000000000000000l) == 0)
4200 && is_power_of_2(n->get_long() + 1));
4201 match(ConL);
4202
4203 op_cost(0);
4204 format %{ %}
4205 interface(CONST_INTER);
4206 %}
4207
4208 operand immI_bitmask()
4209 %{
4210 predicate((n->get_int() != 0)
4211 && ((n->get_int() & 0xc0000000) == 0)
4212 && is_power_of_2(n->get_int() + 1));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immL_positive_bitmaskI()
4221 %{
4222 predicate((n->get_long() != 0)
4223 && ((julong)n->get_long() < 0x80000000ULL)
4224 && is_power_of_2(n->get_long() + 1));
4225 match(ConL);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 // Scale values for scaled offset addressing modes (up to long but not quad)
4233 operand immIScale()
4234 %{
4235 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4236 match(ConI);
4237
4238 op_cost(0);
4239 format %{ %}
4240 interface(CONST_INTER);
4241 %}
4242
4243 // 26 bit signed offset -- for pc-relative branches
4244 operand immI26()
4245 %{
4246 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 // 19 bit signed offset -- for pc-relative loads
4255 operand immI19()
4256 %{
4257 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4258 match(ConI);
4259
4260 op_cost(0);
4261 format %{ %}
4262 interface(CONST_INTER);
4263 %}
4264
4265 // 5 bit signed integer
4266 operand immI5()
4267 %{
4268 predicate(Assembler::is_simm(n->get_int(), 5));
4269 match(ConI);
4270
4271 op_cost(0);
4272 format %{ %}
4273 interface(CONST_INTER);
4274 %}
4275
4276 // 7 bit unsigned integer
4277 operand immIU7()
4278 %{
4279 predicate(Assembler::is_uimm(n->get_int(), 7));
4280 match(ConI);
4281
4282 op_cost(0);
4283 format %{ %}
4284 interface(CONST_INTER);
4285 %}
4286
4287 // 12 bit unsigned offset -- for base plus immediate loads
4288 operand immIU12()
4289 %{
4290 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4291 match(ConI);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immLU12()
4299 %{
4300 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 // Offset for scaled or unscaled immediate loads and stores
4309 operand immIOffset()
4310 %{
4311 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4312 match(ConI);
4313
4314 op_cost(0);
4315 format %{ %}
4316 interface(CONST_INTER);
4317 %}
4318
4319 operand immIOffset1()
4320 %{
4321 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4322 match(ConI);
4323
4324 op_cost(0);
4325 format %{ %}
4326 interface(CONST_INTER);
4327 %}
4328
4329 operand immIOffset2()
4330 %{
4331 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4332 match(ConI);
4333
4334 op_cost(0);
4335 format %{ %}
4336 interface(CONST_INTER);
4337 %}
4338
4339 operand immIOffset4()
4340 %{
4341 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4342 match(ConI);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 operand immIOffset8()
4350 %{
4351 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4352 match(ConI);
4353
4354 op_cost(0);
4355 format %{ %}
4356 interface(CONST_INTER);
4357 %}
4358
4359 operand immIOffset16()
4360 %{
4361 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4362 match(ConI);
4363
4364 op_cost(0);
4365 format %{ %}
4366 interface(CONST_INTER);
4367 %}
4368
4369 operand immLoffset()
4370 %{
4371 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4372 match(ConL);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 operand immLoffset1()
4380 %{
4381 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4382 match(ConL);
4383
4384 op_cost(0);
4385 format %{ %}
4386 interface(CONST_INTER);
4387 %}
4388
4389 operand immLoffset2()
4390 %{
4391 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4392 match(ConL);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 operand immLoffset4()
4400 %{
4401 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4402 match(ConL);
4403
4404 op_cost(0);
4405 format %{ %}
4406 interface(CONST_INTER);
4407 %}
4408
4409 operand immLoffset8()
4410 %{
4411 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4412 match(ConL);
4413
4414 op_cost(0);
4415 format %{ %}
4416 interface(CONST_INTER);
4417 %}
4418
4419 operand immLoffset16()
4420 %{
4421 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4422 match(ConL);
4423
4424 op_cost(0);
4425 format %{ %}
4426 interface(CONST_INTER);
4427 %}
4428
4429 // 5 bit signed long integer
4430 operand immL5()
4431 %{
4432 predicate(Assembler::is_simm(n->get_long(), 5));
4433 match(ConL);
4434
4435 op_cost(0);
4436 format %{ %}
4437 interface(CONST_INTER);
4438 %}
4439
4440 // 7 bit unsigned long integer
4441 operand immLU7()
4442 %{
4443 predicate(Assembler::is_uimm(n->get_long(), 7));
4444 match(ConL);
4445
4446 op_cost(0);
4447 format %{ %}
4448 interface(CONST_INTER);
4449 %}
4450
4451 // 8 bit signed value.
4452 operand immI8()
4453 %{
4454 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4455 match(ConI);
4456
4457 op_cost(0);
4458 format %{ %}
4459 interface(CONST_INTER);
4460 %}
4461
4462 // 8 bit signed value (simm8), or #simm8 LSL 8.
4463 operand immI8_shift8()
4464 %{
4465 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4466 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4467 match(ConI);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 8 bit signed value (simm8), or #simm8 LSL 8.
4475 operand immL8_shift8()
4476 %{
4477 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4478 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4479 match(ConL);
4480
4481 op_cost(0);
4482 format %{ %}
4483 interface(CONST_INTER);
4484 %}
4485
4486 // 8 bit integer valid for vector add sub immediate
4487 operand immBAddSubV()
4488 %{
4489 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4490 match(ConI);
4491
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 // 32 bit integer valid for add sub immediate
4498 operand immIAddSub()
4499 %{
4500 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4501 match(ConI);
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // 32 bit integer valid for vector add sub immediate
4508 operand immIAddSubV()
4509 %{
4510 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4511 match(ConI);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // 32 bit unsigned integer valid for logical immediate
4519
4520 operand immBLog()
4521 %{
4522 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4523 match(ConI);
4524
4525 op_cost(0);
4526 format %{ %}
4527 interface(CONST_INTER);
4528 %}
4529
4530 operand immSLog()
4531 %{
4532 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4533 match(ConI);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 operand immILog()
4541 %{
4542 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4543 match(ConI);
4544
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // Integer operands 64 bit
4551 // 64 bit immediate
4552 operand immL()
4553 %{
4554 match(ConL);
4555
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // 64 bit zero
4562 operand immL0()
4563 %{
4564 predicate(n->get_long() == 0);
4565 match(ConL);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // 64 bit unit increment
4573 operand immL_1()
4574 %{
4575 predicate(n->get_long() == 1);
4576 match(ConL);
4577
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 // 64 bit unit decrement
4584 operand immL_M1()
4585 %{
4586 predicate(n->get_long() == -1);
4587 match(ConL);
4588
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // 32 bit offset of pc in thread anchor
4595
4596 operand immL_pc_off()
4597 %{
4598 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4599 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4600 match(ConL);
4601
4602 op_cost(0);
4603 format %{ %}
4604 interface(CONST_INTER);
4605 %}
4606
4607 // 64 bit integer valid for add sub immediate
4608 operand immLAddSub()
4609 %{
4610 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4611 match(ConL);
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 // 64 bit integer valid for addv subv immediate
4618 operand immLAddSubV()
4619 %{
4620 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4621 match(ConL);
4622
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 // 64 bit integer valid for logical immediate
4629 operand immLLog()
4630 %{
4631 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4632 match(ConL);
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 // Long Immediate: low 32-bit mask
4639 operand immL_32bits()
4640 %{
4641 predicate(n->get_long() == 0xFFFFFFFFL);
4642 match(ConL);
4643 op_cost(0);
4644 format %{ %}
4645 interface(CONST_INTER);
4646 %}
4647
4648 // Pointer operands
4649 // Pointer Immediate
4650 operand immP()
4651 %{
4652 match(ConP);
4653
4654 op_cost(0);
4655 format %{ %}
4656 interface(CONST_INTER);
4657 %}
4658
4659 // NULL Pointer Immediate
4660 operand immP0()
4661 %{
4662 predicate(n->get_ptr() == 0);
4663 match(ConP);
4664
4665 op_cost(0);
4666 format %{ %}
4667 interface(CONST_INTER);
4668 %}
4669
4670 // Pointer Immediate One
4671 // this is used in object initialization (initial object header)
4672 operand immP_1()
4673 %{
4674 predicate(n->get_ptr() == 1);
4675 match(ConP);
4676
4677 op_cost(0);
4678 format %{ %}
4679 interface(CONST_INTER);
4680 %}
4681
4682 // Card Table Byte Map Base
4683 operand immByteMapBase()
4684 %{
4685 // Get base of card map
4686 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4687 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4688 match(ConP);
4689
4690 op_cost(0);
4691 format %{ %}
4692 interface(CONST_INTER);
4693 %}
4694
4695 // Pointer Immediate Minus One
4696 // this is used when we want to write the current PC to the thread anchor
4697 operand immP_M1()
4698 %{
4699 predicate(n->get_ptr() == -1);
4700 match(ConP);
4701
4702 op_cost(0);
4703 format %{ %}
4704 interface(CONST_INTER);
4705 %}
4706
4707 // Pointer Immediate Minus Two
4708 // this is used when we want to write the current PC to the thread anchor
4709 operand immP_M2()
4710 %{
4711 predicate(n->get_ptr() == -2);
4712 match(ConP);
4713
4714 op_cost(0);
4715 format %{ %}
4716 interface(CONST_INTER);
4717 %}
4718
4719 // Float and Double operands
4720 // Double Immediate
4721 operand immD()
4722 %{
4723 match(ConD);
4724 op_cost(0);
4725 format %{ %}
4726 interface(CONST_INTER);
4727 %}
4728
4729 // Double Immediate: +0.0d
4730 operand immD0()
4731 %{
4732 predicate(jlong_cast(n->getd()) == 0);
4733 match(ConD);
4734
4735 op_cost(0);
4736 format %{ %}
4737 interface(CONST_INTER);
4738 %}
4739
4740 // constant 'double +0.0'.
4741 operand immDPacked()
4742 %{
4743 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4744 match(ConD);
4745 op_cost(0);
4746 format %{ %}
4747 interface(CONST_INTER);
4748 %}
4749
4750 // Float Immediate
4751 operand immF()
4752 %{
4753 match(ConF);
4754 op_cost(0);
4755 format %{ %}
4756 interface(CONST_INTER);
4757 %}
4758
4759 // Float Immediate: +0.0f.
4760 operand immF0()
4761 %{
4762 predicate(jint_cast(n->getf()) == 0);
4763 match(ConF);
4764
4765 op_cost(0);
4766 format %{ %}
4767 interface(CONST_INTER);
4768 %}
4769
4770 //
4771 operand immFPacked()
4772 %{
4773 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4774 match(ConF);
4775 op_cost(0);
4776 format %{ %}
4777 interface(CONST_INTER);
4778 %}
4779
4780 // Narrow pointer operands
4781 // Narrow Pointer Immediate
4782 operand immN()
4783 %{
4784 match(ConN);
4785
4786 op_cost(0);
4787 format %{ %}
4788 interface(CONST_INTER);
4789 %}
4790
4791 // Narrow NULL Pointer Immediate
4792 operand immN0()
4793 %{
4794 predicate(n->get_narrowcon() == 0);
4795 match(ConN);
4796
4797 op_cost(0);
4798 format %{ %}
4799 interface(CONST_INTER);
4800 %}
4801
4802 operand immNKlass()
4803 %{
4804 match(ConNKlass);
4805
4806 op_cost(0);
4807 format %{ %}
4808 interface(CONST_INTER);
4809 %}
4810
4811 // Integer 32 bit Register Operands
4812 // Integer 32 bitRegister (excludes SP)
4813 operand iRegI()
4814 %{
4815 constraint(ALLOC_IN_RC(any_reg32));
4816 match(RegI);
4817 match(iRegINoSp);
4818 op_cost(0);
4819 format %{ %}
4820 interface(REG_INTER);
4821 %}
4822
4823 // Integer 32 bit Register not Special
4824 operand iRegINoSp()
4825 %{
4826 constraint(ALLOC_IN_RC(no_special_reg32));
4827 match(RegI);
4828 op_cost(0);
4829 format %{ %}
4830 interface(REG_INTER);
4831 %}
4832
4833 // Integer 64 bit Register Operands
4834 // Integer 64 bit Register (includes SP)
4835 operand iRegL()
4836 %{
4837 constraint(ALLOC_IN_RC(any_reg));
4838 match(RegL);
4839 match(iRegLNoSp);
4840 op_cost(0);
4841 format %{ %}
4842 interface(REG_INTER);
4843 %}
4844
4845 // Integer 64 bit Register not Special
4846 operand iRegLNoSp()
4847 %{
4848 constraint(ALLOC_IN_RC(no_special_reg));
4849 match(RegL);
4850 match(iRegL_R0);
4851 format %{ %}
4852 interface(REG_INTER);
4853 %}
4854
4855 // Pointer Register Operands
4856 // Pointer Register
4857 operand iRegP()
4858 %{
4859 constraint(ALLOC_IN_RC(ptr_reg));
4860 match(RegP);
4861 match(iRegPNoSp);
4862 match(iRegP_R0);
4863 //match(iRegP_R2);
4864 //match(iRegP_R4);
4865 match(iRegP_R5);
4866 match(thread_RegP);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872 // Pointer 64 bit Register not Special
4873 operand iRegPNoSp()
4874 %{
4875 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4876 match(RegP);
4877 // match(iRegP);
4878 // match(iRegP_R0);
4879 // match(iRegP_R2);
4880 // match(iRegP_R4);
4881 // match(iRegP_R5);
4882 // match(thread_RegP);
4883 op_cost(0);
4884 format %{ %}
4885 interface(REG_INTER);
4886 %}
4887
4888 // Pointer 64 bit Register R0 only
4889 operand iRegP_R0()
4890 %{
4891 constraint(ALLOC_IN_RC(r0_reg));
4892 match(RegP);
4893 // match(iRegP);
4894 match(iRegPNoSp);
4895 op_cost(0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Pointer 64 bit Register R1 only
4901 operand iRegP_R1()
4902 %{
4903 constraint(ALLOC_IN_RC(r1_reg));
4904 match(RegP);
4905 // match(iRegP);
4906 match(iRegPNoSp);
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912 // Pointer 64 bit Register R2 only
4913 operand iRegP_R2()
4914 %{
4915 constraint(ALLOC_IN_RC(r2_reg));
4916 match(RegP);
4917 // match(iRegP);
4918 match(iRegPNoSp);
4919 op_cost(0);
4920 format %{ %}
4921 interface(REG_INTER);
4922 %}
4923
4924 // Pointer 64 bit Register R3 only
4925 operand iRegP_R3()
4926 %{
4927 constraint(ALLOC_IN_RC(r3_reg));
4928 match(RegP);
4929 // match(iRegP);
4930 match(iRegPNoSp);
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 // Pointer 64 bit Register R4 only
4937 operand iRegP_R4()
4938 %{
4939 constraint(ALLOC_IN_RC(r4_reg));
4940 match(RegP);
4941 // match(iRegP);
4942 match(iRegPNoSp);
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 // Pointer 64 bit Register R5 only
4949 operand iRegP_R5()
4950 %{
4951 constraint(ALLOC_IN_RC(r5_reg));
4952 match(RegP);
4953 // match(iRegP);
4954 match(iRegPNoSp);
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Pointer 64 bit Register R10 only
4961 operand iRegP_R10()
4962 %{
4963 constraint(ALLOC_IN_RC(r10_reg));
4964 match(RegP);
4965 // match(iRegP);
4966 match(iRegPNoSp);
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Long 64 bit Register R0 only
4973 operand iRegL_R0()
4974 %{
4975 constraint(ALLOC_IN_RC(r0_reg));
4976 match(RegL);
4977 match(iRegLNoSp);
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 // Long 64 bit Register R2 only
4984 operand iRegL_R2()
4985 %{
4986 constraint(ALLOC_IN_RC(r2_reg));
4987 match(RegL);
4988 match(iRegLNoSp);
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 // Long 64 bit Register R3 only
4995 operand iRegL_R3()
4996 %{
4997 constraint(ALLOC_IN_RC(r3_reg));
4998 match(RegL);
4999 match(iRegLNoSp);
5000 op_cost(0);
5001 format %{ %}
5002 interface(REG_INTER);
5003 %}
5004
5005 // Long 64 bit Register R11 only
5006 operand iRegL_R11()
5007 %{
5008 constraint(ALLOC_IN_RC(r11_reg));
5009 match(RegL);
5010 match(iRegLNoSp);
5011 op_cost(0);
5012 format %{ %}
5013 interface(REG_INTER);
5014 %}
5015
5016 // Pointer 64 bit Register FP only
5017 operand iRegP_FP()
5018 %{
5019 constraint(ALLOC_IN_RC(fp_reg));
5020 match(RegP);
5021 // match(iRegP);
5022 op_cost(0);
5023 format %{ %}
5024 interface(REG_INTER);
5025 %}
5026
5027 // Register R0 only
5028 operand iRegI_R0()
5029 %{
5030 constraint(ALLOC_IN_RC(int_r0_reg));
5031 match(RegI);
5032 match(iRegINoSp);
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 // Register R2 only
5039 operand iRegI_R2()
5040 %{
5041 constraint(ALLOC_IN_RC(int_r2_reg));
5042 match(RegI);
5043 match(iRegINoSp);
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 // Register R3 only
5050 operand iRegI_R3()
5051 %{
5052 constraint(ALLOC_IN_RC(int_r3_reg));
5053 match(RegI);
5054 match(iRegINoSp);
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060
5061 // Register R4 only
5062 operand iRegI_R4()
5063 %{
5064 constraint(ALLOC_IN_RC(int_r4_reg));
5065 match(RegI);
5066 match(iRegINoSp);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072
5073 // Pointer Register Operands
5074 // Narrow Pointer Register
5075 operand iRegN()
5076 %{
5077 constraint(ALLOC_IN_RC(any_reg32));
5078 match(RegN);
5079 match(iRegNNoSp);
5080 op_cost(0);
5081 format %{ %}
5082 interface(REG_INTER);
5083 %}
5084
5085 operand iRegN_R0()
5086 %{
5087 constraint(ALLOC_IN_RC(r0_reg));
5088 match(iRegN);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 operand iRegN_R2()
5095 %{
5096 constraint(ALLOC_IN_RC(r2_reg));
5097 match(iRegN);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 operand iRegN_R3()
5104 %{
5105 constraint(ALLOC_IN_RC(r3_reg));
5106 match(iRegN);
5107 op_cost(0);
5108 format %{ %}
5109 interface(REG_INTER);
5110 %}
5111
5112 // Integer 64 bit Register not Special
5113 operand iRegNNoSp()
5114 %{
5115 constraint(ALLOC_IN_RC(no_special_reg32));
5116 match(RegN);
5117 op_cost(0);
5118 format %{ %}
5119 interface(REG_INTER);
5120 %}
5121
5122 // Float Register
5123 // Float register operands
5124 operand vRegF()
5125 %{
5126 constraint(ALLOC_IN_RC(float_reg));
5127 match(RegF);
5128
5129 op_cost(0);
5130 format %{ %}
5131 interface(REG_INTER);
5132 %}
5133
5134 // Double Register
5135 // Double register operands
5136 operand vRegD()
5137 %{
5138 constraint(ALLOC_IN_RC(double_reg));
5139 match(RegD);
5140
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 // Generic vector class. This will be used for
5147 // all vector operands, including NEON and SVE.
5148 operand vReg()
5149 %{
5150 constraint(ALLOC_IN_RC(dynamic));
5151 match(VecA);
5152 match(VecD);
5153 match(VecX);
5154
5155 op_cost(0);
5156 format %{ %}
5157 interface(REG_INTER);
5158 %}
5159
5160 operand vecA()
5161 %{
5162 constraint(ALLOC_IN_RC(vectora_reg));
5163 match(VecA);
5164
5165 op_cost(0);
5166 format %{ %}
5167 interface(REG_INTER);
5168 %}
5169
5170 operand vecD()
5171 %{
5172 constraint(ALLOC_IN_RC(vectord_reg));
5173 match(VecD);
5174
5175 op_cost(0);
5176 format %{ %}
5177 interface(REG_INTER);
5178 %}
5179
5180 operand vecX()
5181 %{
5182 constraint(ALLOC_IN_RC(vectorx_reg));
5183 match(VecX);
5184
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 operand vRegD_V0()
5191 %{
5192 constraint(ALLOC_IN_RC(v0_reg));
5193 match(RegD);
5194 op_cost(0);
5195 format %{ %}
5196 interface(REG_INTER);
5197 %}
5198
5199 operand vRegD_V1()
5200 %{
5201 constraint(ALLOC_IN_RC(v1_reg));
5202 match(RegD);
5203 op_cost(0);
5204 format %{ %}
5205 interface(REG_INTER);
5206 %}
5207
5208 operand vRegD_V2()
5209 %{
5210 constraint(ALLOC_IN_RC(v2_reg));
5211 match(RegD);
5212 op_cost(0);
5213 format %{ %}
5214 interface(REG_INTER);
5215 %}
5216
5217 operand vRegD_V3()
5218 %{
5219 constraint(ALLOC_IN_RC(v3_reg));
5220 match(RegD);
5221 op_cost(0);
5222 format %{ %}
5223 interface(REG_INTER);
5224 %}
5225
5226 operand vRegD_V4()
5227 %{
5228 constraint(ALLOC_IN_RC(v4_reg));
5229 match(RegD);
5230 op_cost(0);
5231 format %{ %}
5232 interface(REG_INTER);
5233 %}
5234
5235 operand vRegD_V5()
5236 %{
5237 constraint(ALLOC_IN_RC(v5_reg));
5238 match(RegD);
5239 op_cost(0);
5240 format %{ %}
5241 interface(REG_INTER);
5242 %}
5243
5244 operand vRegD_V6()
5245 %{
5246 constraint(ALLOC_IN_RC(v6_reg));
5247 match(RegD);
5248 op_cost(0);
5249 format %{ %}
5250 interface(REG_INTER);
5251 %}
5252
5253 operand vRegD_V7()
5254 %{
5255 constraint(ALLOC_IN_RC(v7_reg));
5256 match(RegD);
5257 op_cost(0);
5258 format %{ %}
5259 interface(REG_INTER);
5260 %}
5261
5262 operand vRegD_V8()
5263 %{
5264 constraint(ALLOC_IN_RC(v8_reg));
5265 match(RegD);
5266 op_cost(0);
5267 format %{ %}
5268 interface(REG_INTER);
5269 %}
5270
5271 operand vRegD_V9()
5272 %{
5273 constraint(ALLOC_IN_RC(v9_reg));
5274 match(RegD);
5275 op_cost(0);
5276 format %{ %}
5277 interface(REG_INTER);
5278 %}
5279
5280 operand vRegD_V10()
5281 %{
5282 constraint(ALLOC_IN_RC(v10_reg));
5283 match(RegD);
5284 op_cost(0);
5285 format %{ %}
5286 interface(REG_INTER);
5287 %}
5288
5289 operand vRegD_V11()
5290 %{
5291 constraint(ALLOC_IN_RC(v11_reg));
5292 match(RegD);
5293 op_cost(0);
5294 format %{ %}
5295 interface(REG_INTER);
5296 %}
5297
5298 operand vRegD_V12()
5299 %{
5300 constraint(ALLOC_IN_RC(v12_reg));
5301 match(RegD);
5302 op_cost(0);
5303 format %{ %}
5304 interface(REG_INTER);
5305 %}
5306
5307 operand vRegD_V13()
5308 %{
5309 constraint(ALLOC_IN_RC(v13_reg));
5310 match(RegD);
5311 op_cost(0);
5312 format %{ %}
5313 interface(REG_INTER);
5314 %}
5315
5316 operand vRegD_V14()
5317 %{
5318 constraint(ALLOC_IN_RC(v14_reg));
5319 match(RegD);
5320 op_cost(0);
5321 format %{ %}
5322 interface(REG_INTER);
5323 %}
5324
5325 operand vRegD_V15()
5326 %{
5327 constraint(ALLOC_IN_RC(v15_reg));
5328 match(RegD);
5329 op_cost(0);
5330 format %{ %}
5331 interface(REG_INTER);
5332 %}
5333
5334 operand vRegD_V16()
5335 %{
5336 constraint(ALLOC_IN_RC(v16_reg));
5337 match(RegD);
5338 op_cost(0);
5339 format %{ %}
5340 interface(REG_INTER);
5341 %}
5342
5343 operand vRegD_V17()
5344 %{
5345 constraint(ALLOC_IN_RC(v17_reg));
5346 match(RegD);
5347 op_cost(0);
5348 format %{ %}
5349 interface(REG_INTER);
5350 %}
5351
5352 operand vRegD_V18()
5353 %{
5354 constraint(ALLOC_IN_RC(v18_reg));
5355 match(RegD);
5356 op_cost(0);
5357 format %{ %}
5358 interface(REG_INTER);
5359 %}
5360
5361 operand vRegD_V19()
5362 %{
5363 constraint(ALLOC_IN_RC(v19_reg));
5364 match(RegD);
5365 op_cost(0);
5366 format %{ %}
5367 interface(REG_INTER);
5368 %}
5369
5370 operand vRegD_V20()
5371 %{
5372 constraint(ALLOC_IN_RC(v20_reg));
5373 match(RegD);
5374 op_cost(0);
5375 format %{ %}
5376 interface(REG_INTER);
5377 %}
5378
5379 operand vRegD_V21()
5380 %{
5381 constraint(ALLOC_IN_RC(v21_reg));
5382 match(RegD);
5383 op_cost(0);
5384 format %{ %}
5385 interface(REG_INTER);
5386 %}
5387
5388 operand vRegD_V22()
5389 %{
5390 constraint(ALLOC_IN_RC(v22_reg));
5391 match(RegD);
5392 op_cost(0);
5393 format %{ %}
5394 interface(REG_INTER);
5395 %}
5396
5397 operand vRegD_V23()
5398 %{
5399 constraint(ALLOC_IN_RC(v23_reg));
5400 match(RegD);
5401 op_cost(0);
5402 format %{ %}
5403 interface(REG_INTER);
5404 %}
5405
5406 operand vRegD_V24()
5407 %{
5408 constraint(ALLOC_IN_RC(v24_reg));
5409 match(RegD);
5410 op_cost(0);
5411 format %{ %}
5412 interface(REG_INTER);
5413 %}
5414
5415 operand vRegD_V25()
5416 %{
5417 constraint(ALLOC_IN_RC(v25_reg));
5418 match(RegD);
5419 op_cost(0);
5420 format %{ %}
5421 interface(REG_INTER);
5422 %}
5423
5424 operand vRegD_V26()
5425 %{
5426 constraint(ALLOC_IN_RC(v26_reg));
5427 match(RegD);
5428 op_cost(0);
5429 format %{ %}
5430 interface(REG_INTER);
5431 %}
5432
5433 operand vRegD_V27()
5434 %{
5435 constraint(ALLOC_IN_RC(v27_reg));
5436 match(RegD);
5437 op_cost(0);
5438 format %{ %}
5439 interface(REG_INTER);
5440 %}
5441
5442 operand vRegD_V28()
5443 %{
5444 constraint(ALLOC_IN_RC(v28_reg));
5445 match(RegD);
5446 op_cost(0);
5447 format %{ %}
5448 interface(REG_INTER);
5449 %}
5450
5451 operand vRegD_V29()
5452 %{
5453 constraint(ALLOC_IN_RC(v29_reg));
5454 match(RegD);
5455 op_cost(0);
5456 format %{ %}
5457 interface(REG_INTER);
5458 %}
5459
5460 operand vRegD_V30()
5461 %{
5462 constraint(ALLOC_IN_RC(v30_reg));
5463 match(RegD);
5464 op_cost(0);
5465 format %{ %}
5466 interface(REG_INTER);
5467 %}
5468
5469 operand vRegD_V31()
5470 %{
5471 constraint(ALLOC_IN_RC(v31_reg));
5472 match(RegD);
5473 op_cost(0);
5474 format %{ %}
5475 interface(REG_INTER);
5476 %}
5477
5478 operand pReg()
5479 %{
5480 constraint(ALLOC_IN_RC(pr_reg));
5481 match(RegVectMask);
5482 match(pRegGov);
5483 op_cost(0);
5484 format %{ %}
5485 interface(REG_INTER);
5486 %}
5487
5488 operand pRegGov()
5489 %{
5490 constraint(ALLOC_IN_RC(gov_pr));
5491 match(RegVectMask);
5492 match(pReg);
5493 op_cost(0);
5494 format %{ %}
5495 interface(REG_INTER);
5496 %}
5497
5498 operand pRegGov_P0()
5499 %{
5500 constraint(ALLOC_IN_RC(p0_reg));
5501 match(RegVectMask);
5502 op_cost(0);
5503 format %{ %}
5504 interface(REG_INTER);
5505 %}
5506
5507 operand pRegGov_P1()
5508 %{
5509 constraint(ALLOC_IN_RC(p1_reg));
5510 match(RegVectMask);
5511 op_cost(0);
5512 format %{ %}
5513 interface(REG_INTER);
5514 %}
5515
5516 // Flags register, used as output of signed compare instructions
5517
5518 // note that on AArch64 we also use this register as the output for
5519 // for floating point compare instructions (CmpF CmpD). this ensures
5520 // that ordered inequality tests use GT, GE, LT or LE none of which
5521 // pass through cases where the result is unordered i.e. one or both
5522 // inputs to the compare is a NaN. this means that the ideal code can
5523 // replace e.g. a GT with an LE and not end up capturing the NaN case
5524 // (where the comparison should always fail). EQ and NE tests are
5525 // always generated in ideal code so that unordered folds into the NE
5526 // case, matching the behaviour of AArch64 NE.
5527 //
5528 // This differs from x86 where the outputs of FP compares use a
5529 // special FP flags registers and where compares based on this
5530 // register are distinguished into ordered inequalities (cmpOpUCF) and
5531 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5532 // to explicitly handle the unordered case in branches. x86 also has
5533 // to include extra CMoveX rules to accept a cmpOpUCF input.
5534
5535 operand rFlagsReg()
5536 %{
5537 constraint(ALLOC_IN_RC(int_flags));
5538 match(RegFlags);
5539
5540 op_cost(0);
5541 format %{ "RFLAGS" %}
5542 interface(REG_INTER);
5543 %}
5544
5545 // Flags register, used as output of unsigned compare instructions
5546 operand rFlagsRegU()
5547 %{
5548 constraint(ALLOC_IN_RC(int_flags));
5549 match(RegFlags);
5550
5551 op_cost(0);
5552 format %{ "RFLAGSU" %}
5553 interface(REG_INTER);
5554 %}
5555
5556 // Special Registers
5557
5558 // Method Register
5559 operand inline_cache_RegP(iRegP reg)
5560 %{
5561 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5562 match(reg);
5563 match(iRegPNoSp);
5564 op_cost(0);
5565 format %{ %}
5566 interface(REG_INTER);
5567 %}
5568
5569 // Thread Register
5570 operand thread_RegP(iRegP reg)
5571 %{
5572 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5573 match(reg);
5574 op_cost(0);
5575 format %{ %}
5576 interface(REG_INTER);
5577 %}
5578
5579 operand lr_RegP(iRegP reg)
5580 %{
5581 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5582 match(reg);
5583 op_cost(0);
5584 format %{ %}
5585 interface(REG_INTER);
5586 %}
5587
5588 //----------Memory Operands----------------------------------------------------
5589
5590 operand indirect(iRegP reg)
5591 %{
5592 constraint(ALLOC_IN_RC(ptr_reg));
5593 match(reg);
5594 op_cost(0);
5595 format %{ "[$reg]" %}
5596 interface(MEMORY_INTER) %{
5597 base($reg);
5598 index(0xffffffff);
5599 scale(0x0);
5600 disp(0x0);
5601 %}
5602 %}
5603
5604 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5605 %{
5606 constraint(ALLOC_IN_RC(ptr_reg));
5607 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5608 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5609 op_cost(0);
5610 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5611 interface(MEMORY_INTER) %{
5612 base($reg);
5613 index($ireg);
5614 scale($scale);
5615 disp(0x0);
5616 %}
5617 %}
5618
5619 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5620 %{
5621 constraint(ALLOC_IN_RC(ptr_reg));
5622 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5623 match(AddP reg (LShiftL lreg scale));
5624 op_cost(0);
5625 format %{ "$reg, $lreg lsl($scale)" %}
5626 interface(MEMORY_INTER) %{
5627 base($reg);
5628 index($lreg);
5629 scale($scale);
5630 disp(0x0);
5631 %}
5632 %}
5633
5634 operand indIndexI2L(iRegP reg, iRegI ireg)
5635 %{
5636 constraint(ALLOC_IN_RC(ptr_reg));
5637 match(AddP reg (ConvI2L ireg));
5638 op_cost(0);
5639 format %{ "$reg, $ireg, 0, I2L" %}
5640 interface(MEMORY_INTER) %{
5641 base($reg);
5642 index($ireg);
5643 scale(0x0);
5644 disp(0x0);
5645 %}
5646 %}
5647
5648 operand indIndex(iRegP reg, iRegL lreg)
5649 %{
5650 constraint(ALLOC_IN_RC(ptr_reg));
5651 match(AddP reg lreg);
5652 op_cost(0);
5653 format %{ "$reg, $lreg" %}
5654 interface(MEMORY_INTER) %{
5655 base($reg);
5656 index($lreg);
5657 scale(0x0);
5658 disp(0x0);
5659 %}
5660 %}
5661
5662 operand indOffI(iRegP reg, immIOffset off)
5663 %{
5664 constraint(ALLOC_IN_RC(ptr_reg));
5665 match(AddP reg off);
5666 op_cost(0);
5667 format %{ "[$reg, $off]" %}
5668 interface(MEMORY_INTER) %{
5669 base($reg);
5670 index(0xffffffff);
5671 scale(0x0);
5672 disp($off);
5673 %}
5674 %}
5675
5676 operand indOffI1(iRegP reg, immIOffset1 off)
5677 %{
5678 constraint(ALLOC_IN_RC(ptr_reg));
5679 match(AddP reg off);
5680 op_cost(0);
5681 format %{ "[$reg, $off]" %}
5682 interface(MEMORY_INTER) %{
5683 base($reg);
5684 index(0xffffffff);
5685 scale(0x0);
5686 disp($off);
5687 %}
5688 %}
5689
5690 operand indOffI2(iRegP reg, immIOffset2 off)
5691 %{
5692 constraint(ALLOC_IN_RC(ptr_reg));
5693 match(AddP reg off);
5694 op_cost(0);
5695 format %{ "[$reg, $off]" %}
5696 interface(MEMORY_INTER) %{
5697 base($reg);
5698 index(0xffffffff);
5699 scale(0x0);
5700 disp($off);
5701 %}
5702 %}
5703
5704 operand indOffI4(iRegP reg, immIOffset4 off)
5705 %{
5706 constraint(ALLOC_IN_RC(ptr_reg));
5707 match(AddP reg off);
5708 op_cost(0);
5709 format %{ "[$reg, $off]" %}
5710 interface(MEMORY_INTER) %{
5711 base($reg);
5712 index(0xffffffff);
5713 scale(0x0);
5714 disp($off);
5715 %}
5716 %}
5717
5718 operand indOffI8(iRegP reg, immIOffset8 off)
5719 %{
5720 constraint(ALLOC_IN_RC(ptr_reg));
5721 match(AddP reg off);
5722 op_cost(0);
5723 format %{ "[$reg, $off]" %}
5724 interface(MEMORY_INTER) %{
5725 base($reg);
5726 index(0xffffffff);
5727 scale(0x0);
5728 disp($off);
5729 %}
5730 %}
5731
5732 operand indOffI16(iRegP reg, immIOffset16 off)
5733 %{
5734 constraint(ALLOC_IN_RC(ptr_reg));
5735 match(AddP reg off);
5736 op_cost(0);
5737 format %{ "[$reg, $off]" %}
5738 interface(MEMORY_INTER) %{
5739 base($reg);
5740 index(0xffffffff);
5741 scale(0x0);
5742 disp($off);
5743 %}
5744 %}
5745
5746 operand indOffL(iRegP reg, immLoffset off)
5747 %{
5748 constraint(ALLOC_IN_RC(ptr_reg));
5749 match(AddP reg off);
5750 op_cost(0);
5751 format %{ "[$reg, $off]" %}
5752 interface(MEMORY_INTER) %{
5753 base($reg);
5754 index(0xffffffff);
5755 scale(0x0);
5756 disp($off);
5757 %}
5758 %}
5759
5760 operand indOffL1(iRegP reg, immLoffset1 off)
5761 %{
5762 constraint(ALLOC_IN_RC(ptr_reg));
5763 match(AddP reg off);
5764 op_cost(0);
5765 format %{ "[$reg, $off]" %}
5766 interface(MEMORY_INTER) %{
5767 base($reg);
5768 index(0xffffffff);
5769 scale(0x0);
5770 disp($off);
5771 %}
5772 %}
5773
5774 operand indOffL2(iRegP reg, immLoffset2 off)
5775 %{
5776 constraint(ALLOC_IN_RC(ptr_reg));
5777 match(AddP reg off);
5778 op_cost(0);
5779 format %{ "[$reg, $off]" %}
5780 interface(MEMORY_INTER) %{
5781 base($reg);
5782 index(0xffffffff);
5783 scale(0x0);
5784 disp($off);
5785 %}
5786 %}
5787
5788 operand indOffL4(iRegP reg, immLoffset4 off)
5789 %{
5790 constraint(ALLOC_IN_RC(ptr_reg));
5791 match(AddP reg off);
5792 op_cost(0);
5793 format %{ "[$reg, $off]" %}
5794 interface(MEMORY_INTER) %{
5795 base($reg);
5796 index(0xffffffff);
5797 scale(0x0);
5798 disp($off);
5799 %}
5800 %}
5801
5802 operand indOffL8(iRegP reg, immLoffset8 off)
5803 %{
5804 constraint(ALLOC_IN_RC(ptr_reg));
5805 match(AddP reg off);
5806 op_cost(0);
5807 format %{ "[$reg, $off]" %}
5808 interface(MEMORY_INTER) %{
5809 base($reg);
5810 index(0xffffffff);
5811 scale(0x0);
5812 disp($off);
5813 %}
5814 %}
5815
5816 operand indOffL16(iRegP reg, immLoffset16 off)
5817 %{
5818 constraint(ALLOC_IN_RC(ptr_reg));
5819 match(AddP reg off);
5820 op_cost(0);
5821 format %{ "[$reg, $off]" %}
5822 interface(MEMORY_INTER) %{
5823 base($reg);
5824 index(0xffffffff);
5825 scale(0x0);
5826 disp($off);
5827 %}
5828 %}
5829
5830 operand indirectN(iRegN reg)
5831 %{
5832 predicate(CompressedOops::shift() == 0);
5833 constraint(ALLOC_IN_RC(ptr_reg));
5834 match(DecodeN reg);
5835 op_cost(0);
5836 format %{ "[$reg]\t# narrow" %}
5837 interface(MEMORY_INTER) %{
5838 base($reg);
5839 index(0xffffffff);
5840 scale(0x0);
5841 disp(0x0);
5842 %}
5843 %}
5844
5845 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5846 %{
5847 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5848 constraint(ALLOC_IN_RC(ptr_reg));
5849 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5850 op_cost(0);
5851 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5852 interface(MEMORY_INTER) %{
5853 base($reg);
5854 index($ireg);
5855 scale($scale);
5856 disp(0x0);
5857 %}
5858 %}
5859
5860 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5861 %{
5862 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5863 constraint(ALLOC_IN_RC(ptr_reg));
5864 match(AddP (DecodeN reg) (LShiftL lreg scale));
5865 op_cost(0);
5866 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5867 interface(MEMORY_INTER) %{
5868 base($reg);
5869 index($lreg);
5870 scale($scale);
5871 disp(0x0);
5872 %}
5873 %}
5874
5875 operand indIndexI2LN(iRegN reg, iRegI ireg)
5876 %{
5877 predicate(CompressedOops::shift() == 0);
5878 constraint(ALLOC_IN_RC(ptr_reg));
5879 match(AddP (DecodeN reg) (ConvI2L ireg));
5880 op_cost(0);
5881 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5882 interface(MEMORY_INTER) %{
5883 base($reg);
5884 index($ireg);
5885 scale(0x0);
5886 disp(0x0);
5887 %}
5888 %}
5889
5890 operand indIndexN(iRegN reg, iRegL lreg)
5891 %{
5892 predicate(CompressedOops::shift() == 0);
5893 constraint(ALLOC_IN_RC(ptr_reg));
5894 match(AddP (DecodeN reg) lreg);
5895 op_cost(0);
5896 format %{ "$reg, $lreg\t# narrow" %}
5897 interface(MEMORY_INTER) %{
5898 base($reg);
5899 index($lreg);
5900 scale(0x0);
5901 disp(0x0);
5902 %}
5903 %}
5904
5905 operand indOffIN(iRegN reg, immIOffset off)
5906 %{
5907 predicate(CompressedOops::shift() == 0);
5908 constraint(ALLOC_IN_RC(ptr_reg));
5909 match(AddP (DecodeN reg) off);
5910 op_cost(0);
5911 format %{ "[$reg, $off]\t# narrow" %}
5912 interface(MEMORY_INTER) %{
5913 base($reg);
5914 index(0xffffffff);
5915 scale(0x0);
5916 disp($off);
5917 %}
5918 %}
5919
5920 operand indOffLN(iRegN reg, immLoffset off)
5921 %{
5922 predicate(CompressedOops::shift() == 0);
5923 constraint(ALLOC_IN_RC(ptr_reg));
5924 match(AddP (DecodeN reg) off);
5925 op_cost(0);
5926 format %{ "[$reg, $off]\t# narrow" %}
5927 interface(MEMORY_INTER) %{
5928 base($reg);
5929 index(0xffffffff);
5930 scale(0x0);
5931 disp($off);
5932 %}
5933 %}
5934
5935
5936
5937 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5938 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5939 %{
5940 constraint(ALLOC_IN_RC(ptr_reg));
5941 match(AddP reg off);
5942 op_cost(0);
5943 format %{ "[$reg, $off]" %}
5944 interface(MEMORY_INTER) %{
5945 base($reg);
5946 index(0xffffffff);
5947 scale(0x0);
5948 disp($off);
5949 %}
5950 %}
5951
5952 //----------Special Memory Operands--------------------------------------------
5953 // Stack Slot Operand - This operand is used for loading and storing temporary
5954 // values on the stack where a match requires a value to
5955 // flow through memory.
5956 operand stackSlotP(sRegP reg)
5957 %{
5958 constraint(ALLOC_IN_RC(stack_slots));
5959 op_cost(100);
5960 // No match rule because this operand is only generated in matching
5961 // match(RegP);
5962 format %{ "[$reg]" %}
5963 interface(MEMORY_INTER) %{
5964 base(0x1e); // RSP
5965 index(0x0); // No Index
5966 scale(0x0); // No Scale
5967 disp($reg); // Stack Offset
5968 %}
5969 %}
5970
5971 operand stackSlotI(sRegI reg)
5972 %{
5973 constraint(ALLOC_IN_RC(stack_slots));
5974 // No match rule because this operand is only generated in matching
5975 // match(RegI);
5976 format %{ "[$reg]" %}
5977 interface(MEMORY_INTER) %{
5978 base(0x1e); // RSP
5979 index(0x0); // No Index
5980 scale(0x0); // No Scale
5981 disp($reg); // Stack Offset
5982 %}
5983 %}
5984
5985 operand stackSlotF(sRegF reg)
5986 %{
5987 constraint(ALLOC_IN_RC(stack_slots));
5988 // No match rule because this operand is only generated in matching
5989 // match(RegF);
5990 format %{ "[$reg]" %}
5991 interface(MEMORY_INTER) %{
5992 base(0x1e); // RSP
5993 index(0x0); // No Index
5994 scale(0x0); // No Scale
5995 disp($reg); // Stack Offset
5996 %}
5997 %}
5998
5999 operand stackSlotD(sRegD reg)
6000 %{
6001 constraint(ALLOC_IN_RC(stack_slots));
6002 // No match rule because this operand is only generated in matching
6003 // match(RegD);
6004 format %{ "[$reg]" %}
6005 interface(MEMORY_INTER) %{
6006 base(0x1e); // RSP
6007 index(0x0); // No Index
6008 scale(0x0); // No Scale
6009 disp($reg); // Stack Offset
6010 %}
6011 %}
6012
6013 operand stackSlotL(sRegL reg)
6014 %{
6015 constraint(ALLOC_IN_RC(stack_slots));
6016 // No match rule because this operand is only generated in matching
6017 // match(RegL);
6018 format %{ "[$reg]" %}
6019 interface(MEMORY_INTER) %{
6020 base(0x1e); // RSP
6021 index(0x0); // No Index
6022 scale(0x0); // No Scale
6023 disp($reg); // Stack Offset
6024 %}
6025 %}
6026
6027 // Operands for expressing Control Flow
6028 // NOTE: Label is a predefined operand which should not be redefined in
6029 // the AD file. It is generically handled within the ADLC.
6030
6031 //----------Conditional Branch Operands----------------------------------------
6032 // Comparison Op - This is the operation of the comparison, and is limited to
6033 // the following set of codes:
6034 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6035 //
6036 // Other attributes of the comparison, such as unsignedness, are specified
6037 // by the comparison instruction that sets a condition code flags register.
6038 // That result is represented by a flags operand whose subtype is appropriate
6039 // to the unsignedness (etc.) of the comparison.
6040 //
6041 // Later, the instruction which matches both the Comparison Op (a Bool) and
6042 // the flags (produced by the Cmp) specifies the coding of the comparison op
6043 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6044
6045 // used for signed integral comparisons and fp comparisons
6046
6047 operand cmpOp()
6048 %{
6049 match(Bool);
6050
6051 format %{ "" %}
6052 interface(COND_INTER) %{
6053 equal(0x0, "eq");
6054 not_equal(0x1, "ne");
6055 less(0xb, "lt");
6056 greater_equal(0xa, "ge");
6057 less_equal(0xd, "le");
6058 greater(0xc, "gt");
6059 overflow(0x6, "vs");
6060 no_overflow(0x7, "vc");
6061 %}
6062 %}
6063
6064 // used for unsigned integral comparisons
6065
6066 operand cmpOpU()
6067 %{
6068 match(Bool);
6069
6070 format %{ "" %}
6071 interface(COND_INTER) %{
6072 equal(0x0, "eq");
6073 not_equal(0x1, "ne");
6074 less(0x3, "lo");
6075 greater_equal(0x2, "hs");
6076 less_equal(0x9, "ls");
6077 greater(0x8, "hi");
6078 overflow(0x6, "vs");
6079 no_overflow(0x7, "vc");
6080 %}
6081 %}
6082
6083 // used for certain integral comparisons which can be
6084 // converted to cbxx or tbxx instructions
6085
6086 operand cmpOpEqNe()
6087 %{
6088 match(Bool);
6089 op_cost(0);
6090 predicate(n->as_Bool()->_test._test == BoolTest::ne
6091 || n->as_Bool()->_test._test == BoolTest::eq);
6092
6093 format %{ "" %}
6094 interface(COND_INTER) %{
6095 equal(0x0, "eq");
6096 not_equal(0x1, "ne");
6097 less(0xb, "lt");
6098 greater_equal(0xa, "ge");
6099 less_equal(0xd, "le");
6100 greater(0xc, "gt");
6101 overflow(0x6, "vs");
6102 no_overflow(0x7, "vc");
6103 %}
6104 %}
6105
6106 // used for certain integral comparisons which can be
6107 // converted to cbxx or tbxx instructions
6108
6109 operand cmpOpLtGe()
6110 %{
6111 match(Bool);
6112 op_cost(0);
6113
6114 predicate(n->as_Bool()->_test._test == BoolTest::lt
6115 || n->as_Bool()->_test._test == BoolTest::ge);
6116
6117 format %{ "" %}
6118 interface(COND_INTER) %{
6119 equal(0x0, "eq");
6120 not_equal(0x1, "ne");
6121 less(0xb, "lt");
6122 greater_equal(0xa, "ge");
6123 less_equal(0xd, "le");
6124 greater(0xc, "gt");
6125 overflow(0x6, "vs");
6126 no_overflow(0x7, "vc");
6127 %}
6128 %}
6129
6130 // used for certain unsigned integral comparisons which can be
6131 // converted to cbxx or tbxx instructions
6132
6133 operand cmpOpUEqNeLtGe()
6134 %{
6135 match(Bool);
6136 op_cost(0);
6137
6138 predicate(n->as_Bool()->_test._test == BoolTest::eq
6139 || n->as_Bool()->_test._test == BoolTest::ne
6140 || n->as_Bool()->_test._test == BoolTest::lt
6141 || n->as_Bool()->_test._test == BoolTest::ge);
6142
6143 format %{ "" %}
6144 interface(COND_INTER) %{
6145 equal(0x0, "eq");
6146 not_equal(0x1, "ne");
6147 less(0xb, "lt");
6148 greater_equal(0xa, "ge");
6149 less_equal(0xd, "le");
6150 greater(0xc, "gt");
6151 overflow(0x6, "vs");
6152 no_overflow(0x7, "vc");
6153 %}
6154 %}
6155
6156 // Special operand allowing long args to int ops to be truncated for free
6157
6158 operand iRegL2I(iRegL reg) %{
6159
6160 op_cost(0);
6161
6162 match(ConvL2I reg);
6163
6164 format %{ "l2i($reg)" %}
6165
6166 interface(REG_INTER)
6167 %}
6168
6169 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6170 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6171 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6172 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6173
6174 //----------OPERAND CLASSES----------------------------------------------------
6175 // Operand Classes are groups of operands that are used as to simplify
6176 // instruction definitions by not requiring the AD writer to specify
6177 // separate instructions for every form of operand when the
6178 // instruction accepts multiple operand types with the same basic
6179 // encoding and format. The classic case of this is memory operands.
6180
6181 // memory is used to define read/write location for load/store
6182 // instruction defs. we can turn a memory op into an Address
6183
6184 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6185 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6186
6187 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6188 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6189
6190 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6191 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6192
6193 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6194 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6195
6196 // All of the memory operands. For the pipeline description.
6197 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6198 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6199 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6200
6201
6202 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6203 // operations. it allows the src to be either an iRegI or a (ConvL2I
6204 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6205 // can be elided because the 32-bit instruction will just employ the
6206 // lower 32 bits anyway.
6207 //
6208 // n.b. this does not elide all L2I conversions. if the truncated
6209 // value is consumed by more than one operation then the ConvL2I
6210 // cannot be bundled into the consuming nodes so an l2i gets planted
6211 // (actually a movw $dst $src) and the downstream instructions consume
6212 // the result of the l2i as an iRegI input. That's a shame since the
6213 // movw is actually redundant but its not too costly.
6214
6215 opclass iRegIorL2I(iRegI, iRegL2I);
6216
6217 //----------PIPELINE-----------------------------------------------------------
6218 // Rules which define the behavior of the target architectures pipeline.
6219
6220 // For specific pipelines, eg A53, define the stages of that pipeline
6221 //pipe_desc(ISS, EX1, EX2, WR);
6222 #define ISS S0
6223 #define EX1 S1
6224 #define EX2 S2
6225 #define WR S3
6226
6227 // Integer ALU reg operation
6228 pipeline %{
6229
6230 attributes %{
6231 // ARM instructions are of fixed length
6232 fixed_size_instructions; // Fixed size instructions TODO does
6233 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6234 // ARM instructions come in 32-bit word units
6235 instruction_unit_size = 4; // An instruction is 4 bytes long
6236 instruction_fetch_unit_size = 64; // The processor fetches one line
6237 instruction_fetch_units = 1; // of 64 bytes
6238
6239 // List of nop instructions
6240 nops( MachNop );
6241 %}
6242
6243 // We don't use an actual pipeline model so don't care about resources
6244 // or description. we do use pipeline classes to introduce fixed
6245 // latencies
6246
6247 //----------RESOURCES----------------------------------------------------------
6248 // Resources are the functional units available to the machine
6249
6250 resources( INS0, INS1, INS01 = INS0 | INS1,
6251 ALU0, ALU1, ALU = ALU0 | ALU1,
6252 MAC,
6253 DIV,
6254 BRANCH,
6255 LDST,
6256 NEON_FP);
6257
6258 //----------PIPELINE DESCRIPTION-----------------------------------------------
6259 // Pipeline Description specifies the stages in the machine's pipeline
6260
6261 // Define the pipeline as a generic 6 stage pipeline
6262 pipe_desc(S0, S1, S2, S3, S4, S5);
6263
6264 //----------PIPELINE CLASSES---------------------------------------------------
6265 // Pipeline Classes describe the stages in which input and output are
6266 // referenced by the hardware pipeline.
6267
6268 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6269 %{
6270 single_instruction;
6271 src1 : S1(read);
6272 src2 : S2(read);
6273 dst : S5(write);
6274 INS01 : ISS;
6275 NEON_FP : S5;
6276 %}
6277
6278 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6279 %{
6280 single_instruction;
6281 src1 : S1(read);
6282 src2 : S2(read);
6283 dst : S5(write);
6284 INS01 : ISS;
6285 NEON_FP : S5;
6286 %}
6287
6288 pipe_class fp_uop_s(vRegF dst, vRegF src)
6289 %{
6290 single_instruction;
6291 src : S1(read);
6292 dst : S5(write);
6293 INS01 : ISS;
6294 NEON_FP : S5;
6295 %}
6296
6297 pipe_class fp_uop_d(vRegD dst, vRegD src)
6298 %{
6299 single_instruction;
6300 src : S1(read);
6301 dst : S5(write);
6302 INS01 : ISS;
6303 NEON_FP : S5;
6304 %}
6305
6306 pipe_class fp_d2f(vRegF dst, vRegD src)
6307 %{
6308 single_instruction;
6309 src : S1(read);
6310 dst : S5(write);
6311 INS01 : ISS;
6312 NEON_FP : S5;
6313 %}
6314
6315 pipe_class fp_f2d(vRegD dst, vRegF src)
6316 %{
6317 single_instruction;
6318 src : S1(read);
6319 dst : S5(write);
6320 INS01 : ISS;
6321 NEON_FP : S5;
6322 %}
6323
6324 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6325 %{
6326 single_instruction;
6327 src : S1(read);
6328 dst : S5(write);
6329 INS01 : ISS;
6330 NEON_FP : S5;
6331 %}
6332
6333 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6334 %{
6335 single_instruction;
6336 src : S1(read);
6337 dst : S5(write);
6338 INS01 : ISS;
6339 NEON_FP : S5;
6340 %}
6341
6342 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6343 %{
6344 single_instruction;
6345 src : S1(read);
6346 dst : S5(write);
6347 INS01 : ISS;
6348 NEON_FP : S5;
6349 %}
6350
6351 pipe_class fp_l2f(vRegF dst, iRegL src)
6352 %{
6353 single_instruction;
6354 src : S1(read);
6355 dst : S5(write);
6356 INS01 : ISS;
6357 NEON_FP : S5;
6358 %}
6359
6360 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6361 %{
6362 single_instruction;
6363 src : S1(read);
6364 dst : S5(write);
6365 INS01 : ISS;
6366 NEON_FP : S5;
6367 %}
6368
6369 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6370 %{
6371 single_instruction;
6372 src : S1(read);
6373 dst : S5(write);
6374 INS01 : ISS;
6375 NEON_FP : S5;
6376 %}
6377
6378 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6379 %{
6380 single_instruction;
6381 src : S1(read);
6382 dst : S5(write);
6383 INS01 : ISS;
6384 NEON_FP : S5;
6385 %}
6386
6387 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6388 %{
6389 single_instruction;
6390 src : S1(read);
6391 dst : S5(write);
6392 INS01 : ISS;
6393 NEON_FP : S5;
6394 %}
6395
6396 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6397 %{
6398 single_instruction;
6399 src1 : S1(read);
6400 src2 : S2(read);
6401 dst : S5(write);
6402 INS0 : ISS;
6403 NEON_FP : S5;
6404 %}
6405
6406 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6407 %{
6408 single_instruction;
6409 src1 : S1(read);
6410 src2 : S2(read);
6411 dst : S5(write);
6412 INS0 : ISS;
6413 NEON_FP : S5;
6414 %}
6415
6416 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6417 %{
6418 single_instruction;
6419 cr : S1(read);
6420 src1 : S1(read);
6421 src2 : S1(read);
6422 dst : S3(write);
6423 INS01 : ISS;
6424 NEON_FP : S3;
6425 %}
6426
6427 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6428 %{
6429 single_instruction;
6430 cr : S1(read);
6431 src1 : S1(read);
6432 src2 : S1(read);
6433 dst : S3(write);
6434 INS01 : ISS;
6435 NEON_FP : S3;
6436 %}
6437
6438 pipe_class fp_imm_s(vRegF dst)
6439 %{
6440 single_instruction;
6441 dst : S3(write);
6442 INS01 : ISS;
6443 NEON_FP : S3;
6444 %}
6445
6446 pipe_class fp_imm_d(vRegD dst)
6447 %{
6448 single_instruction;
6449 dst : S3(write);
6450 INS01 : ISS;
6451 NEON_FP : S3;
6452 %}
6453
6454 pipe_class fp_load_constant_s(vRegF dst)
6455 %{
6456 single_instruction;
6457 dst : S4(write);
6458 INS01 : ISS;
6459 NEON_FP : S4;
6460 %}
6461
6462 pipe_class fp_load_constant_d(vRegD dst)
6463 %{
6464 single_instruction;
6465 dst : S4(write);
6466 INS01 : ISS;
6467 NEON_FP : S4;
6468 %}
6469
6470 //------- Integer ALU operations --------------------------
6471
6472 // Integer ALU reg-reg operation
6473 // Operands needed in EX1, result generated in EX2
6474 // Eg. ADD x0, x1, x2
6475 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6476 %{
6477 single_instruction;
6478 dst : EX2(write);
6479 src1 : EX1(read);
6480 src2 : EX1(read);
6481 INS01 : ISS; // Dual issue as instruction 0 or 1
6482 ALU : EX2;
6483 %}
6484
6485 // Integer ALU reg-reg operation with constant shift
6486 // Shifted register must be available in LATE_ISS instead of EX1
6487 // Eg. ADD x0, x1, x2, LSL #2
6488 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6489 %{
6490 single_instruction;
6491 dst : EX2(write);
6492 src1 : EX1(read);
6493 src2 : ISS(read);
6494 INS01 : ISS;
6495 ALU : EX2;
6496 %}
6497
6498 // Integer ALU reg operation with constant shift
6499 // Eg. LSL x0, x1, #shift
6500 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6501 %{
6502 single_instruction;
6503 dst : EX2(write);
6504 src1 : ISS(read);
6505 INS01 : ISS;
6506 ALU : EX2;
6507 %}
6508
6509 // Integer ALU reg-reg operation with variable shift
6510 // Both operands must be available in LATE_ISS instead of EX1
6511 // Result is available in EX1 instead of EX2
6512 // Eg. LSLV x0, x1, x2
6513 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6514 %{
6515 single_instruction;
6516 dst : EX1(write);
6517 src1 : ISS(read);
6518 src2 : ISS(read);
6519 INS01 : ISS;
6520 ALU : EX1;
6521 %}
6522
6523 // Integer ALU reg-reg operation with extract
6524 // As for _vshift above, but result generated in EX2
6525 // Eg. EXTR x0, x1, x2, #N
6526 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6527 %{
6528 single_instruction;
6529 dst : EX2(write);
6530 src1 : ISS(read);
6531 src2 : ISS(read);
6532 INS1 : ISS; // Can only dual issue as Instruction 1
6533 ALU : EX1;
6534 %}
6535
6536 // Integer ALU reg operation
6537 // Eg. NEG x0, x1
6538 pipe_class ialu_reg(iRegI dst, iRegI src)
6539 %{
6540 single_instruction;
6541 dst : EX2(write);
6542 src : EX1(read);
6543 INS01 : ISS;
6544 ALU : EX2;
6545 %}
6546
6547 // Integer ALU reg mmediate operation
6548 // Eg. ADD x0, x1, #N
6549 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6550 %{
6551 single_instruction;
6552 dst : EX2(write);
6553 src1 : EX1(read);
6554 INS01 : ISS;
6555 ALU : EX2;
6556 %}
6557
6558 // Integer ALU immediate operation (no source operands)
6559 // Eg. MOV x0, #N
6560 pipe_class ialu_imm(iRegI dst)
6561 %{
6562 single_instruction;
6563 dst : EX1(write);
6564 INS01 : ISS;
6565 ALU : EX1;
6566 %}
6567
6568 //------- Compare operation -------------------------------
6569
6570 // Compare reg-reg
6571 // Eg. CMP x0, x1
6572 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6573 %{
6574 single_instruction;
6575 // fixed_latency(16);
6576 cr : EX2(write);
6577 op1 : EX1(read);
6578 op2 : EX1(read);
6579 INS01 : ISS;
6580 ALU : EX2;
6581 %}
6582
6583 // Compare reg-reg
6584 // Eg. CMP x0, #N
6585 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6586 %{
6587 single_instruction;
6588 // fixed_latency(16);
6589 cr : EX2(write);
6590 op1 : EX1(read);
6591 INS01 : ISS;
6592 ALU : EX2;
6593 %}
6594
6595 //------- Conditional instructions ------------------------
6596
6597 // Conditional no operands
6598 // Eg. CSINC x0, zr, zr, <cond>
6599 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6600 %{
6601 single_instruction;
6602 cr : EX1(read);
6603 dst : EX2(write);
6604 INS01 : ISS;
6605 ALU : EX2;
6606 %}
6607
6608 // Conditional 2 operand
6609 // EG. CSEL X0, X1, X2, <cond>
6610 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6611 %{
6612 single_instruction;
6613 cr : EX1(read);
6614 src1 : EX1(read);
6615 src2 : EX1(read);
6616 dst : EX2(write);
6617 INS01 : ISS;
6618 ALU : EX2;
6619 %}
6620
6621 // Conditional 2 operand
6622 // EG. CSEL X0, X1, X2, <cond>
6623 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6624 %{
6625 single_instruction;
6626 cr : EX1(read);
6627 src : EX1(read);
6628 dst : EX2(write);
6629 INS01 : ISS;
6630 ALU : EX2;
6631 %}
6632
6633 //------- Multiply pipeline operations --------------------
6634
6635 // Multiply reg-reg
6636 // Eg. MUL w0, w1, w2
6637 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6638 %{
6639 single_instruction;
6640 dst : WR(write);
6641 src1 : ISS(read);
6642 src2 : ISS(read);
6643 INS01 : ISS;
6644 MAC : WR;
6645 %}
6646
6647 // Multiply accumulate
6648 // Eg. MADD w0, w1, w2, w3
6649 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6650 %{
6651 single_instruction;
6652 dst : WR(write);
6653 src1 : ISS(read);
6654 src2 : ISS(read);
6655 src3 : ISS(read);
6656 INS01 : ISS;
6657 MAC : WR;
6658 %}
6659
6660 // Eg. MUL w0, w1, w2
6661 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6662 %{
6663 single_instruction;
6664 fixed_latency(3); // Maximum latency for 64 bit mul
6665 dst : WR(write);
6666 src1 : ISS(read);
6667 src2 : ISS(read);
6668 INS01 : ISS;
6669 MAC : WR;
6670 %}
6671
6672 // Multiply accumulate
6673 // Eg. MADD w0, w1, w2, w3
6674 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6675 %{
6676 single_instruction;
6677 fixed_latency(3); // Maximum latency for 64 bit mul
6678 dst : WR(write);
6679 src1 : ISS(read);
6680 src2 : ISS(read);
6681 src3 : ISS(read);
6682 INS01 : ISS;
6683 MAC : WR;
6684 %}
6685
6686 //------- Divide pipeline operations --------------------
6687
6688 // Eg. SDIV w0, w1, w2
6689 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6690 %{
6691 single_instruction;
6692 fixed_latency(8); // Maximum latency for 32 bit divide
6693 dst : WR(write);
6694 src1 : ISS(read);
6695 src2 : ISS(read);
6696 INS0 : ISS; // Can only dual issue as instruction 0
6697 DIV : WR;
6698 %}
6699
6700 // Eg. SDIV x0, x1, x2
6701 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6702 %{
6703 single_instruction;
6704 fixed_latency(16); // Maximum latency for 64 bit divide
6705 dst : WR(write);
6706 src1 : ISS(read);
6707 src2 : ISS(read);
6708 INS0 : ISS; // Can only dual issue as instruction 0
6709 DIV : WR;
6710 %}
6711
6712 //------- Load pipeline operations ------------------------
6713
6714 // Load - prefetch
6715 // Eg. PFRM <mem>
6716 pipe_class iload_prefetch(memory mem)
6717 %{
6718 single_instruction;
6719 mem : ISS(read);
6720 INS01 : ISS;
6721 LDST : WR;
6722 %}
6723
6724 // Load - reg, mem
6725 // Eg. LDR x0, <mem>
6726 pipe_class iload_reg_mem(iRegI dst, memory mem)
6727 %{
6728 single_instruction;
6729 dst : WR(write);
6730 mem : ISS(read);
6731 INS01 : ISS;
6732 LDST : WR;
6733 %}
6734
6735 // Load - reg, reg
6736 // Eg. LDR x0, [sp, x1]
6737 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6738 %{
6739 single_instruction;
6740 dst : WR(write);
6741 src : ISS(read);
6742 INS01 : ISS;
6743 LDST : WR;
6744 %}
6745
6746 //------- Store pipeline operations -----------------------
6747
6748 // Store - zr, mem
6749 // Eg. STR zr, <mem>
6750 pipe_class istore_mem(memory mem)
6751 %{
6752 single_instruction;
6753 mem : ISS(read);
6754 INS01 : ISS;
6755 LDST : WR;
6756 %}
6757
6758 // Store - reg, mem
6759 // Eg. STR x0, <mem>
6760 pipe_class istore_reg_mem(iRegI src, memory mem)
6761 %{
6762 single_instruction;
6763 mem : ISS(read);
6764 src : EX2(read);
6765 INS01 : ISS;
6766 LDST : WR;
6767 %}
6768
6769 // Store - reg, reg
6770 // Eg. STR x0, [sp, x1]
6771 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6772 %{
6773 single_instruction;
6774 dst : ISS(read);
6775 src : EX2(read);
6776 INS01 : ISS;
6777 LDST : WR;
6778 %}
6779
6780 //------- Store pipeline operations -----------------------
6781
6782 // Branch
6783 pipe_class pipe_branch()
6784 %{
6785 single_instruction;
6786 INS01 : ISS;
6787 BRANCH : EX1;
6788 %}
6789
6790 // Conditional branch
6791 pipe_class pipe_branch_cond(rFlagsReg cr)
6792 %{
6793 single_instruction;
6794 cr : EX1(read);
6795 INS01 : ISS;
6796 BRANCH : EX1;
6797 %}
6798
6799 // Compare & Branch
6800 // EG. CBZ/CBNZ
6801 pipe_class pipe_cmp_branch(iRegI op1)
6802 %{
6803 single_instruction;
6804 op1 : EX1(read);
6805 INS01 : ISS;
6806 BRANCH : EX1;
6807 %}
6808
6809 //------- Synchronisation operations ----------------------
6810
6811 // Any operation requiring serialization.
6812 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6813 pipe_class pipe_serial()
6814 %{
6815 single_instruction;
6816 force_serialization;
6817 fixed_latency(16);
6818 INS01 : ISS(2); // Cannot dual issue with any other instruction
6819 LDST : WR;
6820 %}
6821
6822 // Generic big/slow expanded idiom - also serialized
6823 pipe_class pipe_slow()
6824 %{
6825 instruction_count(10);
6826 multiple_bundles;
6827 force_serialization;
6828 fixed_latency(16);
6829 INS01 : ISS(2); // Cannot dual issue with any other instruction
6830 LDST : WR;
6831 %}
6832
6833 // Empty pipeline class
6834 pipe_class pipe_class_empty()
6835 %{
6836 single_instruction;
6837 fixed_latency(0);
6838 %}
6839
6840 // Default pipeline class.
6841 pipe_class pipe_class_default()
6842 %{
6843 single_instruction;
6844 fixed_latency(2);
6845 %}
6846
6847 // Pipeline class for compares.
6848 pipe_class pipe_class_compare()
6849 %{
6850 single_instruction;
6851 fixed_latency(16);
6852 %}
6853
6854 // Pipeline class for memory operations.
6855 pipe_class pipe_class_memory()
6856 %{
6857 single_instruction;
6858 fixed_latency(16);
6859 %}
6860
6861 // Pipeline class for call.
6862 pipe_class pipe_class_call()
6863 %{
6864 single_instruction;
6865 fixed_latency(100);
6866 %}
6867
6868 // Define the class for the Nop node.
6869 define %{
6870 MachNop = pipe_class_empty;
6871 %}
6872
6873 %}
6874 //----------INSTRUCTIONS-------------------------------------------------------
6875 //
6876 // match -- States which machine-independent subtree may be replaced
6877 // by this instruction.
6878 // ins_cost -- The estimated cost of this instruction is used by instruction
6879 // selection to identify a minimum cost tree of machine
6880 // instructions that matches a tree of machine-independent
6881 // instructions.
6882 // format -- A string providing the disassembly for this instruction.
6883 // The value of an instruction's operand may be inserted
6884 // by referring to it with a '$' prefix.
6885 // opcode -- Three instruction opcodes may be provided. These are referred
6886 // to within an encode class as $primary, $secondary, and $tertiary
6887 // rrspectively. The primary opcode is commonly used to
6888 // indicate the type of machine instruction, while secondary
6889 // and tertiary are often used for prefix options or addressing
6890 // modes.
6891 // ins_encode -- A list of encode classes with parameters. The encode class
6892 // name must have been defined in an 'enc_class' specification
6893 // in the encode section of the architecture description.
6894
6895 // ============================================================================
6896 // Memory (Load/Store) Instructions
6897
6898 // Load Instructions
6899
6900 // Load Byte (8 bit signed)
6901 instruct loadB(iRegINoSp dst, memory1 mem)
6902 %{
6903 match(Set dst (LoadB mem));
6904 predicate(!needs_acquiring_load(n));
6905
6906 ins_cost(4 * INSN_COST);
6907 format %{ "ldrsbw $dst, $mem\t# byte" %}
6908
6909 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6910
6911 ins_pipe(iload_reg_mem);
6912 %}
6913
6914 // Load Byte (8 bit signed) into long
6915 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6916 %{
6917 match(Set dst (ConvI2L (LoadB mem)));
6918 predicate(!needs_acquiring_load(n->in(1)));
6919
6920 ins_cost(4 * INSN_COST);
6921 format %{ "ldrsb $dst, $mem\t# byte" %}
6922
6923 ins_encode(aarch64_enc_ldrsb(dst, mem));
6924
6925 ins_pipe(iload_reg_mem);
6926 %}
6927
6928 // Load Byte (8 bit unsigned)
6929 instruct loadUB(iRegINoSp dst, memory1 mem)
6930 %{
6931 match(Set dst (LoadUB mem));
6932 predicate(!needs_acquiring_load(n));
6933
6934 ins_cost(4 * INSN_COST);
6935 format %{ "ldrbw $dst, $mem\t# byte" %}
6936
6937 ins_encode(aarch64_enc_ldrb(dst, mem));
6938
6939 ins_pipe(iload_reg_mem);
6940 %}
6941
6942 // Load Byte (8 bit unsigned) into long
6943 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6944 %{
6945 match(Set dst (ConvI2L (LoadUB mem)));
6946 predicate(!needs_acquiring_load(n->in(1)));
6947
6948 ins_cost(4 * INSN_COST);
6949 format %{ "ldrb $dst, $mem\t# byte" %}
6950
6951 ins_encode(aarch64_enc_ldrb(dst, mem));
6952
6953 ins_pipe(iload_reg_mem);
6954 %}
6955
6956 // Load Short (16 bit signed)
6957 instruct loadS(iRegINoSp dst, memory2 mem)
6958 %{
6959 match(Set dst (LoadS mem));
6960 predicate(!needs_acquiring_load(n));
6961
6962 ins_cost(4 * INSN_COST);
6963 format %{ "ldrshw $dst, $mem\t# short" %}
6964
6965 ins_encode(aarch64_enc_ldrshw(dst, mem));
6966
6967 ins_pipe(iload_reg_mem);
6968 %}
6969
6970 // Load Short (16 bit signed) into long
6971 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6972 %{
6973 match(Set dst (ConvI2L (LoadS mem)));
6974 predicate(!needs_acquiring_load(n->in(1)));
6975
6976 ins_cost(4 * INSN_COST);
6977 format %{ "ldrsh $dst, $mem\t# short" %}
6978
6979 ins_encode(aarch64_enc_ldrsh(dst, mem));
6980
6981 ins_pipe(iload_reg_mem);
6982 %}
6983
6984 // Load Char (16 bit unsigned)
6985 instruct loadUS(iRegINoSp dst, memory2 mem)
6986 %{
6987 match(Set dst (LoadUS mem));
6988 predicate(!needs_acquiring_load(n));
6989
6990 ins_cost(4 * INSN_COST);
6991 format %{ "ldrh $dst, $mem\t# short" %}
6992
6993 ins_encode(aarch64_enc_ldrh(dst, mem));
6994
6995 ins_pipe(iload_reg_mem);
6996 %}
6997
6998 // Load Short/Char (16 bit unsigned) into long
6999 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7000 %{
7001 match(Set dst (ConvI2L (LoadUS mem)));
7002 predicate(!needs_acquiring_load(n->in(1)));
7003
7004 ins_cost(4 * INSN_COST);
7005 format %{ "ldrh $dst, $mem\t# short" %}
7006
7007 ins_encode(aarch64_enc_ldrh(dst, mem));
7008
7009 ins_pipe(iload_reg_mem);
7010 %}
7011
7012 // Load Integer (32 bit signed)
7013 instruct loadI(iRegINoSp dst, memory4 mem)
7014 %{
7015 match(Set dst (LoadI mem));
7016 predicate(!needs_acquiring_load(n));
7017
7018 ins_cost(4 * INSN_COST);
7019 format %{ "ldrw $dst, $mem\t# int" %}
7020
7021 ins_encode(aarch64_enc_ldrw(dst, mem));
7022
7023 ins_pipe(iload_reg_mem);
7024 %}
7025
7026 // Load Integer (32 bit signed) into long
7027 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7028 %{
7029 match(Set dst (ConvI2L (LoadI mem)));
7030 predicate(!needs_acquiring_load(n->in(1)));
7031
7032 ins_cost(4 * INSN_COST);
7033 format %{ "ldrsw $dst, $mem\t# int" %}
7034
7035 ins_encode(aarch64_enc_ldrsw(dst, mem));
7036
7037 ins_pipe(iload_reg_mem);
7038 %}
7039
7040 // Load Integer (32 bit unsigned) into long
7041 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7042 %{
7043 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7044 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7045
7046 ins_cost(4 * INSN_COST);
7047 format %{ "ldrw $dst, $mem\t# int" %}
7048
7049 ins_encode(aarch64_enc_ldrw(dst, mem));
7050
7051 ins_pipe(iload_reg_mem);
7052 %}
7053
7054 // Load Long (64 bit signed)
7055 instruct loadL(iRegLNoSp dst, memory8 mem)
7056 %{
7057 match(Set dst (LoadL mem));
7058 predicate(!needs_acquiring_load(n));
7059
7060 ins_cost(4 * INSN_COST);
7061 format %{ "ldr $dst, $mem\t# int" %}
7062
7063 ins_encode(aarch64_enc_ldr(dst, mem));
7064
7065 ins_pipe(iload_reg_mem);
7066 %}
7067
7068 // Load Range
7069 instruct loadRange(iRegINoSp dst, memory4 mem)
7070 %{
7071 match(Set dst (LoadRange mem));
7072
7073 ins_cost(4 * INSN_COST);
7074 format %{ "ldrw $dst, $mem\t# range" %}
7075
7076 ins_encode(aarch64_enc_ldrw(dst, mem));
7077
7078 ins_pipe(iload_reg_mem);
7079 %}
7080
7081 // Load Pointer
7082 instruct loadP(iRegPNoSp dst, memory8 mem)
7083 %{
7084 match(Set dst (LoadP mem));
7085 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7086
7087 ins_cost(4 * INSN_COST);
7088 format %{ "ldr $dst, $mem\t# ptr" %}
7089
7090 ins_encode(aarch64_enc_ldr(dst, mem));
7091
7092 ins_pipe(iload_reg_mem);
7093 %}
7094
7095 // Load Compressed Pointer
7096 instruct loadN(iRegNNoSp dst, memory4 mem)
7097 %{
7098 match(Set dst (LoadN mem));
7099 predicate(!needs_acquiring_load(n));
7100
7101 ins_cost(4 * INSN_COST);
7102 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7103
7104 ins_encode(aarch64_enc_ldrw(dst, mem));
7105
7106 ins_pipe(iload_reg_mem);
7107 %}
7108
7109 // Load Klass Pointer
7110 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7111 %{
7112 match(Set dst (LoadKlass mem));
7113 predicate(!needs_acquiring_load(n));
7114
7115 ins_cost(4 * INSN_COST);
7116 format %{ "ldr $dst, $mem\t# class" %}
7117
7118 ins_encode(aarch64_enc_ldr(dst, mem));
7119
7120 ins_pipe(iload_reg_mem);
7121 %}
7122
7123 // Load Narrow Klass Pointer
7124 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7125 %{
7126 match(Set dst (LoadNKlass mem));
7127 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
7128
7129 ins_cost(4 * INSN_COST);
7130 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7131
7132 ins_encode(aarch64_enc_ldrw(dst, mem));
7133
7134 ins_pipe(iload_reg_mem);
7135 %}
7136
7137 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
7138 %{
7139 match(Set dst (LoadNKlass mem));
7140 effect(KILL cr);
7141 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
7142
7143 ins_cost(4 * INSN_COST);
7144 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7145 ins_encode %{
7146 __ load_nklass_compact($dst$$Register, $mem$$base$$Register, $mem$$index$$Register, $mem$$scale, $mem$$disp);
7147 %}
7148 ins_pipe(pipe_slow);
7149 %}
7150
7151 // Load Float
7152 instruct loadF(vRegF dst, memory4 mem)
7153 %{
7154 match(Set dst (LoadF mem));
7155 predicate(!needs_acquiring_load(n));
7156
7157 ins_cost(4 * INSN_COST);
7158 format %{ "ldrs $dst, $mem\t# float" %}
7159
7160 ins_encode( aarch64_enc_ldrs(dst, mem) );
7161
7162 ins_pipe(pipe_class_memory);
7163 %}
7164
7165 // Load Double
7166 instruct loadD(vRegD dst, memory8 mem)
7167 %{
7168 match(Set dst (LoadD mem));
7169 predicate(!needs_acquiring_load(n));
7170
7171 ins_cost(4 * INSN_COST);
7172 format %{ "ldrd $dst, $mem\t# double" %}
7173
7174 ins_encode( aarch64_enc_ldrd(dst, mem) );
7175
7176 ins_pipe(pipe_class_memory);
7177 %}
7178
7179
7180 // Load Int Constant
7181 instruct loadConI(iRegINoSp dst, immI src)
7182 %{
7183 match(Set dst src);
7184
7185 ins_cost(INSN_COST);
7186 format %{ "mov $dst, $src\t# int" %}
7187
7188 ins_encode( aarch64_enc_movw_imm(dst, src) );
7189
7190 ins_pipe(ialu_imm);
7191 %}
7192
7193 // Load Long Constant
7194 instruct loadConL(iRegLNoSp dst, immL src)
7195 %{
7196 match(Set dst src);
7197
7198 ins_cost(INSN_COST);
7199 format %{ "mov $dst, $src\t# long" %}
7200
7201 ins_encode( aarch64_enc_mov_imm(dst, src) );
7202
7203 ins_pipe(ialu_imm);
7204 %}
7205
7206 // Load Pointer Constant
7207
7208 instruct loadConP(iRegPNoSp dst, immP con)
7209 %{
7210 match(Set dst con);
7211
7212 ins_cost(INSN_COST * 4);
7213 format %{
7214 "mov $dst, $con\t# ptr\n\t"
7215 %}
7216
7217 ins_encode(aarch64_enc_mov_p(dst, con));
7218
7219 ins_pipe(ialu_imm);
7220 %}
7221
7222 // Load Null Pointer Constant
7223
7224 instruct loadConP0(iRegPNoSp dst, immP0 con)
7225 %{
7226 match(Set dst con);
7227
7228 ins_cost(INSN_COST);
7229 format %{ "mov $dst, $con\t# NULL ptr" %}
7230
7231 ins_encode(aarch64_enc_mov_p0(dst, con));
7232
7233 ins_pipe(ialu_imm);
7234 %}
7235
7236 // Load Pointer Constant One
7237
7238 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7239 %{
7240 match(Set dst con);
7241
7242 ins_cost(INSN_COST);
7243 format %{ "mov $dst, $con\t# NULL ptr" %}
7244
7245 ins_encode(aarch64_enc_mov_p1(dst, con));
7246
7247 ins_pipe(ialu_imm);
7248 %}
7249
7250 // Load Byte Map Base Constant
7251
7252 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7253 %{
7254 match(Set dst con);
7255
7256 ins_cost(INSN_COST);
7257 format %{ "adr $dst, $con\t# Byte Map Base" %}
7258
7259 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7260
7261 ins_pipe(ialu_imm);
7262 %}
7263
7264 // Load Narrow Pointer Constant
7265
7266 instruct loadConN(iRegNNoSp dst, immN con)
7267 %{
7268 match(Set dst con);
7269
7270 ins_cost(INSN_COST * 4);
7271 format %{ "mov $dst, $con\t# compressed ptr" %}
7272
7273 ins_encode(aarch64_enc_mov_n(dst, con));
7274
7275 ins_pipe(ialu_imm);
7276 %}
7277
7278 // Load Narrow Null Pointer Constant
7279
7280 instruct loadConN0(iRegNNoSp dst, immN0 con)
7281 %{
7282 match(Set dst con);
7283
7284 ins_cost(INSN_COST);
7285 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7286
7287 ins_encode(aarch64_enc_mov_n0(dst, con));
7288
7289 ins_pipe(ialu_imm);
7290 %}
7291
7292 // Load Narrow Klass Constant
7293
7294 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7295 %{
7296 match(Set dst con);
7297
7298 ins_cost(INSN_COST);
7299 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7300
7301 ins_encode(aarch64_enc_mov_nk(dst, con));
7302
7303 ins_pipe(ialu_imm);
7304 %}
7305
7306 // Load Packed Float Constant
7307
7308 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7309 match(Set dst con);
7310 ins_cost(INSN_COST * 4);
7311 format %{ "fmovs $dst, $con"%}
7312 ins_encode %{
7313 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7314 %}
7315
7316 ins_pipe(fp_imm_s);
7317 %}
7318
7319 // Load Float Constant
7320
7321 instruct loadConF(vRegF dst, immF con) %{
7322 match(Set dst con);
7323
7324 ins_cost(INSN_COST * 4);
7325
7326 format %{
7327 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7328 %}
7329
7330 ins_encode %{
7331 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7332 %}
7333
7334 ins_pipe(fp_load_constant_s);
7335 %}
7336
7337 // Load Packed Double Constant
7338
7339 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7340 match(Set dst con);
7341 ins_cost(INSN_COST);
7342 format %{ "fmovd $dst, $con"%}
7343 ins_encode %{
7344 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7345 %}
7346
7347 ins_pipe(fp_imm_d);
7348 %}
7349
7350 // Load Double Constant
7351
7352 instruct loadConD(vRegD dst, immD con) %{
7353 match(Set dst con);
7354
7355 ins_cost(INSN_COST * 5);
7356 format %{
7357 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7358 %}
7359
7360 ins_encode %{
7361 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7362 %}
7363
7364 ins_pipe(fp_load_constant_d);
7365 %}
7366
7367 // Store Instructions
7368
7369 // Store CMS card-mark Immediate
7370 instruct storeimmCM0(immI0 zero, memory1 mem)
7371 %{
7372 match(Set mem (StoreCM mem zero));
7373
7374 ins_cost(INSN_COST);
7375 format %{ "storestore (elided)\n\t"
7376 "strb zr, $mem\t# byte" %}
7377
7378 ins_encode(aarch64_enc_strb0(mem));
7379
7380 ins_pipe(istore_mem);
7381 %}
7382
7383 // Store CMS card-mark Immediate with intervening StoreStore
7384 // needed when using CMS with no conditional card marking
7385 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7386 %{
7387 match(Set mem (StoreCM mem zero));
7388
7389 ins_cost(INSN_COST * 2);
7390 format %{ "storestore\n\t"
7391 "dmb ishst"
7392 "\n\tstrb zr, $mem\t# byte" %}
7393
7394 ins_encode(aarch64_enc_strb0_ordered(mem));
7395
7396 ins_pipe(istore_mem);
7397 %}
7398
7399 // Store Byte
7400 instruct storeB(iRegIorL2I src, memory1 mem)
7401 %{
7402 match(Set mem (StoreB mem src));
7403 predicate(!needs_releasing_store(n));
7404
7405 ins_cost(INSN_COST);
7406 format %{ "strb $src, $mem\t# byte" %}
7407
7408 ins_encode(aarch64_enc_strb(src, mem));
7409
7410 ins_pipe(istore_reg_mem);
7411 %}
7412
7413
7414 instruct storeimmB0(immI0 zero, memory1 mem)
7415 %{
7416 match(Set mem (StoreB mem zero));
7417 predicate(!needs_releasing_store(n));
7418
7419 ins_cost(INSN_COST);
7420 format %{ "strb rscractch2, $mem\t# byte" %}
7421
7422 ins_encode(aarch64_enc_strb0(mem));
7423
7424 ins_pipe(istore_mem);
7425 %}
7426
7427 // Store Char/Short
7428 instruct storeC(iRegIorL2I src, memory2 mem)
7429 %{
7430 match(Set mem (StoreC mem src));
7431 predicate(!needs_releasing_store(n));
7432
7433 ins_cost(INSN_COST);
7434 format %{ "strh $src, $mem\t# short" %}
7435
7436 ins_encode(aarch64_enc_strh(src, mem));
7437
7438 ins_pipe(istore_reg_mem);
7439 %}
7440
7441 instruct storeimmC0(immI0 zero, memory2 mem)
7442 %{
7443 match(Set mem (StoreC mem zero));
7444 predicate(!needs_releasing_store(n));
7445
7446 ins_cost(INSN_COST);
7447 format %{ "strh zr, $mem\t# short" %}
7448
7449 ins_encode(aarch64_enc_strh0(mem));
7450
7451 ins_pipe(istore_mem);
7452 %}
7453
7454 // Store Integer
7455
7456 instruct storeI(iRegIorL2I src, memory4 mem)
7457 %{
7458 match(Set mem(StoreI mem src));
7459 predicate(!needs_releasing_store(n));
7460
7461 ins_cost(INSN_COST);
7462 format %{ "strw $src, $mem\t# int" %}
7463
7464 ins_encode(aarch64_enc_strw(src, mem));
7465
7466 ins_pipe(istore_reg_mem);
7467 %}
7468
7469 instruct storeimmI0(immI0 zero, memory4 mem)
7470 %{
7471 match(Set mem(StoreI mem zero));
7472 predicate(!needs_releasing_store(n));
7473
7474 ins_cost(INSN_COST);
7475 format %{ "strw zr, $mem\t# int" %}
7476
7477 ins_encode(aarch64_enc_strw0(mem));
7478
7479 ins_pipe(istore_mem);
7480 %}
7481
7482 // Store Long (64 bit signed)
7483 instruct storeL(iRegL src, memory8 mem)
7484 %{
7485 match(Set mem (StoreL mem src));
7486 predicate(!needs_releasing_store(n));
7487
7488 ins_cost(INSN_COST);
7489 format %{ "str $src, $mem\t# int" %}
7490
7491 ins_encode(aarch64_enc_str(src, mem));
7492
7493 ins_pipe(istore_reg_mem);
7494 %}
7495
7496 // Store Long (64 bit signed)
7497 instruct storeimmL0(immL0 zero, memory8 mem)
7498 %{
7499 match(Set mem (StoreL mem zero));
7500 predicate(!needs_releasing_store(n));
7501
7502 ins_cost(INSN_COST);
7503 format %{ "str zr, $mem\t# int" %}
7504
7505 ins_encode(aarch64_enc_str0(mem));
7506
7507 ins_pipe(istore_mem);
7508 %}
7509
7510 // Store Pointer
7511 instruct storeP(iRegP src, memory8 mem)
7512 %{
7513 match(Set mem (StoreP mem src));
7514 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7515
7516 ins_cost(INSN_COST);
7517 format %{ "str $src, $mem\t# ptr" %}
7518
7519 ins_encode(aarch64_enc_str(src, mem));
7520
7521 ins_pipe(istore_reg_mem);
7522 %}
7523
7524 // Store Pointer
7525 instruct storeimmP0(immP0 zero, memory8 mem)
7526 %{
7527 match(Set mem (StoreP mem zero));
7528 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7529
7530 ins_cost(INSN_COST);
7531 format %{ "str zr, $mem\t# ptr" %}
7532
7533 ins_encode(aarch64_enc_str0(mem));
7534
7535 ins_pipe(istore_mem);
7536 %}
7537
7538 // Store Compressed Pointer
7539 instruct storeN(iRegN src, memory4 mem)
7540 %{
7541 match(Set mem (StoreN mem src));
7542 predicate(!needs_releasing_store(n));
7543
7544 ins_cost(INSN_COST);
7545 format %{ "strw $src, $mem\t# compressed ptr" %}
7546
7547 ins_encode(aarch64_enc_strw(src, mem));
7548
7549 ins_pipe(istore_reg_mem);
7550 %}
7551
7552 instruct storeImmN0(immN0 zero, memory4 mem)
7553 %{
7554 match(Set mem (StoreN mem zero));
7555 predicate(!needs_releasing_store(n));
7556
7557 ins_cost(INSN_COST);
7558 format %{ "strw zr, $mem\t# compressed ptr" %}
7559
7560 ins_encode(aarch64_enc_strw0(mem));
7561
7562 ins_pipe(istore_mem);
7563 %}
7564
7565 // Store Float
7566 instruct storeF(vRegF src, memory4 mem)
7567 %{
7568 match(Set mem (StoreF mem src));
7569 predicate(!needs_releasing_store(n));
7570
7571 ins_cost(INSN_COST);
7572 format %{ "strs $src, $mem\t# float" %}
7573
7574 ins_encode( aarch64_enc_strs(src, mem) );
7575
7576 ins_pipe(pipe_class_memory);
7577 %}
7578
7579 // TODO
7580 // implement storeImmF0 and storeFImmPacked
7581
7582 // Store Double
7583 instruct storeD(vRegD src, memory8 mem)
7584 %{
7585 match(Set mem (StoreD mem src));
7586 predicate(!needs_releasing_store(n));
7587
7588 ins_cost(INSN_COST);
7589 format %{ "strd $src, $mem\t# double" %}
7590
7591 ins_encode( aarch64_enc_strd(src, mem) );
7592
7593 ins_pipe(pipe_class_memory);
7594 %}
7595
7596 // Store Compressed Klass Pointer
7597 instruct storeNKlass(iRegN src, memory4 mem)
7598 %{
7599 predicate(!needs_releasing_store(n));
7600 match(Set mem (StoreNKlass mem src));
7601
7602 ins_cost(INSN_COST);
7603 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7604
7605 ins_encode(aarch64_enc_strw(src, mem));
7606
7607 ins_pipe(istore_reg_mem);
7608 %}
7609
7610 // TODO
7611 // implement storeImmD0 and storeDImmPacked
7612
7613 // prefetch instructions
7614 // Must be safe to execute with invalid address (cannot fault).
7615
7616 instruct prefetchalloc( memory8 mem ) %{
7617 match(PrefetchAllocation mem);
7618
7619 ins_cost(INSN_COST);
7620 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7621
7622 ins_encode( aarch64_enc_prefetchw(mem) );
7623
7624 ins_pipe(iload_prefetch);
7625 %}
7626
7627 // ---------------- volatile loads and stores ----------------
7628
7629 // Load Byte (8 bit signed)
7630 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7631 %{
7632 match(Set dst (LoadB mem));
7633
7634 ins_cost(VOLATILE_REF_COST);
7635 format %{ "ldarsb $dst, $mem\t# byte" %}
7636
7637 ins_encode(aarch64_enc_ldarsb(dst, mem));
7638
7639 ins_pipe(pipe_serial);
7640 %}
7641
7642 // Load Byte (8 bit signed) into long
7643 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7644 %{
7645 match(Set dst (ConvI2L (LoadB mem)));
7646
7647 ins_cost(VOLATILE_REF_COST);
7648 format %{ "ldarsb $dst, $mem\t# byte" %}
7649
7650 ins_encode(aarch64_enc_ldarsb(dst, mem));
7651
7652 ins_pipe(pipe_serial);
7653 %}
7654
7655 // Load Byte (8 bit unsigned)
7656 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7657 %{
7658 match(Set dst (LoadUB mem));
7659
7660 ins_cost(VOLATILE_REF_COST);
7661 format %{ "ldarb $dst, $mem\t# byte" %}
7662
7663 ins_encode(aarch64_enc_ldarb(dst, mem));
7664
7665 ins_pipe(pipe_serial);
7666 %}
7667
7668 // Load Byte (8 bit unsigned) into long
7669 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7670 %{
7671 match(Set dst (ConvI2L (LoadUB mem)));
7672
7673 ins_cost(VOLATILE_REF_COST);
7674 format %{ "ldarb $dst, $mem\t# byte" %}
7675
7676 ins_encode(aarch64_enc_ldarb(dst, mem));
7677
7678 ins_pipe(pipe_serial);
7679 %}
7680
7681 // Load Short (16 bit signed)
7682 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7683 %{
7684 match(Set dst (LoadS mem));
7685
7686 ins_cost(VOLATILE_REF_COST);
7687 format %{ "ldarshw $dst, $mem\t# short" %}
7688
7689 ins_encode(aarch64_enc_ldarshw(dst, mem));
7690
7691 ins_pipe(pipe_serial);
7692 %}
7693
7694 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7695 %{
7696 match(Set dst (LoadUS mem));
7697
7698 ins_cost(VOLATILE_REF_COST);
7699 format %{ "ldarhw $dst, $mem\t# short" %}
7700
7701 ins_encode(aarch64_enc_ldarhw(dst, mem));
7702
7703 ins_pipe(pipe_serial);
7704 %}
7705
7706 // Load Short/Char (16 bit unsigned) into long
7707 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7708 %{
7709 match(Set dst (ConvI2L (LoadUS mem)));
7710
7711 ins_cost(VOLATILE_REF_COST);
7712 format %{ "ldarh $dst, $mem\t# short" %}
7713
7714 ins_encode(aarch64_enc_ldarh(dst, mem));
7715
7716 ins_pipe(pipe_serial);
7717 %}
7718
7719 // Load Short/Char (16 bit signed) into long
7720 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7721 %{
7722 match(Set dst (ConvI2L (LoadS mem)));
7723
7724 ins_cost(VOLATILE_REF_COST);
7725 format %{ "ldarh $dst, $mem\t# short" %}
7726
7727 ins_encode(aarch64_enc_ldarsh(dst, mem));
7728
7729 ins_pipe(pipe_serial);
7730 %}
7731
7732 // Load Integer (32 bit signed)
7733 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7734 %{
7735 match(Set dst (LoadI mem));
7736
7737 ins_cost(VOLATILE_REF_COST);
7738 format %{ "ldarw $dst, $mem\t# int" %}
7739
7740 ins_encode(aarch64_enc_ldarw(dst, mem));
7741
7742 ins_pipe(pipe_serial);
7743 %}
7744
7745 // Load Integer (32 bit unsigned) into long
7746 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7747 %{
7748 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7749
7750 ins_cost(VOLATILE_REF_COST);
7751 format %{ "ldarw $dst, $mem\t# int" %}
7752
7753 ins_encode(aarch64_enc_ldarw(dst, mem));
7754
7755 ins_pipe(pipe_serial);
7756 %}
7757
7758 // Load Long (64 bit signed)
7759 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7760 %{
7761 match(Set dst (LoadL mem));
7762
7763 ins_cost(VOLATILE_REF_COST);
7764 format %{ "ldar $dst, $mem\t# int" %}
7765
7766 ins_encode(aarch64_enc_ldar(dst, mem));
7767
7768 ins_pipe(pipe_serial);
7769 %}
7770
7771 // Load Pointer
7772 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7773 %{
7774 match(Set dst (LoadP mem));
7775 predicate(n->as_Load()->barrier_data() == 0);
7776
7777 ins_cost(VOLATILE_REF_COST);
7778 format %{ "ldar $dst, $mem\t# ptr" %}
7779
7780 ins_encode(aarch64_enc_ldar(dst, mem));
7781
7782 ins_pipe(pipe_serial);
7783 %}
7784
7785 // Load Compressed Pointer
7786 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7787 %{
7788 match(Set dst (LoadN mem));
7789
7790 ins_cost(VOLATILE_REF_COST);
7791 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7792
7793 ins_encode(aarch64_enc_ldarw(dst, mem));
7794
7795 ins_pipe(pipe_serial);
7796 %}
7797
7798 // Load Float
7799 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7800 %{
7801 match(Set dst (LoadF mem));
7802
7803 ins_cost(VOLATILE_REF_COST);
7804 format %{ "ldars $dst, $mem\t# float" %}
7805
7806 ins_encode( aarch64_enc_fldars(dst, mem) );
7807
7808 ins_pipe(pipe_serial);
7809 %}
7810
7811 // Load Double
7812 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7813 %{
7814 match(Set dst (LoadD mem));
7815
7816 ins_cost(VOLATILE_REF_COST);
7817 format %{ "ldard $dst, $mem\t# double" %}
7818
7819 ins_encode( aarch64_enc_fldard(dst, mem) );
7820
7821 ins_pipe(pipe_serial);
7822 %}
7823
7824 // Store Byte
7825 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7826 %{
7827 match(Set mem (StoreB mem src));
7828
7829 ins_cost(VOLATILE_REF_COST);
7830 format %{ "stlrb $src, $mem\t# byte" %}
7831
7832 ins_encode(aarch64_enc_stlrb(src, mem));
7833
7834 ins_pipe(pipe_class_memory);
7835 %}
7836
7837 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7838 %{
7839 match(Set mem (StoreB mem zero));
7840
7841 ins_cost(VOLATILE_REF_COST);
7842 format %{ "stlrb zr, $mem\t# byte" %}
7843
7844 ins_encode(aarch64_enc_stlrb0(mem));
7845
7846 ins_pipe(pipe_class_memory);
7847 %}
7848
7849 // Store Char/Short
7850 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7851 %{
7852 match(Set mem (StoreC mem src));
7853
7854 ins_cost(VOLATILE_REF_COST);
7855 format %{ "stlrh $src, $mem\t# short" %}
7856
7857 ins_encode(aarch64_enc_stlrh(src, mem));
7858
7859 ins_pipe(pipe_class_memory);
7860 %}
7861
7862 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7863 %{
7864 match(Set mem (StoreC mem zero));
7865
7866 ins_cost(VOLATILE_REF_COST);
7867 format %{ "stlrh zr, $mem\t# short" %}
7868
7869 ins_encode(aarch64_enc_stlrh0(mem));
7870
7871 ins_pipe(pipe_class_memory);
7872 %}
7873
7874 // Store Integer
7875
7876 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7877 %{
7878 match(Set mem(StoreI mem src));
7879
7880 ins_cost(VOLATILE_REF_COST);
7881 format %{ "stlrw $src, $mem\t# int" %}
7882
7883 ins_encode(aarch64_enc_stlrw(src, mem));
7884
7885 ins_pipe(pipe_class_memory);
7886 %}
7887
7888 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7889 %{
7890 match(Set mem(StoreI mem zero));
7891
7892 ins_cost(VOLATILE_REF_COST);
7893 format %{ "stlrw zr, $mem\t# int" %}
7894
7895 ins_encode(aarch64_enc_stlrw0(mem));
7896
7897 ins_pipe(pipe_class_memory);
7898 %}
7899
7900 // Store Long (64 bit signed)
7901 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7902 %{
7903 match(Set mem (StoreL mem src));
7904
7905 ins_cost(VOLATILE_REF_COST);
7906 format %{ "stlr $src, $mem\t# int" %}
7907
7908 ins_encode(aarch64_enc_stlr(src, mem));
7909
7910 ins_pipe(pipe_class_memory);
7911 %}
7912
7913 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7914 %{
7915 match(Set mem (StoreL mem zero));
7916
7917 ins_cost(VOLATILE_REF_COST);
7918 format %{ "stlr zr, $mem\t# int" %}
7919
7920 ins_encode(aarch64_enc_stlr0(mem));
7921
7922 ins_pipe(pipe_class_memory);
7923 %}
7924
7925 // Store Pointer
7926 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7927 %{
7928 match(Set mem (StoreP mem src));
7929 predicate(n->as_Store()->barrier_data() == 0);
7930
7931 ins_cost(VOLATILE_REF_COST);
7932 format %{ "stlr $src, $mem\t# ptr" %}
7933
7934 ins_encode(aarch64_enc_stlr(src, mem));
7935
7936 ins_pipe(pipe_class_memory);
7937 %}
7938
7939 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7940 %{
7941 match(Set mem (StoreP mem zero));
7942 predicate(n->as_Store()->barrier_data() == 0);
7943
7944 ins_cost(VOLATILE_REF_COST);
7945 format %{ "stlr zr, $mem\t# ptr" %}
7946
7947 ins_encode(aarch64_enc_stlr0(mem));
7948
7949 ins_pipe(pipe_class_memory);
7950 %}
7951
7952 // Store Compressed Pointer
7953 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7954 %{
7955 match(Set mem (StoreN mem src));
7956
7957 ins_cost(VOLATILE_REF_COST);
7958 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7959
7960 ins_encode(aarch64_enc_stlrw(src, mem));
7961
7962 ins_pipe(pipe_class_memory);
7963 %}
7964
7965 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7966 %{
7967 match(Set mem (StoreN mem zero));
7968
7969 ins_cost(VOLATILE_REF_COST);
7970 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7971
7972 ins_encode(aarch64_enc_stlrw0(mem));
7973
7974 ins_pipe(pipe_class_memory);
7975 %}
7976
7977 // Store Float
7978 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7979 %{
7980 match(Set mem (StoreF mem src));
7981
7982 ins_cost(VOLATILE_REF_COST);
7983 format %{ "stlrs $src, $mem\t# float" %}
7984
7985 ins_encode( aarch64_enc_fstlrs(src, mem) );
7986
7987 ins_pipe(pipe_class_memory);
7988 %}
7989
7990 // TODO
7991 // implement storeImmF0 and storeFImmPacked
7992
7993 // Store Double
7994 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7995 %{
7996 match(Set mem (StoreD mem src));
7997
7998 ins_cost(VOLATILE_REF_COST);
7999 format %{ "stlrd $src, $mem\t# double" %}
8000
8001 ins_encode( aarch64_enc_fstlrd(src, mem) );
8002
8003 ins_pipe(pipe_class_memory);
8004 %}
8005
8006 // ---------------- end of volatile loads and stores ----------------
8007
8008 instruct cacheWB(indirect addr)
8009 %{
8010 predicate(VM_Version::supports_data_cache_line_flush());
8011 match(CacheWB addr);
8012
8013 ins_cost(100);
8014 format %{"cache wb $addr" %}
8015 ins_encode %{
8016 assert($addr->index_position() < 0, "should be");
8017 assert($addr$$disp == 0, "should be");
8018 __ cache_wb(Address($addr$$base$$Register, 0));
8019 %}
8020 ins_pipe(pipe_slow); // XXX
8021 %}
8022
8023 instruct cacheWBPreSync()
8024 %{
8025 predicate(VM_Version::supports_data_cache_line_flush());
8026 match(CacheWBPreSync);
8027
8028 ins_cost(100);
8029 format %{"cache wb presync" %}
8030 ins_encode %{
8031 __ cache_wbsync(true);
8032 %}
8033 ins_pipe(pipe_slow); // XXX
8034 %}
8035
8036 instruct cacheWBPostSync()
8037 %{
8038 predicate(VM_Version::supports_data_cache_line_flush());
8039 match(CacheWBPostSync);
8040
8041 ins_cost(100);
8042 format %{"cache wb postsync" %}
8043 ins_encode %{
8044 __ cache_wbsync(false);
8045 %}
8046 ins_pipe(pipe_slow); // XXX
8047 %}
8048
8049 // ============================================================================
8050 // BSWAP Instructions
8051
8052 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8053 match(Set dst (ReverseBytesI src));
8054
8055 ins_cost(INSN_COST);
8056 format %{ "revw $dst, $src" %}
8057
8058 ins_encode %{
8059 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8060 %}
8061
8062 ins_pipe(ialu_reg);
8063 %}
8064
8065 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8066 match(Set dst (ReverseBytesL src));
8067
8068 ins_cost(INSN_COST);
8069 format %{ "rev $dst, $src" %}
8070
8071 ins_encode %{
8072 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8073 %}
8074
8075 ins_pipe(ialu_reg);
8076 %}
8077
8078 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8079 match(Set dst (ReverseBytesUS src));
8080
8081 ins_cost(INSN_COST);
8082 format %{ "rev16w $dst, $src" %}
8083
8084 ins_encode %{
8085 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8086 %}
8087
8088 ins_pipe(ialu_reg);
8089 %}
8090
8091 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8092 match(Set dst (ReverseBytesS src));
8093
8094 ins_cost(INSN_COST);
8095 format %{ "rev16w $dst, $src\n\t"
8096 "sbfmw $dst, $dst, #0, #15" %}
8097
8098 ins_encode %{
8099 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8100 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8101 %}
8102
8103 ins_pipe(ialu_reg);
8104 %}
8105
8106 // ============================================================================
8107 // Zero Count Instructions
8108
8109 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8110 match(Set dst (CountLeadingZerosI src));
8111
8112 ins_cost(INSN_COST);
8113 format %{ "clzw $dst, $src" %}
8114 ins_encode %{
8115 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8116 %}
8117
8118 ins_pipe(ialu_reg);
8119 %}
8120
8121 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8122 match(Set dst (CountLeadingZerosL src));
8123
8124 ins_cost(INSN_COST);
8125 format %{ "clz $dst, $src" %}
8126 ins_encode %{
8127 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8128 %}
8129
8130 ins_pipe(ialu_reg);
8131 %}
8132
8133 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8134 match(Set dst (CountTrailingZerosI src));
8135
8136 ins_cost(INSN_COST * 2);
8137 format %{ "rbitw $dst, $src\n\t"
8138 "clzw $dst, $dst" %}
8139 ins_encode %{
8140 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8141 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8142 %}
8143
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8148 match(Set dst (CountTrailingZerosL src));
8149
8150 ins_cost(INSN_COST * 2);
8151 format %{ "rbit $dst, $src\n\t"
8152 "clz $dst, $dst" %}
8153 ins_encode %{
8154 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8155 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8156 %}
8157
8158 ins_pipe(ialu_reg);
8159 %}
8160
8161 //---------- Population Count Instructions -------------------------------------
8162 //
8163
8164 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8165 match(Set dst (PopCountI src));
8166 effect(TEMP tmp);
8167 ins_cost(INSN_COST * 13);
8168
8169 format %{ "movw $src, $src\n\t"
8170 "mov $tmp, $src\t# vector (1D)\n\t"
8171 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8172 "addv $tmp, $tmp\t# vector (8B)\n\t"
8173 "mov $dst, $tmp\t# vector (1D)" %}
8174 ins_encode %{
8175 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8176 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8177 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8178 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8179 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8180 %}
8181
8182 ins_pipe(pipe_class_default);
8183 %}
8184
8185 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8186 match(Set dst (PopCountI (LoadI mem)));
8187 effect(TEMP tmp);
8188 ins_cost(INSN_COST * 13);
8189
8190 format %{ "ldrs $tmp, $mem\n\t"
8191 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8192 "addv $tmp, $tmp\t# vector (8B)\n\t"
8193 "mov $dst, $tmp\t# vector (1D)" %}
8194 ins_encode %{
8195 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8196 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8197 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8198 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8199 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8200 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8201 %}
8202
8203 ins_pipe(pipe_class_default);
8204 %}
8205
8206 // Note: Long.bitCount(long) returns an int.
8207 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8208 match(Set dst (PopCountL src));
8209 effect(TEMP tmp);
8210 ins_cost(INSN_COST * 13);
8211
8212 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8213 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8214 "addv $tmp, $tmp\t# vector (8B)\n\t"
8215 "mov $dst, $tmp\t# vector (1D)" %}
8216 ins_encode %{
8217 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8218 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8219 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8220 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8221 %}
8222
8223 ins_pipe(pipe_class_default);
8224 %}
8225
8226 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8227 match(Set dst (PopCountL (LoadL mem)));
8228 effect(TEMP tmp);
8229 ins_cost(INSN_COST * 13);
8230
8231 format %{ "ldrd $tmp, $mem\n\t"
8232 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8233 "addv $tmp, $tmp\t# vector (8B)\n\t"
8234 "mov $dst, $tmp\t# vector (1D)" %}
8235 ins_encode %{
8236 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8237 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8238 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8239 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8240 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8241 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8242 %}
8243
8244 ins_pipe(pipe_class_default);
8245 %}
8246
8247 // ============================================================================
8248 // MemBar Instruction
8249
8250 instruct load_fence() %{
8251 match(LoadFence);
8252 ins_cost(VOLATILE_REF_COST);
8253
8254 format %{ "load_fence" %}
8255
8256 ins_encode %{
8257 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8258 %}
8259 ins_pipe(pipe_serial);
8260 %}
8261
8262 instruct unnecessary_membar_acquire() %{
8263 predicate(unnecessary_acquire(n));
8264 match(MemBarAcquire);
8265 ins_cost(0);
8266
8267 format %{ "membar_acquire (elided)" %}
8268
8269 ins_encode %{
8270 __ block_comment("membar_acquire (elided)");
8271 %}
8272
8273 ins_pipe(pipe_class_empty);
8274 %}
8275
8276 instruct membar_acquire() %{
8277 match(MemBarAcquire);
8278 ins_cost(VOLATILE_REF_COST);
8279
8280 format %{ "membar_acquire\n\t"
8281 "dmb ish" %}
8282
8283 ins_encode %{
8284 __ block_comment("membar_acquire");
8285 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8286 %}
8287
8288 ins_pipe(pipe_serial);
8289 %}
8290
8291
8292 instruct membar_acquire_lock() %{
8293 match(MemBarAcquireLock);
8294 ins_cost(VOLATILE_REF_COST);
8295
8296 format %{ "membar_acquire_lock (elided)" %}
8297
8298 ins_encode %{
8299 __ block_comment("membar_acquire_lock (elided)");
8300 %}
8301
8302 ins_pipe(pipe_serial);
8303 %}
8304
8305 instruct store_fence() %{
8306 match(StoreFence);
8307 ins_cost(VOLATILE_REF_COST);
8308
8309 format %{ "store_fence" %}
8310
8311 ins_encode %{
8312 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8313 %}
8314 ins_pipe(pipe_serial);
8315 %}
8316
8317 instruct unnecessary_membar_release() %{
8318 predicate(unnecessary_release(n));
8319 match(MemBarRelease);
8320 ins_cost(0);
8321
8322 format %{ "membar_release (elided)" %}
8323
8324 ins_encode %{
8325 __ block_comment("membar_release (elided)");
8326 %}
8327 ins_pipe(pipe_serial);
8328 %}
8329
8330 instruct membar_release() %{
8331 match(MemBarRelease);
8332 ins_cost(VOLATILE_REF_COST);
8333
8334 format %{ "membar_release\n\t"
8335 "dmb ish" %}
8336
8337 ins_encode %{
8338 __ block_comment("membar_release");
8339 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8340 %}
8341 ins_pipe(pipe_serial);
8342 %}
8343
8344 instruct membar_storestore() %{
8345 match(MemBarStoreStore);
8346 match(StoreStoreFence);
8347 ins_cost(VOLATILE_REF_COST);
8348
8349 format %{ "MEMBAR-store-store" %}
8350
8351 ins_encode %{
8352 __ membar(Assembler::StoreStore);
8353 %}
8354 ins_pipe(pipe_serial);
8355 %}
8356
8357 instruct membar_release_lock() %{
8358 match(MemBarReleaseLock);
8359 ins_cost(VOLATILE_REF_COST);
8360
8361 format %{ "membar_release_lock (elided)" %}
8362
8363 ins_encode %{
8364 __ block_comment("membar_release_lock (elided)");
8365 %}
8366
8367 ins_pipe(pipe_serial);
8368 %}
8369
8370 instruct unnecessary_membar_volatile() %{
8371 predicate(unnecessary_volatile(n));
8372 match(MemBarVolatile);
8373 ins_cost(0);
8374
8375 format %{ "membar_volatile (elided)" %}
8376
8377 ins_encode %{
8378 __ block_comment("membar_volatile (elided)");
8379 %}
8380
8381 ins_pipe(pipe_serial);
8382 %}
8383
8384 instruct membar_volatile() %{
8385 match(MemBarVolatile);
8386 ins_cost(VOLATILE_REF_COST*100);
8387
8388 format %{ "membar_volatile\n\t"
8389 "dmb ish"%}
8390
8391 ins_encode %{
8392 __ block_comment("membar_volatile");
8393 __ membar(Assembler::StoreLoad);
8394 %}
8395
8396 ins_pipe(pipe_serial);
8397 %}
8398
8399 // ============================================================================
8400 // Cast/Convert Instructions
8401
8402 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8403 match(Set dst (CastX2P src));
8404
8405 ins_cost(INSN_COST);
8406 format %{ "mov $dst, $src\t# long -> ptr" %}
8407
8408 ins_encode %{
8409 if ($dst$$reg != $src$$reg) {
8410 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8411 }
8412 %}
8413
8414 ins_pipe(ialu_reg);
8415 %}
8416
8417 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8418 match(Set dst (CastP2X src));
8419
8420 ins_cost(INSN_COST);
8421 format %{ "mov $dst, $src\t# ptr -> long" %}
8422
8423 ins_encode %{
8424 if ($dst$$reg != $src$$reg) {
8425 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8426 }
8427 %}
8428
8429 ins_pipe(ialu_reg);
8430 %}
8431
8432 // Convert oop into int for vectors alignment masking
8433 instruct convP2I(iRegINoSp dst, iRegP src) %{
8434 match(Set dst (ConvL2I (CastP2X src)));
8435
8436 ins_cost(INSN_COST);
8437 format %{ "movw $dst, $src\t# ptr -> int" %}
8438 ins_encode %{
8439 __ movw($dst$$Register, $src$$Register);
8440 %}
8441
8442 ins_pipe(ialu_reg);
8443 %}
8444
8445 // Convert compressed oop into int for vectors alignment masking
8446 // in case of 32bit oops (heap < 4Gb).
8447 instruct convN2I(iRegINoSp dst, iRegN src)
8448 %{
8449 predicate(CompressedOops::shift() == 0);
8450 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8451
8452 ins_cost(INSN_COST);
8453 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8454 ins_encode %{
8455 __ movw($dst$$Register, $src$$Register);
8456 %}
8457
8458 ins_pipe(ialu_reg);
8459 %}
8460
8461
8462 // Convert oop pointer into compressed form
8463 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8464 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8465 match(Set dst (EncodeP src));
8466 effect(KILL cr);
8467 ins_cost(INSN_COST * 3);
8468 format %{ "encode_heap_oop $dst, $src" %}
8469 ins_encode %{
8470 Register s = $src$$Register;
8471 Register d = $dst$$Register;
8472 __ encode_heap_oop(d, s);
8473 %}
8474 ins_pipe(ialu_reg);
8475 %}
8476
8477 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8478 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8479 match(Set dst (EncodeP src));
8480 ins_cost(INSN_COST * 3);
8481 format %{ "encode_heap_oop_not_null $dst, $src" %}
8482 ins_encode %{
8483 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8484 %}
8485 ins_pipe(ialu_reg);
8486 %}
8487
8488 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8489 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8490 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8491 match(Set dst (DecodeN src));
8492 ins_cost(INSN_COST * 3);
8493 format %{ "decode_heap_oop $dst, $src" %}
8494 ins_encode %{
8495 Register s = $src$$Register;
8496 Register d = $dst$$Register;
8497 __ decode_heap_oop(d, s);
8498 %}
8499 ins_pipe(ialu_reg);
8500 %}
8501
8502 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8503 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8504 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8505 match(Set dst (DecodeN src));
8506 ins_cost(INSN_COST * 3);
8507 format %{ "decode_heap_oop_not_null $dst, $src" %}
8508 ins_encode %{
8509 Register s = $src$$Register;
8510 Register d = $dst$$Register;
8511 __ decode_heap_oop_not_null(d, s);
8512 %}
8513 ins_pipe(ialu_reg);
8514 %}
8515
8516 // n.b. AArch64 implementations of encode_klass_not_null and
8517 // decode_klass_not_null do not modify the flags register so, unlike
8518 // Intel, we don't kill CR as a side effect here
8519
8520 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8521 match(Set dst (EncodePKlass src));
8522
8523 ins_cost(INSN_COST * 3);
8524 format %{ "encode_klass_not_null $dst,$src" %}
8525
8526 ins_encode %{
8527 Register src_reg = as_Register($src$$reg);
8528 Register dst_reg = as_Register($dst$$reg);
8529 __ encode_klass_not_null(dst_reg, src_reg);
8530 %}
8531
8532 ins_pipe(ialu_reg);
8533 %}
8534
8535 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8536 match(Set dst (DecodeNKlass src));
8537
8538 ins_cost(INSN_COST * 3);
8539 format %{ "decode_klass_not_null $dst,$src" %}
8540
8541 ins_encode %{
8542 Register src_reg = as_Register($src$$reg);
8543 Register dst_reg = as_Register($dst$$reg);
8544 if (dst_reg != src_reg) {
8545 __ decode_klass_not_null(dst_reg, src_reg);
8546 } else {
8547 __ decode_klass_not_null(dst_reg);
8548 }
8549 %}
8550
8551 ins_pipe(ialu_reg);
8552 %}
8553
8554 instruct checkCastPP(iRegPNoSp dst)
8555 %{
8556 match(Set dst (CheckCastPP dst));
8557
8558 size(0);
8559 format %{ "# checkcastPP of $dst" %}
8560 ins_encode(/* empty encoding */);
8561 ins_pipe(pipe_class_empty);
8562 %}
8563
8564 instruct castPP(iRegPNoSp dst)
8565 %{
8566 match(Set dst (CastPP dst));
8567
8568 size(0);
8569 format %{ "# castPP of $dst" %}
8570 ins_encode(/* empty encoding */);
8571 ins_pipe(pipe_class_empty);
8572 %}
8573
8574 instruct castII(iRegI dst)
8575 %{
8576 match(Set dst (CastII dst));
8577
8578 size(0);
8579 format %{ "# castII of $dst" %}
8580 ins_encode(/* empty encoding */);
8581 ins_cost(0);
8582 ins_pipe(pipe_class_empty);
8583 %}
8584
8585 instruct castLL(iRegL dst)
8586 %{
8587 match(Set dst (CastLL dst));
8588
8589 size(0);
8590 format %{ "# castLL of $dst" %}
8591 ins_encode(/* empty encoding */);
8592 ins_cost(0);
8593 ins_pipe(pipe_class_empty);
8594 %}
8595
8596 instruct castFF(vRegF dst)
8597 %{
8598 match(Set dst (CastFF dst));
8599
8600 size(0);
8601 format %{ "# castFF of $dst" %}
8602 ins_encode(/* empty encoding */);
8603 ins_cost(0);
8604 ins_pipe(pipe_class_empty);
8605 %}
8606
8607 instruct castDD(vRegD dst)
8608 %{
8609 match(Set dst (CastDD dst));
8610
8611 size(0);
8612 format %{ "# castDD of $dst" %}
8613 ins_encode(/* empty encoding */);
8614 ins_cost(0);
8615 ins_pipe(pipe_class_empty);
8616 %}
8617
8618 instruct castVV(vReg dst)
8619 %{
8620 match(Set dst (CastVV dst));
8621
8622 size(0);
8623 format %{ "# castVV of $dst" %}
8624 ins_encode(/* empty encoding */);
8625 ins_cost(0);
8626 ins_pipe(pipe_class_empty);
8627 %}
8628
8629 instruct castVVMask(pRegGov dst)
8630 %{
8631 match(Set dst (CastVV dst));
8632
8633 size(0);
8634 format %{ "# castVV of $dst" %}
8635 ins_encode(/* empty encoding */);
8636 ins_cost(0);
8637 ins_pipe(pipe_class_empty);
8638 %}
8639
8640 // ============================================================================
8641 // Atomic operation instructions
8642 //
8643
8644 // standard CompareAndSwapX when we are using barriers
8645 // these have higher priority than the rules selected by a predicate
8646
8647 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8648 // can't match them
8649
8650 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8651
8652 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8653 ins_cost(2 * VOLATILE_REF_COST);
8654
8655 effect(KILL cr);
8656
8657 format %{
8658 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8659 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8660 %}
8661
8662 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8663 aarch64_enc_cset_eq(res));
8664
8665 ins_pipe(pipe_slow);
8666 %}
8667
8668 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8669
8670 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8671 ins_cost(2 * VOLATILE_REF_COST);
8672
8673 effect(KILL cr);
8674
8675 format %{
8676 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8677 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8678 %}
8679
8680 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8681 aarch64_enc_cset_eq(res));
8682
8683 ins_pipe(pipe_slow);
8684 %}
8685
8686 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8687
8688 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8689 ins_cost(2 * VOLATILE_REF_COST);
8690
8691 effect(KILL cr);
8692
8693 format %{
8694 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8695 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8696 %}
8697
8698 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8699 aarch64_enc_cset_eq(res));
8700
8701 ins_pipe(pipe_slow);
8702 %}
8703
8704 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8705
8706 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8707 ins_cost(2 * VOLATILE_REF_COST);
8708
8709 effect(KILL cr);
8710
8711 format %{
8712 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8713 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8714 %}
8715
8716 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8717 aarch64_enc_cset_eq(res));
8718
8719 ins_pipe(pipe_slow);
8720 %}
8721
8722 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8723
8724 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8725 predicate(n->as_LoadStore()->barrier_data() == 0);
8726 ins_cost(2 * VOLATILE_REF_COST);
8727
8728 effect(KILL cr);
8729
8730 format %{
8731 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8732 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8733 %}
8734
8735 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8736 aarch64_enc_cset_eq(res));
8737
8738 ins_pipe(pipe_slow);
8739 %}
8740
8741 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8742
8743 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8744 ins_cost(2 * VOLATILE_REF_COST);
8745
8746 effect(KILL cr);
8747
8748 format %{
8749 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8750 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8751 %}
8752
8753 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8754 aarch64_enc_cset_eq(res));
8755
8756 ins_pipe(pipe_slow);
8757 %}
8758
8759 // alternative CompareAndSwapX when we are eliding barriers
8760
8761 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8762
8763 predicate(needs_acquiring_load_exclusive(n));
8764 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8765 ins_cost(VOLATILE_REF_COST);
8766
8767 effect(KILL cr);
8768
8769 format %{
8770 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8771 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8772 %}
8773
8774 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8775 aarch64_enc_cset_eq(res));
8776
8777 ins_pipe(pipe_slow);
8778 %}
8779
8780 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8781
8782 predicate(needs_acquiring_load_exclusive(n));
8783 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8784 ins_cost(VOLATILE_REF_COST);
8785
8786 effect(KILL cr);
8787
8788 format %{
8789 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8790 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8791 %}
8792
8793 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8794 aarch64_enc_cset_eq(res));
8795
8796 ins_pipe(pipe_slow);
8797 %}
8798
8799 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8800
8801 predicate(needs_acquiring_load_exclusive(n));
8802 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8803 ins_cost(VOLATILE_REF_COST);
8804
8805 effect(KILL cr);
8806
8807 format %{
8808 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8809 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8810 %}
8811
8812 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8813 aarch64_enc_cset_eq(res));
8814
8815 ins_pipe(pipe_slow);
8816 %}
8817
8818 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8819
8820 predicate(needs_acquiring_load_exclusive(n));
8821 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8822 ins_cost(VOLATILE_REF_COST);
8823
8824 effect(KILL cr);
8825
8826 format %{
8827 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8828 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8829 %}
8830
8831 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8832 aarch64_enc_cset_eq(res));
8833
8834 ins_pipe(pipe_slow);
8835 %}
8836
8837 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8838
8839 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8840 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8841 ins_cost(VOLATILE_REF_COST);
8842
8843 effect(KILL cr);
8844
8845 format %{
8846 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8847 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8848 %}
8849
8850 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8851 aarch64_enc_cset_eq(res));
8852
8853 ins_pipe(pipe_slow);
8854 %}
8855
8856 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8857
8858 predicate(needs_acquiring_load_exclusive(n));
8859 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8860 ins_cost(VOLATILE_REF_COST);
8861
8862 effect(KILL cr);
8863
8864 format %{
8865 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8866 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8867 %}
8868
8869 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8870 aarch64_enc_cset_eq(res));
8871
8872 ins_pipe(pipe_slow);
8873 %}
8874
8875
8876 // ---------------------------------------------------------------------
8877
8878 // BEGIN This section of the file is automatically generated. Do not edit --------------
8879
8880 // Sundry CAS operations. Note that release is always true,
8881 // regardless of the memory ordering of the CAS. This is because we
8882 // need the volatile case to be sequentially consistent but there is
8883 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8884 // can't check the type of memory ordering here, so we always emit a
8885 // STLXR.
8886
8887 // This section is generated from cas.m4
8888
8889
8890 // This pattern is generated automatically from cas.m4.
8891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8892 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8893 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8894 ins_cost(2 * VOLATILE_REF_COST);
8895 effect(TEMP_DEF res, KILL cr);
8896 format %{
8897 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8898 %}
8899 ins_encode %{
8900 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8901 Assembler::byte, /*acquire*/ false, /*release*/ true,
8902 /*weak*/ false, $res$$Register);
8903 __ sxtbw($res$$Register, $res$$Register);
8904 %}
8905 ins_pipe(pipe_slow);
8906 %}
8907
8908 // This pattern is generated automatically from cas.m4.
8909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8910 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8911 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8912 ins_cost(2 * VOLATILE_REF_COST);
8913 effect(TEMP_DEF res, KILL cr);
8914 format %{
8915 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8916 %}
8917 ins_encode %{
8918 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8919 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8920 /*weak*/ false, $res$$Register);
8921 __ sxthw($res$$Register, $res$$Register);
8922 %}
8923 ins_pipe(pipe_slow);
8924 %}
8925
8926 // This pattern is generated automatically from cas.m4.
8927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8928 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8929 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8930 ins_cost(2 * VOLATILE_REF_COST);
8931 effect(TEMP_DEF res, KILL cr);
8932 format %{
8933 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8934 %}
8935 ins_encode %{
8936 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8937 Assembler::word, /*acquire*/ false, /*release*/ true,
8938 /*weak*/ false, $res$$Register);
8939 %}
8940 ins_pipe(pipe_slow);
8941 %}
8942
8943 // This pattern is generated automatically from cas.m4.
8944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8945 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8946 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8947 ins_cost(2 * VOLATILE_REF_COST);
8948 effect(TEMP_DEF res, KILL cr);
8949 format %{
8950 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8951 %}
8952 ins_encode %{
8953 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8954 Assembler::xword, /*acquire*/ false, /*release*/ true,
8955 /*weak*/ false, $res$$Register);
8956 %}
8957 ins_pipe(pipe_slow);
8958 %}
8959
8960 // This pattern is generated automatically from cas.m4.
8961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8962 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8963 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8964 ins_cost(2 * VOLATILE_REF_COST);
8965 effect(TEMP_DEF res, KILL cr);
8966 format %{
8967 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8968 %}
8969 ins_encode %{
8970 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8971 Assembler::word, /*acquire*/ false, /*release*/ true,
8972 /*weak*/ false, $res$$Register);
8973 %}
8974 ins_pipe(pipe_slow);
8975 %}
8976
8977 // This pattern is generated automatically from cas.m4.
8978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8979 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8980 predicate(n->as_LoadStore()->barrier_data() == 0);
8981 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8982 ins_cost(2 * VOLATILE_REF_COST);
8983 effect(TEMP_DEF res, KILL cr);
8984 format %{
8985 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8986 %}
8987 ins_encode %{
8988 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8989 Assembler::xword, /*acquire*/ false, /*release*/ true,
8990 /*weak*/ false, $res$$Register);
8991 %}
8992 ins_pipe(pipe_slow);
8993 %}
8994
8995 // This pattern is generated automatically from cas.m4.
8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8997 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8998 predicate(needs_acquiring_load_exclusive(n));
8999 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9000 ins_cost(VOLATILE_REF_COST);
9001 effect(TEMP_DEF res, KILL cr);
9002 format %{
9003 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9004 %}
9005 ins_encode %{
9006 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9007 Assembler::byte, /*acquire*/ true, /*release*/ true,
9008 /*weak*/ false, $res$$Register);
9009 __ sxtbw($res$$Register, $res$$Register);
9010 %}
9011 ins_pipe(pipe_slow);
9012 %}
9013
9014 // This pattern is generated automatically from cas.m4.
9015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9016 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9017 predicate(needs_acquiring_load_exclusive(n));
9018 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9019 ins_cost(VOLATILE_REF_COST);
9020 effect(TEMP_DEF res, KILL cr);
9021 format %{
9022 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9023 %}
9024 ins_encode %{
9025 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9026 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9027 /*weak*/ false, $res$$Register);
9028 __ sxthw($res$$Register, $res$$Register);
9029 %}
9030 ins_pipe(pipe_slow);
9031 %}
9032
9033 // This pattern is generated automatically from cas.m4.
9034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9035 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9036 predicate(needs_acquiring_load_exclusive(n));
9037 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9038 ins_cost(VOLATILE_REF_COST);
9039 effect(TEMP_DEF res, KILL cr);
9040 format %{
9041 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9042 %}
9043 ins_encode %{
9044 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9045 Assembler::word, /*acquire*/ true, /*release*/ true,
9046 /*weak*/ false, $res$$Register);
9047 %}
9048 ins_pipe(pipe_slow);
9049 %}
9050
9051 // This pattern is generated automatically from cas.m4.
9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9053 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9054 predicate(needs_acquiring_load_exclusive(n));
9055 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9056 ins_cost(VOLATILE_REF_COST);
9057 effect(TEMP_DEF res, KILL cr);
9058 format %{
9059 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9060 %}
9061 ins_encode %{
9062 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9063 Assembler::xword, /*acquire*/ true, /*release*/ true,
9064 /*weak*/ false, $res$$Register);
9065 %}
9066 ins_pipe(pipe_slow);
9067 %}
9068
9069 // This pattern is generated automatically from cas.m4.
9070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9071 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9072 predicate(needs_acquiring_load_exclusive(n));
9073 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9074 ins_cost(VOLATILE_REF_COST);
9075 effect(TEMP_DEF res, KILL cr);
9076 format %{
9077 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9078 %}
9079 ins_encode %{
9080 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9081 Assembler::word, /*acquire*/ true, /*release*/ true,
9082 /*weak*/ false, $res$$Register);
9083 %}
9084 ins_pipe(pipe_slow);
9085 %}
9086
9087 // This pattern is generated automatically from cas.m4.
9088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9089 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9090 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9091 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9092 ins_cost(VOLATILE_REF_COST);
9093 effect(TEMP_DEF res, KILL cr);
9094 format %{
9095 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9096 %}
9097 ins_encode %{
9098 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9099 Assembler::xword, /*acquire*/ true, /*release*/ true,
9100 /*weak*/ false, $res$$Register);
9101 %}
9102 ins_pipe(pipe_slow);
9103 %}
9104
9105 // This pattern is generated automatically from cas.m4.
9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9107 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9108 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9109 ins_cost(2 * VOLATILE_REF_COST);
9110 effect(KILL cr);
9111 format %{
9112 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9113 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9114 %}
9115 ins_encode %{
9116 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9117 Assembler::byte, /*acquire*/ false, /*release*/ true,
9118 /*weak*/ true, noreg);
9119 __ csetw($res$$Register, Assembler::EQ);
9120 %}
9121 ins_pipe(pipe_slow);
9122 %}
9123
9124 // This pattern is generated automatically from cas.m4.
9125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9126 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9127 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9128 ins_cost(2 * VOLATILE_REF_COST);
9129 effect(KILL cr);
9130 format %{
9131 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9132 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9133 %}
9134 ins_encode %{
9135 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9136 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9137 /*weak*/ true, noreg);
9138 __ csetw($res$$Register, Assembler::EQ);
9139 %}
9140 ins_pipe(pipe_slow);
9141 %}
9142
9143 // This pattern is generated automatically from cas.m4.
9144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9145 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9146 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9147 ins_cost(2 * VOLATILE_REF_COST);
9148 effect(KILL cr);
9149 format %{
9150 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9151 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9152 %}
9153 ins_encode %{
9154 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9155 Assembler::word, /*acquire*/ false, /*release*/ true,
9156 /*weak*/ true, noreg);
9157 __ csetw($res$$Register, Assembler::EQ);
9158 %}
9159 ins_pipe(pipe_slow);
9160 %}
9161
9162 // This pattern is generated automatically from cas.m4.
9163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9164 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9165 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9166 ins_cost(2 * VOLATILE_REF_COST);
9167 effect(KILL cr);
9168 format %{
9169 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9170 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9171 %}
9172 ins_encode %{
9173 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9174 Assembler::xword, /*acquire*/ false, /*release*/ true,
9175 /*weak*/ true, noreg);
9176 __ csetw($res$$Register, Assembler::EQ);
9177 %}
9178 ins_pipe(pipe_slow);
9179 %}
9180
9181 // This pattern is generated automatically from cas.m4.
9182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9183 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9184 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9185 ins_cost(2 * VOLATILE_REF_COST);
9186 effect(KILL cr);
9187 format %{
9188 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9189 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9190 %}
9191 ins_encode %{
9192 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9193 Assembler::word, /*acquire*/ false, /*release*/ true,
9194 /*weak*/ true, noreg);
9195 __ csetw($res$$Register, Assembler::EQ);
9196 %}
9197 ins_pipe(pipe_slow);
9198 %}
9199
9200 // This pattern is generated automatically from cas.m4.
9201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9202 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9203 predicate(n->as_LoadStore()->barrier_data() == 0);
9204 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9205 ins_cost(2 * VOLATILE_REF_COST);
9206 effect(KILL cr);
9207 format %{
9208 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9209 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9210 %}
9211 ins_encode %{
9212 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9213 Assembler::xword, /*acquire*/ false, /*release*/ true,
9214 /*weak*/ true, noreg);
9215 __ csetw($res$$Register, Assembler::EQ);
9216 %}
9217 ins_pipe(pipe_slow);
9218 %}
9219
9220 // This pattern is generated automatically from cas.m4.
9221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9222 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9223 predicate(needs_acquiring_load_exclusive(n));
9224 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9225 ins_cost(VOLATILE_REF_COST);
9226 effect(KILL cr);
9227 format %{
9228 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9229 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9230 %}
9231 ins_encode %{
9232 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9233 Assembler::byte, /*acquire*/ true, /*release*/ true,
9234 /*weak*/ true, noreg);
9235 __ csetw($res$$Register, Assembler::EQ);
9236 %}
9237 ins_pipe(pipe_slow);
9238 %}
9239
9240 // This pattern is generated automatically from cas.m4.
9241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9242 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9243 predicate(needs_acquiring_load_exclusive(n));
9244 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9245 ins_cost(VOLATILE_REF_COST);
9246 effect(KILL cr);
9247 format %{
9248 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9249 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9250 %}
9251 ins_encode %{
9252 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9253 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9254 /*weak*/ true, noreg);
9255 __ csetw($res$$Register, Assembler::EQ);
9256 %}
9257 ins_pipe(pipe_slow);
9258 %}
9259
9260 // This pattern is generated automatically from cas.m4.
9261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9262 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9263 predicate(needs_acquiring_load_exclusive(n));
9264 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9265 ins_cost(VOLATILE_REF_COST);
9266 effect(KILL cr);
9267 format %{
9268 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9269 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9270 %}
9271 ins_encode %{
9272 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9273 Assembler::word, /*acquire*/ true, /*release*/ true,
9274 /*weak*/ true, noreg);
9275 __ csetw($res$$Register, Assembler::EQ);
9276 %}
9277 ins_pipe(pipe_slow);
9278 %}
9279
9280 // This pattern is generated automatically from cas.m4.
9281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9282 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9283 predicate(needs_acquiring_load_exclusive(n));
9284 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9285 ins_cost(VOLATILE_REF_COST);
9286 effect(KILL cr);
9287 format %{
9288 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9289 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9290 %}
9291 ins_encode %{
9292 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9293 Assembler::xword, /*acquire*/ true, /*release*/ true,
9294 /*weak*/ true, noreg);
9295 __ csetw($res$$Register, Assembler::EQ);
9296 %}
9297 ins_pipe(pipe_slow);
9298 %}
9299
9300 // This pattern is generated automatically from cas.m4.
9301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9302 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9303 predicate(needs_acquiring_load_exclusive(n));
9304 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9305 ins_cost(VOLATILE_REF_COST);
9306 effect(KILL cr);
9307 format %{
9308 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9309 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9310 %}
9311 ins_encode %{
9312 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9313 Assembler::word, /*acquire*/ true, /*release*/ true,
9314 /*weak*/ true, noreg);
9315 __ csetw($res$$Register, Assembler::EQ);
9316 %}
9317 ins_pipe(pipe_slow);
9318 %}
9319
9320 // This pattern is generated automatically from cas.m4.
9321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9322 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9323 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9324 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9325 ins_cost(VOLATILE_REF_COST);
9326 effect(KILL cr);
9327 format %{
9328 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9329 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9330 %}
9331 ins_encode %{
9332 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9333 Assembler::xword, /*acquire*/ true, /*release*/ true,
9334 /*weak*/ true, noreg);
9335 __ csetw($res$$Register, Assembler::EQ);
9336 %}
9337 ins_pipe(pipe_slow);
9338 %}
9339
9340 // END This section of the file is automatically generated. Do not edit --------------
9341 // ---------------------------------------------------------------------
9342
9343 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9344 match(Set prev (GetAndSetI mem newv));
9345 ins_cost(2 * VOLATILE_REF_COST);
9346 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9347 ins_encode %{
9348 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9349 %}
9350 ins_pipe(pipe_serial);
9351 %}
9352
9353 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9354 match(Set prev (GetAndSetL mem newv));
9355 ins_cost(2 * VOLATILE_REF_COST);
9356 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9357 ins_encode %{
9358 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9359 %}
9360 ins_pipe(pipe_serial);
9361 %}
9362
9363 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9364 match(Set prev (GetAndSetN mem newv));
9365 ins_cost(2 * VOLATILE_REF_COST);
9366 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9367 ins_encode %{
9368 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9369 %}
9370 ins_pipe(pipe_serial);
9371 %}
9372
9373 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9374 predicate(n->as_LoadStore()->barrier_data() == 0);
9375 match(Set prev (GetAndSetP mem newv));
9376 ins_cost(2 * VOLATILE_REF_COST);
9377 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9378 ins_encode %{
9379 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9380 %}
9381 ins_pipe(pipe_serial);
9382 %}
9383
9384 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9385 predicate(needs_acquiring_load_exclusive(n));
9386 match(Set prev (GetAndSetI mem newv));
9387 ins_cost(VOLATILE_REF_COST);
9388 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9389 ins_encode %{
9390 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9391 %}
9392 ins_pipe(pipe_serial);
9393 %}
9394
9395 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9396 predicate(needs_acquiring_load_exclusive(n));
9397 match(Set prev (GetAndSetL mem newv));
9398 ins_cost(VOLATILE_REF_COST);
9399 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9400 ins_encode %{
9401 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9402 %}
9403 ins_pipe(pipe_serial);
9404 %}
9405
9406 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9407 predicate(needs_acquiring_load_exclusive(n));
9408 match(Set prev (GetAndSetN mem newv));
9409 ins_cost(VOLATILE_REF_COST);
9410 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9411 ins_encode %{
9412 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9413 %}
9414 ins_pipe(pipe_serial);
9415 %}
9416
9417 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9418 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9419 match(Set prev (GetAndSetP mem newv));
9420 ins_cost(VOLATILE_REF_COST);
9421 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9422 ins_encode %{
9423 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9424 %}
9425 ins_pipe(pipe_serial);
9426 %}
9427
9428
9429 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9430 match(Set newval (GetAndAddL mem incr));
9431 ins_cost(2 * VOLATILE_REF_COST + 1);
9432 format %{ "get_and_addL $newval, [$mem], $incr" %}
9433 ins_encode %{
9434 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9435 %}
9436 ins_pipe(pipe_serial);
9437 %}
9438
9439 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9440 predicate(n->as_LoadStore()->result_not_used());
9441 match(Set dummy (GetAndAddL mem incr));
9442 ins_cost(2 * VOLATILE_REF_COST);
9443 format %{ "get_and_addL [$mem], $incr" %}
9444 ins_encode %{
9445 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9446 %}
9447 ins_pipe(pipe_serial);
9448 %}
9449
9450 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9451 match(Set newval (GetAndAddL mem incr));
9452 ins_cost(2 * VOLATILE_REF_COST + 1);
9453 format %{ "get_and_addL $newval, [$mem], $incr" %}
9454 ins_encode %{
9455 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9456 %}
9457 ins_pipe(pipe_serial);
9458 %}
9459
9460 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9461 predicate(n->as_LoadStore()->result_not_used());
9462 match(Set dummy (GetAndAddL mem incr));
9463 ins_cost(2 * VOLATILE_REF_COST);
9464 format %{ "get_and_addL [$mem], $incr" %}
9465 ins_encode %{
9466 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9467 %}
9468 ins_pipe(pipe_serial);
9469 %}
9470
9471 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9472 match(Set newval (GetAndAddI mem incr));
9473 ins_cost(2 * VOLATILE_REF_COST + 1);
9474 format %{ "get_and_addI $newval, [$mem], $incr" %}
9475 ins_encode %{
9476 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9477 %}
9478 ins_pipe(pipe_serial);
9479 %}
9480
9481 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9482 predicate(n->as_LoadStore()->result_not_used());
9483 match(Set dummy (GetAndAddI mem incr));
9484 ins_cost(2 * VOLATILE_REF_COST);
9485 format %{ "get_and_addI [$mem], $incr" %}
9486 ins_encode %{
9487 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9488 %}
9489 ins_pipe(pipe_serial);
9490 %}
9491
9492 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9493 match(Set newval (GetAndAddI mem incr));
9494 ins_cost(2 * VOLATILE_REF_COST + 1);
9495 format %{ "get_and_addI $newval, [$mem], $incr" %}
9496 ins_encode %{
9497 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9498 %}
9499 ins_pipe(pipe_serial);
9500 %}
9501
9502 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9503 predicate(n->as_LoadStore()->result_not_used());
9504 match(Set dummy (GetAndAddI mem incr));
9505 ins_cost(2 * VOLATILE_REF_COST);
9506 format %{ "get_and_addI [$mem], $incr" %}
9507 ins_encode %{
9508 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9509 %}
9510 ins_pipe(pipe_serial);
9511 %}
9512
9513 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9514 predicate(needs_acquiring_load_exclusive(n));
9515 match(Set newval (GetAndAddL mem incr));
9516 ins_cost(VOLATILE_REF_COST + 1);
9517 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9518 ins_encode %{
9519 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9520 %}
9521 ins_pipe(pipe_serial);
9522 %}
9523
9524 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9525 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9526 match(Set dummy (GetAndAddL mem incr));
9527 ins_cost(VOLATILE_REF_COST);
9528 format %{ "get_and_addL_acq [$mem], $incr" %}
9529 ins_encode %{
9530 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9531 %}
9532 ins_pipe(pipe_serial);
9533 %}
9534
9535 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9536 predicate(needs_acquiring_load_exclusive(n));
9537 match(Set newval (GetAndAddL mem incr));
9538 ins_cost(VOLATILE_REF_COST + 1);
9539 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9540 ins_encode %{
9541 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9542 %}
9543 ins_pipe(pipe_serial);
9544 %}
9545
9546 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9547 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9548 match(Set dummy (GetAndAddL mem incr));
9549 ins_cost(VOLATILE_REF_COST);
9550 format %{ "get_and_addL_acq [$mem], $incr" %}
9551 ins_encode %{
9552 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9553 %}
9554 ins_pipe(pipe_serial);
9555 %}
9556
9557 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9558 predicate(needs_acquiring_load_exclusive(n));
9559 match(Set newval (GetAndAddI mem incr));
9560 ins_cost(VOLATILE_REF_COST + 1);
9561 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9562 ins_encode %{
9563 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9564 %}
9565 ins_pipe(pipe_serial);
9566 %}
9567
9568 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9569 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9570 match(Set dummy (GetAndAddI mem incr));
9571 ins_cost(VOLATILE_REF_COST);
9572 format %{ "get_and_addI_acq [$mem], $incr" %}
9573 ins_encode %{
9574 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9575 %}
9576 ins_pipe(pipe_serial);
9577 %}
9578
9579 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9580 predicate(needs_acquiring_load_exclusive(n));
9581 match(Set newval (GetAndAddI mem incr));
9582 ins_cost(VOLATILE_REF_COST + 1);
9583 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9584 ins_encode %{
9585 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9586 %}
9587 ins_pipe(pipe_serial);
9588 %}
9589
9590 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9591 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9592 match(Set dummy (GetAndAddI mem incr));
9593 ins_cost(VOLATILE_REF_COST);
9594 format %{ "get_and_addI_acq [$mem], $incr" %}
9595 ins_encode %{
9596 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9597 %}
9598 ins_pipe(pipe_serial);
9599 %}
9600
9601 // Manifest a CmpU result in an integer register.
9602 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9603 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9604 %{
9605 match(Set dst (CmpU3 src1 src2));
9606 effect(KILL flags);
9607
9608 ins_cost(INSN_COST * 3);
9609 format %{
9610 "cmpw $src1, $src2\n\t"
9611 "csetw $dst, ne\n\t"
9612 "cnegw $dst, lo\t# CmpU3(reg)"
9613 %}
9614 ins_encode %{
9615 __ cmpw($src1$$Register, $src2$$Register);
9616 __ csetw($dst$$Register, Assembler::NE);
9617 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9618 %}
9619
9620 ins_pipe(pipe_class_default);
9621 %}
9622
9623 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9624 %{
9625 match(Set dst (CmpU3 src1 src2));
9626 effect(KILL flags);
9627
9628 ins_cost(INSN_COST * 3);
9629 format %{
9630 "subsw zr, $src1, $src2\n\t"
9631 "csetw $dst, ne\n\t"
9632 "cnegw $dst, lo\t# CmpU3(imm)"
9633 %}
9634 ins_encode %{
9635 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9636 __ csetw($dst$$Register, Assembler::NE);
9637 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9638 %}
9639
9640 ins_pipe(pipe_class_default);
9641 %}
9642
9643 // Manifest a CmpUL result in an integer register.
9644 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9645 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9646 %{
9647 match(Set dst (CmpUL3 src1 src2));
9648 effect(KILL flags);
9649
9650 ins_cost(INSN_COST * 3);
9651 format %{
9652 "cmp $src1, $src2\n\t"
9653 "csetw $dst, ne\n\t"
9654 "cnegw $dst, lo\t# CmpUL3(reg)"
9655 %}
9656 ins_encode %{
9657 __ cmp($src1$$Register, $src2$$Register);
9658 __ csetw($dst$$Register, Assembler::NE);
9659 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9660 %}
9661
9662 ins_pipe(pipe_class_default);
9663 %}
9664
9665 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9666 %{
9667 match(Set dst (CmpUL3 src1 src2));
9668 effect(KILL flags);
9669
9670 ins_cost(INSN_COST * 3);
9671 format %{
9672 "subs zr, $src1, $src2\n\t"
9673 "csetw $dst, ne\n\t"
9674 "cnegw $dst, lo\t# CmpUL3(imm)"
9675 %}
9676 ins_encode %{
9677 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9678 __ csetw($dst$$Register, Assembler::NE);
9679 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9680 %}
9681
9682 ins_pipe(pipe_class_default);
9683 %}
9684
9685 // Manifest a CmpL result in an integer register.
9686 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9687 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9688 %{
9689 match(Set dst (CmpL3 src1 src2));
9690 effect(KILL flags);
9691
9692 ins_cost(INSN_COST * 3);
9693 format %{
9694 "cmp $src1, $src2\n\t"
9695 "csetw $dst, ne\n\t"
9696 "cnegw $dst, lt\t# CmpL3(reg)"
9697 %}
9698 ins_encode %{
9699 __ cmp($src1$$Register, $src2$$Register);
9700 __ csetw($dst$$Register, Assembler::NE);
9701 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9702 %}
9703
9704 ins_pipe(pipe_class_default);
9705 %}
9706
9707 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9708 %{
9709 match(Set dst (CmpL3 src1 src2));
9710 effect(KILL flags);
9711
9712 ins_cost(INSN_COST * 3);
9713 format %{
9714 "subs zr, $src1, $src2\n\t"
9715 "csetw $dst, ne\n\t"
9716 "cnegw $dst, lt\t# CmpL3(imm)"
9717 %}
9718 ins_encode %{
9719 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9720 __ csetw($dst$$Register, Assembler::NE);
9721 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9722 %}
9723
9724 ins_pipe(pipe_class_default);
9725 %}
9726
9727 // ============================================================================
9728 // Conditional Move Instructions
9729
9730 // n.b. we have identical rules for both a signed compare op (cmpOp)
9731 // and an unsigned compare op (cmpOpU). it would be nice if we could
9732 // define an op class which merged both inputs and use it to type the
9733 // argument to a single rule. unfortunatelyt his fails because the
9734 // opclass does not live up to the COND_INTER interface of its
9735 // component operands. When the generic code tries to negate the
9736 // operand it ends up running the generci Machoper::negate method
9737 // which throws a ShouldNotHappen. So, we have to provide two flavours
9738 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9739
9740 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9741 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9742
9743 ins_cost(INSN_COST * 2);
9744 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9745
9746 ins_encode %{
9747 __ cselw(as_Register($dst$$reg),
9748 as_Register($src2$$reg),
9749 as_Register($src1$$reg),
9750 (Assembler::Condition)$cmp$$cmpcode);
9751 %}
9752
9753 ins_pipe(icond_reg_reg);
9754 %}
9755
9756 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9757 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9758
9759 ins_cost(INSN_COST * 2);
9760 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9761
9762 ins_encode %{
9763 __ cselw(as_Register($dst$$reg),
9764 as_Register($src2$$reg),
9765 as_Register($src1$$reg),
9766 (Assembler::Condition)$cmp$$cmpcode);
9767 %}
9768
9769 ins_pipe(icond_reg_reg);
9770 %}
9771
9772 // special cases where one arg is zero
9773
9774 // n.b. this is selected in preference to the rule above because it
9775 // avoids loading constant 0 into a source register
9776
9777 // TODO
9778 // we ought only to be able to cull one of these variants as the ideal
9779 // transforms ought always to order the zero consistently (to left/right?)
9780
9781 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9782 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9783
9784 ins_cost(INSN_COST * 2);
9785 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9786
9787 ins_encode %{
9788 __ cselw(as_Register($dst$$reg),
9789 as_Register($src$$reg),
9790 zr,
9791 (Assembler::Condition)$cmp$$cmpcode);
9792 %}
9793
9794 ins_pipe(icond_reg);
9795 %}
9796
9797 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9798 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9799
9800 ins_cost(INSN_COST * 2);
9801 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9802
9803 ins_encode %{
9804 __ cselw(as_Register($dst$$reg),
9805 as_Register($src$$reg),
9806 zr,
9807 (Assembler::Condition)$cmp$$cmpcode);
9808 %}
9809
9810 ins_pipe(icond_reg);
9811 %}
9812
9813 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9814 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9815
9816 ins_cost(INSN_COST * 2);
9817 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9818
9819 ins_encode %{
9820 __ cselw(as_Register($dst$$reg),
9821 zr,
9822 as_Register($src$$reg),
9823 (Assembler::Condition)$cmp$$cmpcode);
9824 %}
9825
9826 ins_pipe(icond_reg);
9827 %}
9828
9829 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9830 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9831
9832 ins_cost(INSN_COST * 2);
9833 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9834
9835 ins_encode %{
9836 __ cselw(as_Register($dst$$reg),
9837 zr,
9838 as_Register($src$$reg),
9839 (Assembler::Condition)$cmp$$cmpcode);
9840 %}
9841
9842 ins_pipe(icond_reg);
9843 %}
9844
9845 // special case for creating a boolean 0 or 1
9846
9847 // n.b. this is selected in preference to the rule above because it
9848 // avoids loading constants 0 and 1 into a source register
9849
9850 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9851 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9852
9853 ins_cost(INSN_COST * 2);
9854 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9855
9856 ins_encode %{
9857 // equivalently
9858 // cset(as_Register($dst$$reg),
9859 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9860 __ csincw(as_Register($dst$$reg),
9861 zr,
9862 zr,
9863 (Assembler::Condition)$cmp$$cmpcode);
9864 %}
9865
9866 ins_pipe(icond_none);
9867 %}
9868
9869 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9870 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9871
9872 ins_cost(INSN_COST * 2);
9873 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9874
9875 ins_encode %{
9876 // equivalently
9877 // cset(as_Register($dst$$reg),
9878 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9879 __ csincw(as_Register($dst$$reg),
9880 zr,
9881 zr,
9882 (Assembler::Condition)$cmp$$cmpcode);
9883 %}
9884
9885 ins_pipe(icond_none);
9886 %}
9887
9888 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9889 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9890
9891 ins_cost(INSN_COST * 2);
9892 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9893
9894 ins_encode %{
9895 __ csel(as_Register($dst$$reg),
9896 as_Register($src2$$reg),
9897 as_Register($src1$$reg),
9898 (Assembler::Condition)$cmp$$cmpcode);
9899 %}
9900
9901 ins_pipe(icond_reg_reg);
9902 %}
9903
9904 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9905 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9906
9907 ins_cost(INSN_COST * 2);
9908 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9909
9910 ins_encode %{
9911 __ csel(as_Register($dst$$reg),
9912 as_Register($src2$$reg),
9913 as_Register($src1$$reg),
9914 (Assembler::Condition)$cmp$$cmpcode);
9915 %}
9916
9917 ins_pipe(icond_reg_reg);
9918 %}
9919
9920 // special cases where one arg is zero
9921
9922 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9923 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9924
9925 ins_cost(INSN_COST * 2);
9926 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9927
9928 ins_encode %{
9929 __ csel(as_Register($dst$$reg),
9930 zr,
9931 as_Register($src$$reg),
9932 (Assembler::Condition)$cmp$$cmpcode);
9933 %}
9934
9935 ins_pipe(icond_reg);
9936 %}
9937
9938 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9939 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9940
9941 ins_cost(INSN_COST * 2);
9942 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9943
9944 ins_encode %{
9945 __ csel(as_Register($dst$$reg),
9946 zr,
9947 as_Register($src$$reg),
9948 (Assembler::Condition)$cmp$$cmpcode);
9949 %}
9950
9951 ins_pipe(icond_reg);
9952 %}
9953
9954 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9955 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9956
9957 ins_cost(INSN_COST * 2);
9958 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9959
9960 ins_encode %{
9961 __ csel(as_Register($dst$$reg),
9962 as_Register($src$$reg),
9963 zr,
9964 (Assembler::Condition)$cmp$$cmpcode);
9965 %}
9966
9967 ins_pipe(icond_reg);
9968 %}
9969
9970 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9971 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9972
9973 ins_cost(INSN_COST * 2);
9974 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9975
9976 ins_encode %{
9977 __ csel(as_Register($dst$$reg),
9978 as_Register($src$$reg),
9979 zr,
9980 (Assembler::Condition)$cmp$$cmpcode);
9981 %}
9982
9983 ins_pipe(icond_reg);
9984 %}
9985
9986 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9987 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9988
9989 ins_cost(INSN_COST * 2);
9990 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9991
9992 ins_encode %{
9993 __ csel(as_Register($dst$$reg),
9994 as_Register($src2$$reg),
9995 as_Register($src1$$reg),
9996 (Assembler::Condition)$cmp$$cmpcode);
9997 %}
9998
9999 ins_pipe(icond_reg_reg);
10000 %}
10001
10002 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10003 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10004
10005 ins_cost(INSN_COST * 2);
10006 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10007
10008 ins_encode %{
10009 __ csel(as_Register($dst$$reg),
10010 as_Register($src2$$reg),
10011 as_Register($src1$$reg),
10012 (Assembler::Condition)$cmp$$cmpcode);
10013 %}
10014
10015 ins_pipe(icond_reg_reg);
10016 %}
10017
10018 // special cases where one arg is zero
10019
10020 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10021 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10022
10023 ins_cost(INSN_COST * 2);
10024 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10025
10026 ins_encode %{
10027 __ csel(as_Register($dst$$reg),
10028 zr,
10029 as_Register($src$$reg),
10030 (Assembler::Condition)$cmp$$cmpcode);
10031 %}
10032
10033 ins_pipe(icond_reg);
10034 %}
10035
10036 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10037 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10038
10039 ins_cost(INSN_COST * 2);
10040 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10041
10042 ins_encode %{
10043 __ csel(as_Register($dst$$reg),
10044 zr,
10045 as_Register($src$$reg),
10046 (Assembler::Condition)$cmp$$cmpcode);
10047 %}
10048
10049 ins_pipe(icond_reg);
10050 %}
10051
10052 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10053 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10054
10055 ins_cost(INSN_COST * 2);
10056 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10057
10058 ins_encode %{
10059 __ csel(as_Register($dst$$reg),
10060 as_Register($src$$reg),
10061 zr,
10062 (Assembler::Condition)$cmp$$cmpcode);
10063 %}
10064
10065 ins_pipe(icond_reg);
10066 %}
10067
10068 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10069 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10070
10071 ins_cost(INSN_COST * 2);
10072 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10073
10074 ins_encode %{
10075 __ csel(as_Register($dst$$reg),
10076 as_Register($src$$reg),
10077 zr,
10078 (Assembler::Condition)$cmp$$cmpcode);
10079 %}
10080
10081 ins_pipe(icond_reg);
10082 %}
10083
10084 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10085 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10086
10087 ins_cost(INSN_COST * 2);
10088 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10089
10090 ins_encode %{
10091 __ cselw(as_Register($dst$$reg),
10092 as_Register($src2$$reg),
10093 as_Register($src1$$reg),
10094 (Assembler::Condition)$cmp$$cmpcode);
10095 %}
10096
10097 ins_pipe(icond_reg_reg);
10098 %}
10099
10100 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10101 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10102
10103 ins_cost(INSN_COST * 2);
10104 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10105
10106 ins_encode %{
10107 __ cselw(as_Register($dst$$reg),
10108 as_Register($src2$$reg),
10109 as_Register($src1$$reg),
10110 (Assembler::Condition)$cmp$$cmpcode);
10111 %}
10112
10113 ins_pipe(icond_reg_reg);
10114 %}
10115
10116 // special cases where one arg is zero
10117
10118 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10119 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10120
10121 ins_cost(INSN_COST * 2);
10122 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10123
10124 ins_encode %{
10125 __ cselw(as_Register($dst$$reg),
10126 zr,
10127 as_Register($src$$reg),
10128 (Assembler::Condition)$cmp$$cmpcode);
10129 %}
10130
10131 ins_pipe(icond_reg);
10132 %}
10133
10134 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10135 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10136
10137 ins_cost(INSN_COST * 2);
10138 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10139
10140 ins_encode %{
10141 __ cselw(as_Register($dst$$reg),
10142 zr,
10143 as_Register($src$$reg),
10144 (Assembler::Condition)$cmp$$cmpcode);
10145 %}
10146
10147 ins_pipe(icond_reg);
10148 %}
10149
10150 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10151 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10152
10153 ins_cost(INSN_COST * 2);
10154 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10155
10156 ins_encode %{
10157 __ cselw(as_Register($dst$$reg),
10158 as_Register($src$$reg),
10159 zr,
10160 (Assembler::Condition)$cmp$$cmpcode);
10161 %}
10162
10163 ins_pipe(icond_reg);
10164 %}
10165
10166 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10167 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10168
10169 ins_cost(INSN_COST * 2);
10170 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10171
10172 ins_encode %{
10173 __ cselw(as_Register($dst$$reg),
10174 as_Register($src$$reg),
10175 zr,
10176 (Assembler::Condition)$cmp$$cmpcode);
10177 %}
10178
10179 ins_pipe(icond_reg);
10180 %}
10181
10182 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10183 %{
10184 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10185
10186 ins_cost(INSN_COST * 3);
10187
10188 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10189 ins_encode %{
10190 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10191 __ fcsels(as_FloatRegister($dst$$reg),
10192 as_FloatRegister($src2$$reg),
10193 as_FloatRegister($src1$$reg),
10194 cond);
10195 %}
10196
10197 ins_pipe(fp_cond_reg_reg_s);
10198 %}
10199
10200 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10201 %{
10202 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10203
10204 ins_cost(INSN_COST * 3);
10205
10206 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10207 ins_encode %{
10208 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10209 __ fcsels(as_FloatRegister($dst$$reg),
10210 as_FloatRegister($src2$$reg),
10211 as_FloatRegister($src1$$reg),
10212 cond);
10213 %}
10214
10215 ins_pipe(fp_cond_reg_reg_s);
10216 %}
10217
10218 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10219 %{
10220 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10221
10222 ins_cost(INSN_COST * 3);
10223
10224 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10225 ins_encode %{
10226 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10227 __ fcseld(as_FloatRegister($dst$$reg),
10228 as_FloatRegister($src2$$reg),
10229 as_FloatRegister($src1$$reg),
10230 cond);
10231 %}
10232
10233 ins_pipe(fp_cond_reg_reg_d);
10234 %}
10235
10236 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10237 %{
10238 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10239
10240 ins_cost(INSN_COST * 3);
10241
10242 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10243 ins_encode %{
10244 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10245 __ fcseld(as_FloatRegister($dst$$reg),
10246 as_FloatRegister($src2$$reg),
10247 as_FloatRegister($src1$$reg),
10248 cond);
10249 %}
10250
10251 ins_pipe(fp_cond_reg_reg_d);
10252 %}
10253
10254 // ============================================================================
10255 // Arithmetic Instructions
10256 //
10257
10258 // Integer Addition
10259
10260 // TODO
10261 // these currently employ operations which do not set CR and hence are
10262 // not flagged as killing CR but we would like to isolate the cases
10263 // where we want to set flags from those where we don't. need to work
10264 // out how to do that.
10265
10266 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10267 match(Set dst (AddI src1 src2));
10268
10269 ins_cost(INSN_COST);
10270 format %{ "addw $dst, $src1, $src2" %}
10271
10272 ins_encode %{
10273 __ addw(as_Register($dst$$reg),
10274 as_Register($src1$$reg),
10275 as_Register($src2$$reg));
10276 %}
10277
10278 ins_pipe(ialu_reg_reg);
10279 %}
10280
10281 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10282 match(Set dst (AddI src1 src2));
10283
10284 ins_cost(INSN_COST);
10285 format %{ "addw $dst, $src1, $src2" %}
10286
10287 // use opcode to indicate that this is an add not a sub
10288 opcode(0x0);
10289
10290 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10291
10292 ins_pipe(ialu_reg_imm);
10293 %}
10294
10295 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10296 match(Set dst (AddI (ConvL2I src1) src2));
10297
10298 ins_cost(INSN_COST);
10299 format %{ "addw $dst, $src1, $src2" %}
10300
10301 // use opcode to indicate that this is an add not a sub
10302 opcode(0x0);
10303
10304 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10305
10306 ins_pipe(ialu_reg_imm);
10307 %}
10308
10309 // Pointer Addition
10310 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10311 match(Set dst (AddP src1 src2));
10312
10313 ins_cost(INSN_COST);
10314 format %{ "add $dst, $src1, $src2\t# ptr" %}
10315
10316 ins_encode %{
10317 __ add(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg));
10320 %}
10321
10322 ins_pipe(ialu_reg_reg);
10323 %}
10324
10325 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10326 match(Set dst (AddP src1 (ConvI2L src2)));
10327
10328 ins_cost(1.9 * INSN_COST);
10329 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10330
10331 ins_encode %{
10332 __ add(as_Register($dst$$reg),
10333 as_Register($src1$$reg),
10334 as_Register($src2$$reg), ext::sxtw);
10335 %}
10336
10337 ins_pipe(ialu_reg_reg);
10338 %}
10339
10340 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10341 match(Set dst (AddP src1 (LShiftL src2 scale)));
10342
10343 ins_cost(1.9 * INSN_COST);
10344 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10345
10346 ins_encode %{
10347 __ lea(as_Register($dst$$reg),
10348 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10349 Address::lsl($scale$$constant)));
10350 %}
10351
10352 ins_pipe(ialu_reg_reg_shift);
10353 %}
10354
10355 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10356 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10357
10358 ins_cost(1.9 * INSN_COST);
10359 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10360
10361 ins_encode %{
10362 __ lea(as_Register($dst$$reg),
10363 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10364 Address::sxtw($scale$$constant)));
10365 %}
10366
10367 ins_pipe(ialu_reg_reg_shift);
10368 %}
10369
10370 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10371 match(Set dst (LShiftL (ConvI2L src) scale));
10372
10373 ins_cost(INSN_COST);
10374 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10375
10376 ins_encode %{
10377 __ sbfiz(as_Register($dst$$reg),
10378 as_Register($src$$reg),
10379 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10380 %}
10381
10382 ins_pipe(ialu_reg_shift);
10383 %}
10384
10385 // Pointer Immediate Addition
10386 // n.b. this needs to be more expensive than using an indirect memory
10387 // operand
10388 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10389 match(Set dst (AddP src1 src2));
10390
10391 ins_cost(INSN_COST);
10392 format %{ "add $dst, $src1, $src2\t# ptr" %}
10393
10394 // use opcode to indicate that this is an add not a sub
10395 opcode(0x0);
10396
10397 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10398
10399 ins_pipe(ialu_reg_imm);
10400 %}
10401
10402 // Long Addition
10403 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10404
10405 match(Set dst (AddL src1 src2));
10406
10407 ins_cost(INSN_COST);
10408 format %{ "add $dst, $src1, $src2" %}
10409
10410 ins_encode %{
10411 __ add(as_Register($dst$$reg),
10412 as_Register($src1$$reg),
10413 as_Register($src2$$reg));
10414 %}
10415
10416 ins_pipe(ialu_reg_reg);
10417 %}
10418
10419 // No constant pool entries requiredLong Immediate Addition.
10420 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10421 match(Set dst (AddL src1 src2));
10422
10423 ins_cost(INSN_COST);
10424 format %{ "add $dst, $src1, $src2" %}
10425
10426 // use opcode to indicate that this is an add not a sub
10427 opcode(0x0);
10428
10429 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10430
10431 ins_pipe(ialu_reg_imm);
10432 %}
10433
10434 // Integer Subtraction
10435 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10436 match(Set dst (SubI src1 src2));
10437
10438 ins_cost(INSN_COST);
10439 format %{ "subw $dst, $src1, $src2" %}
10440
10441 ins_encode %{
10442 __ subw(as_Register($dst$$reg),
10443 as_Register($src1$$reg),
10444 as_Register($src2$$reg));
10445 %}
10446
10447 ins_pipe(ialu_reg_reg);
10448 %}
10449
10450 // Immediate Subtraction
10451 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10452 match(Set dst (SubI src1 src2));
10453
10454 ins_cost(INSN_COST);
10455 format %{ "subw $dst, $src1, $src2" %}
10456
10457 // use opcode to indicate that this is a sub not an add
10458 opcode(0x1);
10459
10460 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10461
10462 ins_pipe(ialu_reg_imm);
10463 %}
10464
10465 // Long Subtraction
10466 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10467
10468 match(Set dst (SubL src1 src2));
10469
10470 ins_cost(INSN_COST);
10471 format %{ "sub $dst, $src1, $src2" %}
10472
10473 ins_encode %{
10474 __ sub(as_Register($dst$$reg),
10475 as_Register($src1$$reg),
10476 as_Register($src2$$reg));
10477 %}
10478
10479 ins_pipe(ialu_reg_reg);
10480 %}
10481
10482 // No constant pool entries requiredLong Immediate Subtraction.
10483 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10484 match(Set dst (SubL src1 src2));
10485
10486 ins_cost(INSN_COST);
10487 format %{ "sub$dst, $src1, $src2" %}
10488
10489 // use opcode to indicate that this is a sub not an add
10490 opcode(0x1);
10491
10492 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10493
10494 ins_pipe(ialu_reg_imm);
10495 %}
10496
10497 // Integer Negation (special case for sub)
10498
10499 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10500 match(Set dst (SubI zero src));
10501
10502 ins_cost(INSN_COST);
10503 format %{ "negw $dst, $src\t# int" %}
10504
10505 ins_encode %{
10506 __ negw(as_Register($dst$$reg),
10507 as_Register($src$$reg));
10508 %}
10509
10510 ins_pipe(ialu_reg);
10511 %}
10512
10513 // Long Negation
10514
10515 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10516 match(Set dst (SubL zero src));
10517
10518 ins_cost(INSN_COST);
10519 format %{ "neg $dst, $src\t# long" %}
10520
10521 ins_encode %{
10522 __ neg(as_Register($dst$$reg),
10523 as_Register($src$$reg));
10524 %}
10525
10526 ins_pipe(ialu_reg);
10527 %}
10528
10529 // Integer Multiply
10530
10531 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10532 match(Set dst (MulI src1 src2));
10533
10534 ins_cost(INSN_COST * 3);
10535 format %{ "mulw $dst, $src1, $src2" %}
10536
10537 ins_encode %{
10538 __ mulw(as_Register($dst$$reg),
10539 as_Register($src1$$reg),
10540 as_Register($src2$$reg));
10541 %}
10542
10543 ins_pipe(imul_reg_reg);
10544 %}
10545
10546 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10547 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10548
10549 ins_cost(INSN_COST * 3);
10550 format %{ "smull $dst, $src1, $src2" %}
10551
10552 ins_encode %{
10553 __ smull(as_Register($dst$$reg),
10554 as_Register($src1$$reg),
10555 as_Register($src2$$reg));
10556 %}
10557
10558 ins_pipe(imul_reg_reg);
10559 %}
10560
10561 // Long Multiply
10562
10563 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10564 match(Set dst (MulL src1 src2));
10565
10566 ins_cost(INSN_COST * 5);
10567 format %{ "mul $dst, $src1, $src2" %}
10568
10569 ins_encode %{
10570 __ mul(as_Register($dst$$reg),
10571 as_Register($src1$$reg),
10572 as_Register($src2$$reg));
10573 %}
10574
10575 ins_pipe(lmul_reg_reg);
10576 %}
10577
10578 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10579 %{
10580 match(Set dst (MulHiL src1 src2));
10581
10582 ins_cost(INSN_COST * 7);
10583 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10584
10585 ins_encode %{
10586 __ smulh(as_Register($dst$$reg),
10587 as_Register($src1$$reg),
10588 as_Register($src2$$reg));
10589 %}
10590
10591 ins_pipe(lmul_reg_reg);
10592 %}
10593
10594 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10595 %{
10596 match(Set dst (UMulHiL src1 src2));
10597
10598 ins_cost(INSN_COST * 7);
10599 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10600
10601 ins_encode %{
10602 __ umulh(as_Register($dst$$reg),
10603 as_Register($src1$$reg),
10604 as_Register($src2$$reg));
10605 %}
10606
10607 ins_pipe(lmul_reg_reg);
10608 %}
10609
10610 // Combined Integer Multiply & Add/Sub
10611
10612 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10613 match(Set dst (AddI src3 (MulI src1 src2)));
10614
10615 ins_cost(INSN_COST * 3);
10616 format %{ "madd $dst, $src1, $src2, $src3" %}
10617
10618 ins_encode %{
10619 __ maddw(as_Register($dst$$reg),
10620 as_Register($src1$$reg),
10621 as_Register($src2$$reg),
10622 as_Register($src3$$reg));
10623 %}
10624
10625 ins_pipe(imac_reg_reg);
10626 %}
10627
10628 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10629 match(Set dst (SubI src3 (MulI src1 src2)));
10630
10631 ins_cost(INSN_COST * 3);
10632 format %{ "msub $dst, $src1, $src2, $src3" %}
10633
10634 ins_encode %{
10635 __ msubw(as_Register($dst$$reg),
10636 as_Register($src1$$reg),
10637 as_Register($src2$$reg),
10638 as_Register($src3$$reg));
10639 %}
10640
10641 ins_pipe(imac_reg_reg);
10642 %}
10643
10644 // Combined Integer Multiply & Neg
10645
10646 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10647 match(Set dst (MulI (SubI zero src1) src2));
10648
10649 ins_cost(INSN_COST * 3);
10650 format %{ "mneg $dst, $src1, $src2" %}
10651
10652 ins_encode %{
10653 __ mnegw(as_Register($dst$$reg),
10654 as_Register($src1$$reg),
10655 as_Register($src2$$reg));
10656 %}
10657
10658 ins_pipe(imac_reg_reg);
10659 %}
10660
10661 // Combined Long Multiply & Add/Sub
10662
10663 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10664 match(Set dst (AddL src3 (MulL src1 src2)));
10665
10666 ins_cost(INSN_COST * 5);
10667 format %{ "madd $dst, $src1, $src2, $src3" %}
10668
10669 ins_encode %{
10670 __ madd(as_Register($dst$$reg),
10671 as_Register($src1$$reg),
10672 as_Register($src2$$reg),
10673 as_Register($src3$$reg));
10674 %}
10675
10676 ins_pipe(lmac_reg_reg);
10677 %}
10678
10679 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10680 match(Set dst (SubL src3 (MulL src1 src2)));
10681
10682 ins_cost(INSN_COST * 5);
10683 format %{ "msub $dst, $src1, $src2, $src3" %}
10684
10685 ins_encode %{
10686 __ msub(as_Register($dst$$reg),
10687 as_Register($src1$$reg),
10688 as_Register($src2$$reg),
10689 as_Register($src3$$reg));
10690 %}
10691
10692 ins_pipe(lmac_reg_reg);
10693 %}
10694
10695 // Combined Long Multiply & Neg
10696
10697 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10698 match(Set dst (MulL (SubL zero src1) src2));
10699
10700 ins_cost(INSN_COST * 5);
10701 format %{ "mneg $dst, $src1, $src2" %}
10702
10703 ins_encode %{
10704 __ mneg(as_Register($dst$$reg),
10705 as_Register($src1$$reg),
10706 as_Register($src2$$reg));
10707 %}
10708
10709 ins_pipe(lmac_reg_reg);
10710 %}
10711
10712 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10713
10714 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10715 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10716
10717 ins_cost(INSN_COST * 3);
10718 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10719
10720 ins_encode %{
10721 __ smaddl(as_Register($dst$$reg),
10722 as_Register($src1$$reg),
10723 as_Register($src2$$reg),
10724 as_Register($src3$$reg));
10725 %}
10726
10727 ins_pipe(imac_reg_reg);
10728 %}
10729
10730 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10731 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10732
10733 ins_cost(INSN_COST * 3);
10734 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10735
10736 ins_encode %{
10737 __ smsubl(as_Register($dst$$reg),
10738 as_Register($src1$$reg),
10739 as_Register($src2$$reg),
10740 as_Register($src3$$reg));
10741 %}
10742
10743 ins_pipe(imac_reg_reg);
10744 %}
10745
10746 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10747 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10748
10749 ins_cost(INSN_COST * 3);
10750 format %{ "smnegl $dst, $src1, $src2" %}
10751
10752 ins_encode %{
10753 __ smnegl(as_Register($dst$$reg),
10754 as_Register($src1$$reg),
10755 as_Register($src2$$reg));
10756 %}
10757
10758 ins_pipe(imac_reg_reg);
10759 %}
10760
10761 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10762
10763 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10764 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10765
10766 ins_cost(INSN_COST * 5);
10767 format %{ "mulw rscratch1, $src1, $src2\n\t"
10768 "maddw $dst, $src3, $src4, rscratch1" %}
10769
10770 ins_encode %{
10771 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10772 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10773
10774 ins_pipe(imac_reg_reg);
10775 %}
10776
10777 // Integer Divide
10778
10779 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10780 match(Set dst (DivI src1 src2));
10781
10782 ins_cost(INSN_COST * 19);
10783 format %{ "sdivw $dst, $src1, $src2" %}
10784
10785 ins_encode(aarch64_enc_divw(dst, src1, src2));
10786 ins_pipe(idiv_reg_reg);
10787 %}
10788
10789 // Long Divide
10790
10791 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10792 match(Set dst (DivL src1 src2));
10793
10794 ins_cost(INSN_COST * 35);
10795 format %{ "sdiv $dst, $src1, $src2" %}
10796
10797 ins_encode(aarch64_enc_div(dst, src1, src2));
10798 ins_pipe(ldiv_reg_reg);
10799 %}
10800
10801 // Integer Remainder
10802
10803 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10804 match(Set dst (ModI src1 src2));
10805
10806 ins_cost(INSN_COST * 22);
10807 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10808 "msubw $dst, rscratch1, $src2, $src1" %}
10809
10810 ins_encode(aarch64_enc_modw(dst, src1, src2));
10811 ins_pipe(idiv_reg_reg);
10812 %}
10813
10814 // Long Remainder
10815
10816 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10817 match(Set dst (ModL src1 src2));
10818
10819 ins_cost(INSN_COST * 38);
10820 format %{ "sdiv rscratch1, $src1, $src2\n"
10821 "msub $dst, rscratch1, $src2, $src1" %}
10822
10823 ins_encode(aarch64_enc_mod(dst, src1, src2));
10824 ins_pipe(ldiv_reg_reg);
10825 %}
10826
10827 // Unsigned Integer Divide
10828
10829 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10830 match(Set dst (UDivI src1 src2));
10831
10832 ins_cost(INSN_COST * 19);
10833 format %{ "udivw $dst, $src1, $src2" %}
10834
10835 ins_encode %{
10836 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10837 %}
10838
10839 ins_pipe(idiv_reg_reg);
10840 %}
10841
10842 // Unsigned Long Divide
10843
10844 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10845 match(Set dst (UDivL src1 src2));
10846
10847 ins_cost(INSN_COST * 35);
10848 format %{ "udiv $dst, $src1, $src2" %}
10849
10850 ins_encode %{
10851 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10852 %}
10853
10854 ins_pipe(ldiv_reg_reg);
10855 %}
10856
10857 // Unsigned Integer Remainder
10858
10859 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10860 match(Set dst (UModI src1 src2));
10861
10862 ins_cost(INSN_COST * 22);
10863 format %{ "udivw rscratch1, $src1, $src2\n\t"
10864 "msubw $dst, rscratch1, $src2, $src1" %}
10865
10866 ins_encode %{
10867 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10868 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10869 %}
10870
10871 ins_pipe(idiv_reg_reg);
10872 %}
10873
10874 // Unsigned Long Remainder
10875
10876 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10877 match(Set dst (UModL src1 src2));
10878
10879 ins_cost(INSN_COST * 38);
10880 format %{ "udiv rscratch1, $src1, $src2\n"
10881 "msub $dst, rscratch1, $src2, $src1" %}
10882
10883 ins_encode %{
10884 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10885 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10886 %}
10887
10888 ins_pipe(ldiv_reg_reg);
10889 %}
10890
10891 // Integer Shifts
10892
10893 // Shift Left Register
10894 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10895 match(Set dst (LShiftI src1 src2));
10896
10897 ins_cost(INSN_COST * 2);
10898 format %{ "lslvw $dst, $src1, $src2" %}
10899
10900 ins_encode %{
10901 __ lslvw(as_Register($dst$$reg),
10902 as_Register($src1$$reg),
10903 as_Register($src2$$reg));
10904 %}
10905
10906 ins_pipe(ialu_reg_reg_vshift);
10907 %}
10908
10909 // Shift Left Immediate
10910 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10911 match(Set dst (LShiftI src1 src2));
10912
10913 ins_cost(INSN_COST);
10914 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10915
10916 ins_encode %{
10917 __ lslw(as_Register($dst$$reg),
10918 as_Register($src1$$reg),
10919 $src2$$constant & 0x1f);
10920 %}
10921
10922 ins_pipe(ialu_reg_shift);
10923 %}
10924
10925 // Shift Right Logical Register
10926 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10927 match(Set dst (URShiftI src1 src2));
10928
10929 ins_cost(INSN_COST * 2);
10930 format %{ "lsrvw $dst, $src1, $src2" %}
10931
10932 ins_encode %{
10933 __ lsrvw(as_Register($dst$$reg),
10934 as_Register($src1$$reg),
10935 as_Register($src2$$reg));
10936 %}
10937
10938 ins_pipe(ialu_reg_reg_vshift);
10939 %}
10940
10941 // Shift Right Logical Immediate
10942 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10943 match(Set dst (URShiftI src1 src2));
10944
10945 ins_cost(INSN_COST);
10946 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10947
10948 ins_encode %{
10949 __ lsrw(as_Register($dst$$reg),
10950 as_Register($src1$$reg),
10951 $src2$$constant & 0x1f);
10952 %}
10953
10954 ins_pipe(ialu_reg_shift);
10955 %}
10956
10957 // Shift Right Arithmetic Register
10958 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10959 match(Set dst (RShiftI src1 src2));
10960
10961 ins_cost(INSN_COST * 2);
10962 format %{ "asrvw $dst, $src1, $src2" %}
10963
10964 ins_encode %{
10965 __ asrvw(as_Register($dst$$reg),
10966 as_Register($src1$$reg),
10967 as_Register($src2$$reg));
10968 %}
10969
10970 ins_pipe(ialu_reg_reg_vshift);
10971 %}
10972
10973 // Shift Right Arithmetic Immediate
10974 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10975 match(Set dst (RShiftI src1 src2));
10976
10977 ins_cost(INSN_COST);
10978 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10979
10980 ins_encode %{
10981 __ asrw(as_Register($dst$$reg),
10982 as_Register($src1$$reg),
10983 $src2$$constant & 0x1f);
10984 %}
10985
10986 ins_pipe(ialu_reg_shift);
10987 %}
10988
10989 // Combined Int Mask and Right Shift (using UBFM)
10990 // TODO
10991
10992 // Long Shifts
10993
10994 // Shift Left Register
10995 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10996 match(Set dst (LShiftL src1 src2));
10997
10998 ins_cost(INSN_COST * 2);
10999 format %{ "lslv $dst, $src1, $src2" %}
11000
11001 ins_encode %{
11002 __ lslv(as_Register($dst$$reg),
11003 as_Register($src1$$reg),
11004 as_Register($src2$$reg));
11005 %}
11006
11007 ins_pipe(ialu_reg_reg_vshift);
11008 %}
11009
11010 // Shift Left Immediate
11011 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11012 match(Set dst (LShiftL src1 src2));
11013
11014 ins_cost(INSN_COST);
11015 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11016
11017 ins_encode %{
11018 __ lsl(as_Register($dst$$reg),
11019 as_Register($src1$$reg),
11020 $src2$$constant & 0x3f);
11021 %}
11022
11023 ins_pipe(ialu_reg_shift);
11024 %}
11025
11026 // Shift Right Logical Register
11027 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11028 match(Set dst (URShiftL src1 src2));
11029
11030 ins_cost(INSN_COST * 2);
11031 format %{ "lsrv $dst, $src1, $src2" %}
11032
11033 ins_encode %{
11034 __ lsrv(as_Register($dst$$reg),
11035 as_Register($src1$$reg),
11036 as_Register($src2$$reg));
11037 %}
11038
11039 ins_pipe(ialu_reg_reg_vshift);
11040 %}
11041
11042 // Shift Right Logical Immediate
11043 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11044 match(Set dst (URShiftL src1 src2));
11045
11046 ins_cost(INSN_COST);
11047 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11048
11049 ins_encode %{
11050 __ lsr(as_Register($dst$$reg),
11051 as_Register($src1$$reg),
11052 $src2$$constant & 0x3f);
11053 %}
11054
11055 ins_pipe(ialu_reg_shift);
11056 %}
11057
11058 // A special-case pattern for card table stores.
11059 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11060 match(Set dst (URShiftL (CastP2X src1) src2));
11061
11062 ins_cost(INSN_COST);
11063 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11064
11065 ins_encode %{
11066 __ lsr(as_Register($dst$$reg),
11067 as_Register($src1$$reg),
11068 $src2$$constant & 0x3f);
11069 %}
11070
11071 ins_pipe(ialu_reg_shift);
11072 %}
11073
11074 // Shift Right Arithmetic Register
11075 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11076 match(Set dst (RShiftL src1 src2));
11077
11078 ins_cost(INSN_COST * 2);
11079 format %{ "asrv $dst, $src1, $src2" %}
11080
11081 ins_encode %{
11082 __ asrv(as_Register($dst$$reg),
11083 as_Register($src1$$reg),
11084 as_Register($src2$$reg));
11085 %}
11086
11087 ins_pipe(ialu_reg_reg_vshift);
11088 %}
11089
11090 // Shift Right Arithmetic Immediate
11091 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11092 match(Set dst (RShiftL src1 src2));
11093
11094 ins_cost(INSN_COST);
11095 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11096
11097 ins_encode %{
11098 __ asr(as_Register($dst$$reg),
11099 as_Register($src1$$reg),
11100 $src2$$constant & 0x3f);
11101 %}
11102
11103 ins_pipe(ialu_reg_shift);
11104 %}
11105
11106 // BEGIN This section of the file is automatically generated. Do not edit --------------
11107 // This section is generated from aarch64_ad.m4
11108
11109 // This pattern is automatically generated from aarch64_ad.m4
11110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11111 instruct regL_not_reg(iRegLNoSp dst,
11112 iRegL src1, immL_M1 m1,
11113 rFlagsReg cr) %{
11114 match(Set dst (XorL src1 m1));
11115 ins_cost(INSN_COST);
11116 format %{ "eon $dst, $src1, zr" %}
11117
11118 ins_encode %{
11119 __ eon(as_Register($dst$$reg),
11120 as_Register($src1$$reg),
11121 zr,
11122 Assembler::LSL, 0);
11123 %}
11124
11125 ins_pipe(ialu_reg);
11126 %}
11127
11128 // This pattern is automatically generated from aarch64_ad.m4
11129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11130 instruct regI_not_reg(iRegINoSp dst,
11131 iRegIorL2I src1, immI_M1 m1,
11132 rFlagsReg cr) %{
11133 match(Set dst (XorI src1 m1));
11134 ins_cost(INSN_COST);
11135 format %{ "eonw $dst, $src1, zr" %}
11136
11137 ins_encode %{
11138 __ eonw(as_Register($dst$$reg),
11139 as_Register($src1$$reg),
11140 zr,
11141 Assembler::LSL, 0);
11142 %}
11143
11144 ins_pipe(ialu_reg);
11145 %}
11146
11147 // This pattern is automatically generated from aarch64_ad.m4
11148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11149 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11150 immI0 zero, iRegIorL2I src1, immI src2) %{
11151 match(Set dst (SubI zero (URShiftI src1 src2)));
11152
11153 ins_cost(1.9 * INSN_COST);
11154 format %{ "negw $dst, $src1, LSR $src2" %}
11155
11156 ins_encode %{
11157 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11158 Assembler::LSR, $src2$$constant & 0x1f);
11159 %}
11160
11161 ins_pipe(ialu_reg_shift);
11162 %}
11163
11164 // This pattern is automatically generated from aarch64_ad.m4
11165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11166 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11167 immI0 zero, iRegIorL2I src1, immI src2) %{
11168 match(Set dst (SubI zero (RShiftI src1 src2)));
11169
11170 ins_cost(1.9 * INSN_COST);
11171 format %{ "negw $dst, $src1, ASR $src2" %}
11172
11173 ins_encode %{
11174 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11175 Assembler::ASR, $src2$$constant & 0x1f);
11176 %}
11177
11178 ins_pipe(ialu_reg_shift);
11179 %}
11180
11181 // This pattern is automatically generated from aarch64_ad.m4
11182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11183 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11184 immI0 zero, iRegIorL2I src1, immI src2) %{
11185 match(Set dst (SubI zero (LShiftI src1 src2)));
11186
11187 ins_cost(1.9 * INSN_COST);
11188 format %{ "negw $dst, $src1, LSL $src2" %}
11189
11190 ins_encode %{
11191 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11192 Assembler::LSL, $src2$$constant & 0x1f);
11193 %}
11194
11195 ins_pipe(ialu_reg_shift);
11196 %}
11197
11198 // This pattern is automatically generated from aarch64_ad.m4
11199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11200 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11201 immL0 zero, iRegL src1, immI src2) %{
11202 match(Set dst (SubL zero (URShiftL src1 src2)));
11203
11204 ins_cost(1.9 * INSN_COST);
11205 format %{ "neg $dst, $src1, LSR $src2" %}
11206
11207 ins_encode %{
11208 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11209 Assembler::LSR, $src2$$constant & 0x3f);
11210 %}
11211
11212 ins_pipe(ialu_reg_shift);
11213 %}
11214
11215 // This pattern is automatically generated from aarch64_ad.m4
11216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11217 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11218 immL0 zero, iRegL src1, immI src2) %{
11219 match(Set dst (SubL zero (RShiftL src1 src2)));
11220
11221 ins_cost(1.9 * INSN_COST);
11222 format %{ "neg $dst, $src1, ASR $src2" %}
11223
11224 ins_encode %{
11225 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11226 Assembler::ASR, $src2$$constant & 0x3f);
11227 %}
11228
11229 ins_pipe(ialu_reg_shift);
11230 %}
11231
11232 // This pattern is automatically generated from aarch64_ad.m4
11233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11234 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11235 immL0 zero, iRegL src1, immI src2) %{
11236 match(Set dst (SubL zero (LShiftL src1 src2)));
11237
11238 ins_cost(1.9 * INSN_COST);
11239 format %{ "neg $dst, $src1, LSL $src2" %}
11240
11241 ins_encode %{
11242 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11243 Assembler::LSL, $src2$$constant & 0x3f);
11244 %}
11245
11246 ins_pipe(ialu_reg_shift);
11247 %}
11248
11249 // This pattern is automatically generated from aarch64_ad.m4
11250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11251 instruct AndI_reg_not_reg(iRegINoSp dst,
11252 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11253 match(Set dst (AndI src1 (XorI src2 m1)));
11254 ins_cost(INSN_COST);
11255 format %{ "bicw $dst, $src1, $src2" %}
11256
11257 ins_encode %{
11258 __ bicw(as_Register($dst$$reg),
11259 as_Register($src1$$reg),
11260 as_Register($src2$$reg),
11261 Assembler::LSL, 0);
11262 %}
11263
11264 ins_pipe(ialu_reg_reg);
11265 %}
11266
11267 // This pattern is automatically generated from aarch64_ad.m4
11268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11269 instruct AndL_reg_not_reg(iRegLNoSp dst,
11270 iRegL src1, iRegL src2, immL_M1 m1) %{
11271 match(Set dst (AndL src1 (XorL src2 m1)));
11272 ins_cost(INSN_COST);
11273 format %{ "bic $dst, $src1, $src2" %}
11274
11275 ins_encode %{
11276 __ bic(as_Register($dst$$reg),
11277 as_Register($src1$$reg),
11278 as_Register($src2$$reg),
11279 Assembler::LSL, 0);
11280 %}
11281
11282 ins_pipe(ialu_reg_reg);
11283 %}
11284
11285 // This pattern is automatically generated from aarch64_ad.m4
11286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11287 instruct OrI_reg_not_reg(iRegINoSp dst,
11288 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11289 match(Set dst (OrI src1 (XorI src2 m1)));
11290 ins_cost(INSN_COST);
11291 format %{ "ornw $dst, $src1, $src2" %}
11292
11293 ins_encode %{
11294 __ ornw(as_Register($dst$$reg),
11295 as_Register($src1$$reg),
11296 as_Register($src2$$reg),
11297 Assembler::LSL, 0);
11298 %}
11299
11300 ins_pipe(ialu_reg_reg);
11301 %}
11302
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 instruct OrL_reg_not_reg(iRegLNoSp dst,
11306 iRegL src1, iRegL src2, immL_M1 m1) %{
11307 match(Set dst (OrL src1 (XorL src2 m1)));
11308 ins_cost(INSN_COST);
11309 format %{ "orn $dst, $src1, $src2" %}
11310
11311 ins_encode %{
11312 __ orn(as_Register($dst$$reg),
11313 as_Register($src1$$reg),
11314 as_Register($src2$$reg),
11315 Assembler::LSL, 0);
11316 %}
11317
11318 ins_pipe(ialu_reg_reg);
11319 %}
11320
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 instruct XorI_reg_not_reg(iRegINoSp dst,
11324 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11325 match(Set dst (XorI m1 (XorI src2 src1)));
11326 ins_cost(INSN_COST);
11327 format %{ "eonw $dst, $src1, $src2" %}
11328
11329 ins_encode %{
11330 __ eonw(as_Register($dst$$reg),
11331 as_Register($src1$$reg),
11332 as_Register($src2$$reg),
11333 Assembler::LSL, 0);
11334 %}
11335
11336 ins_pipe(ialu_reg_reg);
11337 %}
11338
11339 // This pattern is automatically generated from aarch64_ad.m4
11340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11341 instruct XorL_reg_not_reg(iRegLNoSp dst,
11342 iRegL src1, iRegL src2, immL_M1 m1) %{
11343 match(Set dst (XorL m1 (XorL src2 src1)));
11344 ins_cost(INSN_COST);
11345 format %{ "eon $dst, $src1, $src2" %}
11346
11347 ins_encode %{
11348 __ eon(as_Register($dst$$reg),
11349 as_Register($src1$$reg),
11350 as_Register($src2$$reg),
11351 Assembler::LSL, 0);
11352 %}
11353
11354 ins_pipe(ialu_reg_reg);
11355 %}
11356
11357 // This pattern is automatically generated from aarch64_ad.m4
11358 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11359 // val & (-1 ^ (val >>> shift)) ==> bicw
11360 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11361 iRegIorL2I src1, iRegIorL2I src2,
11362 immI src3, immI_M1 src4) %{
11363 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11364 ins_cost(1.9 * INSN_COST);
11365 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11366
11367 ins_encode %{
11368 __ bicw(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg),
11371 Assembler::LSR,
11372 $src3$$constant & 0x1f);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg_shift);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 // val & (-1 ^ (val >>> shift)) ==> bic
11381 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11382 iRegL src1, iRegL src2,
11383 immI src3, immL_M1 src4) %{
11384 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11387
11388 ins_encode %{
11389 __ bic(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSR,
11393 $src3$$constant & 0x3f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 // val & (-1 ^ (val >> shift)) ==> bicw
11402 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11403 iRegIorL2I src1, iRegIorL2I src2,
11404 immI src3, immI_M1 src4) %{
11405 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11408
11409 ins_encode %{
11410 __ bicw(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::ASR,
11414 $src3$$constant & 0x1f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 // val & (-1 ^ (val >> shift)) ==> bic
11423 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11424 iRegL src1, iRegL src2,
11425 immI src3, immL_M1 src4) %{
11426 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11427 ins_cost(1.9 * INSN_COST);
11428 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11429
11430 ins_encode %{
11431 __ bic(as_Register($dst$$reg),
11432 as_Register($src1$$reg),
11433 as_Register($src2$$reg),
11434 Assembler::ASR,
11435 $src3$$constant & 0x3f);
11436 %}
11437
11438 ins_pipe(ialu_reg_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 // val & (-1 ^ (val ror shift)) ==> bicw
11444 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11445 iRegIorL2I src1, iRegIorL2I src2,
11446 immI src3, immI_M1 src4) %{
11447 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11448 ins_cost(1.9 * INSN_COST);
11449 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11450
11451 ins_encode %{
11452 __ bicw(as_Register($dst$$reg),
11453 as_Register($src1$$reg),
11454 as_Register($src2$$reg),
11455 Assembler::ROR,
11456 $src3$$constant & 0x1f);
11457 %}
11458
11459 ins_pipe(ialu_reg_reg_shift);
11460 %}
11461
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 // val & (-1 ^ (val ror shift)) ==> bic
11465 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11466 iRegL src1, iRegL src2,
11467 immI src3, immL_M1 src4) %{
11468 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11469 ins_cost(1.9 * INSN_COST);
11470 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11471
11472 ins_encode %{
11473 __ bic(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::ROR,
11477 $src3$$constant & 0x3f);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg_shift);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 // val & (-1 ^ (val << shift)) ==> bicw
11486 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11487 iRegIorL2I src1, iRegIorL2I src2,
11488 immI src3, immI_M1 src4) %{
11489 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11490 ins_cost(1.9 * INSN_COST);
11491 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11492
11493 ins_encode %{
11494 __ bicw(as_Register($dst$$reg),
11495 as_Register($src1$$reg),
11496 as_Register($src2$$reg),
11497 Assembler::LSL,
11498 $src3$$constant & 0x1f);
11499 %}
11500
11501 ins_pipe(ialu_reg_reg_shift);
11502 %}
11503
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 // val & (-1 ^ (val << shift)) ==> bic
11507 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11508 iRegL src1, iRegL src2,
11509 immI src3, immL_M1 src4) %{
11510 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11511 ins_cost(1.9 * INSN_COST);
11512 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11513
11514 ins_encode %{
11515 __ bic(as_Register($dst$$reg),
11516 as_Register($src1$$reg),
11517 as_Register($src2$$reg),
11518 Assembler::LSL,
11519 $src3$$constant & 0x3f);
11520 %}
11521
11522 ins_pipe(ialu_reg_reg_shift);
11523 %}
11524
11525 // This pattern is automatically generated from aarch64_ad.m4
11526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11527 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11528 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11529 iRegIorL2I src1, iRegIorL2I src2,
11530 immI src3, immI_M1 src4) %{
11531 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11532 ins_cost(1.9 * INSN_COST);
11533 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11534
11535 ins_encode %{
11536 __ eonw(as_Register($dst$$reg),
11537 as_Register($src1$$reg),
11538 as_Register($src2$$reg),
11539 Assembler::LSR,
11540 $src3$$constant & 0x1f);
11541 %}
11542
11543 ins_pipe(ialu_reg_reg_shift);
11544 %}
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 // val ^ (-1 ^ (val >>> shift)) ==> eon
11549 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11550 iRegL src1, iRegL src2,
11551 immI src3, immL_M1 src4) %{
11552 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11553 ins_cost(1.9 * INSN_COST);
11554 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11555
11556 ins_encode %{
11557 __ eon(as_Register($dst$$reg),
11558 as_Register($src1$$reg),
11559 as_Register($src2$$reg),
11560 Assembler::LSR,
11561 $src3$$constant & 0x3f);
11562 %}
11563
11564 ins_pipe(ialu_reg_reg_shift);
11565 %}
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 // val ^ (-1 ^ (val >> shift)) ==> eonw
11570 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11571 iRegIorL2I src1, iRegIorL2I src2,
11572 immI src3, immI_M1 src4) %{
11573 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11574 ins_cost(1.9 * INSN_COST);
11575 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11576
11577 ins_encode %{
11578 __ eonw(as_Register($dst$$reg),
11579 as_Register($src1$$reg),
11580 as_Register($src2$$reg),
11581 Assembler::ASR,
11582 $src3$$constant & 0x1f);
11583 %}
11584
11585 ins_pipe(ialu_reg_reg_shift);
11586 %}
11587
11588 // This pattern is automatically generated from aarch64_ad.m4
11589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11590 // val ^ (-1 ^ (val >> shift)) ==> eon
11591 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11592 iRegL src1, iRegL src2,
11593 immI src3, immL_M1 src4) %{
11594 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11595 ins_cost(1.9 * INSN_COST);
11596 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11597
11598 ins_encode %{
11599 __ eon(as_Register($dst$$reg),
11600 as_Register($src1$$reg),
11601 as_Register($src2$$reg),
11602 Assembler::ASR,
11603 $src3$$constant & 0x3f);
11604 %}
11605
11606 ins_pipe(ialu_reg_reg_shift);
11607 %}
11608
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611 // val ^ (-1 ^ (val ror shift)) ==> eonw
11612 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11613 iRegIorL2I src1, iRegIorL2I src2,
11614 immI src3, immI_M1 src4) %{
11615 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11616 ins_cost(1.9 * INSN_COST);
11617 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11618
11619 ins_encode %{
11620 __ eonw(as_Register($dst$$reg),
11621 as_Register($src1$$reg),
11622 as_Register($src2$$reg),
11623 Assembler::ROR,
11624 $src3$$constant & 0x1f);
11625 %}
11626
11627 ins_pipe(ialu_reg_reg_shift);
11628 %}
11629
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 // val ^ (-1 ^ (val ror shift)) ==> eon
11633 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11634 iRegL src1, iRegL src2,
11635 immI src3, immL_M1 src4) %{
11636 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11637 ins_cost(1.9 * INSN_COST);
11638 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11639
11640 ins_encode %{
11641 __ eon(as_Register($dst$$reg),
11642 as_Register($src1$$reg),
11643 as_Register($src2$$reg),
11644 Assembler::ROR,
11645 $src3$$constant & 0x3f);
11646 %}
11647
11648 ins_pipe(ialu_reg_reg_shift);
11649 %}
11650
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653 // val ^ (-1 ^ (val << shift)) ==> eonw
11654 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11655 iRegIorL2I src1, iRegIorL2I src2,
11656 immI src3, immI_M1 src4) %{
11657 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11658 ins_cost(1.9 * INSN_COST);
11659 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11660
11661 ins_encode %{
11662 __ eonw(as_Register($dst$$reg),
11663 as_Register($src1$$reg),
11664 as_Register($src2$$reg),
11665 Assembler::LSL,
11666 $src3$$constant & 0x1f);
11667 %}
11668
11669 ins_pipe(ialu_reg_reg_shift);
11670 %}
11671
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 // val ^ (-1 ^ (val << shift)) ==> eon
11675 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11676 iRegL src1, iRegL src2,
11677 immI src3, immL_M1 src4) %{
11678 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11679 ins_cost(1.9 * INSN_COST);
11680 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11681
11682 ins_encode %{
11683 __ eon(as_Register($dst$$reg),
11684 as_Register($src1$$reg),
11685 as_Register($src2$$reg),
11686 Assembler::LSL,
11687 $src3$$constant & 0x3f);
11688 %}
11689
11690 ins_pipe(ialu_reg_reg_shift);
11691 %}
11692
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695 // val | (-1 ^ (val >>> shift)) ==> ornw
11696 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11697 iRegIorL2I src1, iRegIorL2I src2,
11698 immI src3, immI_M1 src4) %{
11699 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11700 ins_cost(1.9 * INSN_COST);
11701 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11702
11703 ins_encode %{
11704 __ ornw(as_Register($dst$$reg),
11705 as_Register($src1$$reg),
11706 as_Register($src2$$reg),
11707 Assembler::LSR,
11708 $src3$$constant & 0x1f);
11709 %}
11710
11711 ins_pipe(ialu_reg_reg_shift);
11712 %}
11713
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 // val | (-1 ^ (val >>> shift)) ==> orn
11717 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11718 iRegL src1, iRegL src2,
11719 immI src3, immL_M1 src4) %{
11720 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11721 ins_cost(1.9 * INSN_COST);
11722 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11723
11724 ins_encode %{
11725 __ orn(as_Register($dst$$reg),
11726 as_Register($src1$$reg),
11727 as_Register($src2$$reg),
11728 Assembler::LSR,
11729 $src3$$constant & 0x3f);
11730 %}
11731
11732 ins_pipe(ialu_reg_reg_shift);
11733 %}
11734
11735 // This pattern is automatically generated from aarch64_ad.m4
11736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11737 // val | (-1 ^ (val >> shift)) ==> ornw
11738 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11739 iRegIorL2I src1, iRegIorL2I src2,
11740 immI src3, immI_M1 src4) %{
11741 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11742 ins_cost(1.9 * INSN_COST);
11743 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11744
11745 ins_encode %{
11746 __ ornw(as_Register($dst$$reg),
11747 as_Register($src1$$reg),
11748 as_Register($src2$$reg),
11749 Assembler::ASR,
11750 $src3$$constant & 0x1f);
11751 %}
11752
11753 ins_pipe(ialu_reg_reg_shift);
11754 %}
11755
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 // val | (-1 ^ (val >> shift)) ==> orn
11759 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11760 iRegL src1, iRegL src2,
11761 immI src3, immL_M1 src4) %{
11762 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11763 ins_cost(1.9 * INSN_COST);
11764 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11765
11766 ins_encode %{
11767 __ orn(as_Register($dst$$reg),
11768 as_Register($src1$$reg),
11769 as_Register($src2$$reg),
11770 Assembler::ASR,
11771 $src3$$constant & 0x3f);
11772 %}
11773
11774 ins_pipe(ialu_reg_reg_shift);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 // val | (-1 ^ (val ror shift)) ==> ornw
11780 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11781 iRegIorL2I src1, iRegIorL2I src2,
11782 immI src3, immI_M1 src4) %{
11783 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11784 ins_cost(1.9 * INSN_COST);
11785 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11786
11787 ins_encode %{
11788 __ ornw(as_Register($dst$$reg),
11789 as_Register($src1$$reg),
11790 as_Register($src2$$reg),
11791 Assembler::ROR,
11792 $src3$$constant & 0x1f);
11793 %}
11794
11795 ins_pipe(ialu_reg_reg_shift);
11796 %}
11797
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 // val | (-1 ^ (val ror shift)) ==> orn
11801 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11802 iRegL src1, iRegL src2,
11803 immI src3, immL_M1 src4) %{
11804 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11805 ins_cost(1.9 * INSN_COST);
11806 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11807
11808 ins_encode %{
11809 __ orn(as_Register($dst$$reg),
11810 as_Register($src1$$reg),
11811 as_Register($src2$$reg),
11812 Assembler::ROR,
11813 $src3$$constant & 0x3f);
11814 %}
11815
11816 ins_pipe(ialu_reg_reg_shift);
11817 %}
11818
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 // val | (-1 ^ (val << shift)) ==> ornw
11822 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11823 iRegIorL2I src1, iRegIorL2I src2,
11824 immI src3, immI_M1 src4) %{
11825 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11826 ins_cost(1.9 * INSN_COST);
11827 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11828
11829 ins_encode %{
11830 __ ornw(as_Register($dst$$reg),
11831 as_Register($src1$$reg),
11832 as_Register($src2$$reg),
11833 Assembler::LSL,
11834 $src3$$constant & 0x1f);
11835 %}
11836
11837 ins_pipe(ialu_reg_reg_shift);
11838 %}
11839
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 // val | (-1 ^ (val << shift)) ==> orn
11843 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11844 iRegL src1, iRegL src2,
11845 immI src3, immL_M1 src4) %{
11846 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11847 ins_cost(1.9 * INSN_COST);
11848 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11849
11850 ins_encode %{
11851 __ orn(as_Register($dst$$reg),
11852 as_Register($src1$$reg),
11853 as_Register($src2$$reg),
11854 Assembler::LSL,
11855 $src3$$constant & 0x3f);
11856 %}
11857
11858 ins_pipe(ialu_reg_reg_shift);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11864 iRegIorL2I src1, iRegIorL2I src2,
11865 immI src3) %{
11866 match(Set dst (AndI src1 (URShiftI src2 src3)));
11867
11868 ins_cost(1.9 * INSN_COST);
11869 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11870
11871 ins_encode %{
11872 __ andw(as_Register($dst$$reg),
11873 as_Register($src1$$reg),
11874 as_Register($src2$$reg),
11875 Assembler::LSR,
11876 $src3$$constant & 0x1f);
11877 %}
11878
11879 ins_pipe(ialu_reg_reg_shift);
11880 %}
11881
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11885 iRegL src1, iRegL src2,
11886 immI src3) %{
11887 match(Set dst (AndL src1 (URShiftL src2 src3)));
11888
11889 ins_cost(1.9 * INSN_COST);
11890 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11891
11892 ins_encode %{
11893 __ andr(as_Register($dst$$reg),
11894 as_Register($src1$$reg),
11895 as_Register($src2$$reg),
11896 Assembler::LSR,
11897 $src3$$constant & 0x3f);
11898 %}
11899
11900 ins_pipe(ialu_reg_reg_shift);
11901 %}
11902
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11906 iRegIorL2I src1, iRegIorL2I src2,
11907 immI src3) %{
11908 match(Set dst (AndI src1 (RShiftI src2 src3)));
11909
11910 ins_cost(1.9 * INSN_COST);
11911 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11912
11913 ins_encode %{
11914 __ andw(as_Register($dst$$reg),
11915 as_Register($src1$$reg),
11916 as_Register($src2$$reg),
11917 Assembler::ASR,
11918 $src3$$constant & 0x1f);
11919 %}
11920
11921 ins_pipe(ialu_reg_reg_shift);
11922 %}
11923
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11927 iRegL src1, iRegL src2,
11928 immI src3) %{
11929 match(Set dst (AndL src1 (RShiftL src2 src3)));
11930
11931 ins_cost(1.9 * INSN_COST);
11932 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11933
11934 ins_encode %{
11935 __ andr(as_Register($dst$$reg),
11936 as_Register($src1$$reg),
11937 as_Register($src2$$reg),
11938 Assembler::ASR,
11939 $src3$$constant & 0x3f);
11940 %}
11941
11942 ins_pipe(ialu_reg_reg_shift);
11943 %}
11944
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11948 iRegIorL2I src1, iRegIorL2I src2,
11949 immI src3) %{
11950 match(Set dst (AndI src1 (LShiftI src2 src3)));
11951
11952 ins_cost(1.9 * INSN_COST);
11953 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11954
11955 ins_encode %{
11956 __ andw(as_Register($dst$$reg),
11957 as_Register($src1$$reg),
11958 as_Register($src2$$reg),
11959 Assembler::LSL,
11960 $src3$$constant & 0x1f);
11961 %}
11962
11963 ins_pipe(ialu_reg_reg_shift);
11964 %}
11965
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11969 iRegL src1, iRegL src2,
11970 immI src3) %{
11971 match(Set dst (AndL src1 (LShiftL src2 src3)));
11972
11973 ins_cost(1.9 * INSN_COST);
11974 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11975
11976 ins_encode %{
11977 __ andr(as_Register($dst$$reg),
11978 as_Register($src1$$reg),
11979 as_Register($src2$$reg),
11980 Assembler::LSL,
11981 $src3$$constant & 0x3f);
11982 %}
11983
11984 ins_pipe(ialu_reg_reg_shift);
11985 %}
11986
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11990 iRegIorL2I src1, iRegIorL2I src2,
11991 immI src3) %{
11992 match(Set dst (AndI src1 (RotateRight src2 src3)));
11993
11994 ins_cost(1.9 * INSN_COST);
11995 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11996
11997 ins_encode %{
11998 __ andw(as_Register($dst$$reg),
11999 as_Register($src1$$reg),
12000 as_Register($src2$$reg),
12001 Assembler::ROR,
12002 $src3$$constant & 0x1f);
12003 %}
12004
12005 ins_pipe(ialu_reg_reg_shift);
12006 %}
12007
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12011 iRegL src1, iRegL src2,
12012 immI src3) %{
12013 match(Set dst (AndL src1 (RotateRight src2 src3)));
12014
12015 ins_cost(1.9 * INSN_COST);
12016 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12017
12018 ins_encode %{
12019 __ andr(as_Register($dst$$reg),
12020 as_Register($src1$$reg),
12021 as_Register($src2$$reg),
12022 Assembler::ROR,
12023 $src3$$constant & 0x3f);
12024 %}
12025
12026 ins_pipe(ialu_reg_reg_shift);
12027 %}
12028
12029 // This pattern is automatically generated from aarch64_ad.m4
12030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12031 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12032 iRegIorL2I src1, iRegIorL2I src2,
12033 immI src3) %{
12034 match(Set dst (XorI src1 (URShiftI src2 src3)));
12035
12036 ins_cost(1.9 * INSN_COST);
12037 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12038
12039 ins_encode %{
12040 __ eorw(as_Register($dst$$reg),
12041 as_Register($src1$$reg),
12042 as_Register($src2$$reg),
12043 Assembler::LSR,
12044 $src3$$constant & 0x1f);
12045 %}
12046
12047 ins_pipe(ialu_reg_reg_shift);
12048 %}
12049
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12053 iRegL src1, iRegL src2,
12054 immI src3) %{
12055 match(Set dst (XorL src1 (URShiftL src2 src3)));
12056
12057 ins_cost(1.9 * INSN_COST);
12058 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12059
12060 ins_encode %{
12061 __ eor(as_Register($dst$$reg),
12062 as_Register($src1$$reg),
12063 as_Register($src2$$reg),
12064 Assembler::LSR,
12065 $src3$$constant & 0x3f);
12066 %}
12067
12068 ins_pipe(ialu_reg_reg_shift);
12069 %}
12070
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12074 iRegIorL2I src1, iRegIorL2I src2,
12075 immI src3) %{
12076 match(Set dst (XorI src1 (RShiftI src2 src3)));
12077
12078 ins_cost(1.9 * INSN_COST);
12079 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12080
12081 ins_encode %{
12082 __ eorw(as_Register($dst$$reg),
12083 as_Register($src1$$reg),
12084 as_Register($src2$$reg),
12085 Assembler::ASR,
12086 $src3$$constant & 0x1f);
12087 %}
12088
12089 ins_pipe(ialu_reg_reg_shift);
12090 %}
12091
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12095 iRegL src1, iRegL src2,
12096 immI src3) %{
12097 match(Set dst (XorL src1 (RShiftL src2 src3)));
12098
12099 ins_cost(1.9 * INSN_COST);
12100 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12101
12102 ins_encode %{
12103 __ eor(as_Register($dst$$reg),
12104 as_Register($src1$$reg),
12105 as_Register($src2$$reg),
12106 Assembler::ASR,
12107 $src3$$constant & 0x3f);
12108 %}
12109
12110 ins_pipe(ialu_reg_reg_shift);
12111 %}
12112
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12116 iRegIorL2I src1, iRegIorL2I src2,
12117 immI src3) %{
12118 match(Set dst (XorI src1 (LShiftI src2 src3)));
12119
12120 ins_cost(1.9 * INSN_COST);
12121 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12122
12123 ins_encode %{
12124 __ eorw(as_Register($dst$$reg),
12125 as_Register($src1$$reg),
12126 as_Register($src2$$reg),
12127 Assembler::LSL,
12128 $src3$$constant & 0x1f);
12129 %}
12130
12131 ins_pipe(ialu_reg_reg_shift);
12132 %}
12133
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12137 iRegL src1, iRegL src2,
12138 immI src3) %{
12139 match(Set dst (XorL src1 (LShiftL src2 src3)));
12140
12141 ins_cost(1.9 * INSN_COST);
12142 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12143
12144 ins_encode %{
12145 __ eor(as_Register($dst$$reg),
12146 as_Register($src1$$reg),
12147 as_Register($src2$$reg),
12148 Assembler::LSL,
12149 $src3$$constant & 0x3f);
12150 %}
12151
12152 ins_pipe(ialu_reg_reg_shift);
12153 %}
12154
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12158 iRegIorL2I src1, iRegIorL2I src2,
12159 immI src3) %{
12160 match(Set dst (XorI src1 (RotateRight src2 src3)));
12161
12162 ins_cost(1.9 * INSN_COST);
12163 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12164
12165 ins_encode %{
12166 __ eorw(as_Register($dst$$reg),
12167 as_Register($src1$$reg),
12168 as_Register($src2$$reg),
12169 Assembler::ROR,
12170 $src3$$constant & 0x1f);
12171 %}
12172
12173 ins_pipe(ialu_reg_reg_shift);
12174 %}
12175
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12179 iRegL src1, iRegL src2,
12180 immI src3) %{
12181 match(Set dst (XorL src1 (RotateRight src2 src3)));
12182
12183 ins_cost(1.9 * INSN_COST);
12184 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12185
12186 ins_encode %{
12187 __ eor(as_Register($dst$$reg),
12188 as_Register($src1$$reg),
12189 as_Register($src2$$reg),
12190 Assembler::ROR,
12191 $src3$$constant & 0x3f);
12192 %}
12193
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12200 iRegIorL2I src1, iRegIorL2I src2,
12201 immI src3) %{
12202 match(Set dst (OrI src1 (URShiftI src2 src3)));
12203
12204 ins_cost(1.9 * INSN_COST);
12205 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12206
12207 ins_encode %{
12208 __ orrw(as_Register($dst$$reg),
12209 as_Register($src1$$reg),
12210 as_Register($src2$$reg),
12211 Assembler::LSR,
12212 $src3$$constant & 0x1f);
12213 %}
12214
12215 ins_pipe(ialu_reg_reg_shift);
12216 %}
12217
12218 // This pattern is automatically generated from aarch64_ad.m4
12219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12220 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12221 iRegL src1, iRegL src2,
12222 immI src3) %{
12223 match(Set dst (OrL src1 (URShiftL src2 src3)));
12224
12225 ins_cost(1.9 * INSN_COST);
12226 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12227
12228 ins_encode %{
12229 __ orr(as_Register($dst$$reg),
12230 as_Register($src1$$reg),
12231 as_Register($src2$$reg),
12232 Assembler::LSR,
12233 $src3$$constant & 0x3f);
12234 %}
12235
12236 ins_pipe(ialu_reg_reg_shift);
12237 %}
12238
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12242 iRegIorL2I src1, iRegIorL2I src2,
12243 immI src3) %{
12244 match(Set dst (OrI src1 (RShiftI src2 src3)));
12245
12246 ins_cost(1.9 * INSN_COST);
12247 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12248
12249 ins_encode %{
12250 __ orrw(as_Register($dst$$reg),
12251 as_Register($src1$$reg),
12252 as_Register($src2$$reg),
12253 Assembler::ASR,
12254 $src3$$constant & 0x1f);
12255 %}
12256
12257 ins_pipe(ialu_reg_reg_shift);
12258 %}
12259
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12263 iRegL src1, iRegL src2,
12264 immI src3) %{
12265 match(Set dst (OrL src1 (RShiftL src2 src3)));
12266
12267 ins_cost(1.9 * INSN_COST);
12268 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12269
12270 ins_encode %{
12271 __ orr(as_Register($dst$$reg),
12272 as_Register($src1$$reg),
12273 as_Register($src2$$reg),
12274 Assembler::ASR,
12275 $src3$$constant & 0x3f);
12276 %}
12277
12278 ins_pipe(ialu_reg_reg_shift);
12279 %}
12280
12281 // This pattern is automatically generated from aarch64_ad.m4
12282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12283 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12284 iRegIorL2I src1, iRegIorL2I src2,
12285 immI src3) %{
12286 match(Set dst (OrI src1 (LShiftI src2 src3)));
12287
12288 ins_cost(1.9 * INSN_COST);
12289 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12290
12291 ins_encode %{
12292 __ orrw(as_Register($dst$$reg),
12293 as_Register($src1$$reg),
12294 as_Register($src2$$reg),
12295 Assembler::LSL,
12296 $src3$$constant & 0x1f);
12297 %}
12298
12299 ins_pipe(ialu_reg_reg_shift);
12300 %}
12301
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12305 iRegL src1, iRegL src2,
12306 immI src3) %{
12307 match(Set dst (OrL src1 (LShiftL src2 src3)));
12308
12309 ins_cost(1.9 * INSN_COST);
12310 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12311
12312 ins_encode %{
12313 __ orr(as_Register($dst$$reg),
12314 as_Register($src1$$reg),
12315 as_Register($src2$$reg),
12316 Assembler::LSL,
12317 $src3$$constant & 0x3f);
12318 %}
12319
12320 ins_pipe(ialu_reg_reg_shift);
12321 %}
12322
12323 // This pattern is automatically generated from aarch64_ad.m4
12324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12325 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12326 iRegIorL2I src1, iRegIorL2I src2,
12327 immI src3) %{
12328 match(Set dst (OrI src1 (RotateRight src2 src3)));
12329
12330 ins_cost(1.9 * INSN_COST);
12331 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12332
12333 ins_encode %{
12334 __ orrw(as_Register($dst$$reg),
12335 as_Register($src1$$reg),
12336 as_Register($src2$$reg),
12337 Assembler::ROR,
12338 $src3$$constant & 0x1f);
12339 %}
12340
12341 ins_pipe(ialu_reg_reg_shift);
12342 %}
12343
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12347 iRegL src1, iRegL src2,
12348 immI src3) %{
12349 match(Set dst (OrL src1 (RotateRight src2 src3)));
12350
12351 ins_cost(1.9 * INSN_COST);
12352 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12353
12354 ins_encode %{
12355 __ orr(as_Register($dst$$reg),
12356 as_Register($src1$$reg),
12357 as_Register($src2$$reg),
12358 Assembler::ROR,
12359 $src3$$constant & 0x3f);
12360 %}
12361
12362 ins_pipe(ialu_reg_reg_shift);
12363 %}
12364
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12368 iRegIorL2I src1, iRegIorL2I src2,
12369 immI src3) %{
12370 match(Set dst (AddI src1 (URShiftI src2 src3)));
12371
12372 ins_cost(1.9 * INSN_COST);
12373 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12374
12375 ins_encode %{
12376 __ addw(as_Register($dst$$reg),
12377 as_Register($src1$$reg),
12378 as_Register($src2$$reg),
12379 Assembler::LSR,
12380 $src3$$constant & 0x1f);
12381 %}
12382
12383 ins_pipe(ialu_reg_reg_shift);
12384 %}
12385
12386 // This pattern is automatically generated from aarch64_ad.m4
12387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12388 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12389 iRegL src1, iRegL src2,
12390 immI src3) %{
12391 match(Set dst (AddL src1 (URShiftL src2 src3)));
12392
12393 ins_cost(1.9 * INSN_COST);
12394 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12395
12396 ins_encode %{
12397 __ add(as_Register($dst$$reg),
12398 as_Register($src1$$reg),
12399 as_Register($src2$$reg),
12400 Assembler::LSR,
12401 $src3$$constant & 0x3f);
12402 %}
12403
12404 ins_pipe(ialu_reg_reg_shift);
12405 %}
12406
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12410 iRegIorL2I src1, iRegIorL2I src2,
12411 immI src3) %{
12412 match(Set dst (AddI src1 (RShiftI src2 src3)));
12413
12414 ins_cost(1.9 * INSN_COST);
12415 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12416
12417 ins_encode %{
12418 __ addw(as_Register($dst$$reg),
12419 as_Register($src1$$reg),
12420 as_Register($src2$$reg),
12421 Assembler::ASR,
12422 $src3$$constant & 0x1f);
12423 %}
12424
12425 ins_pipe(ialu_reg_reg_shift);
12426 %}
12427
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12431 iRegL src1, iRegL src2,
12432 immI src3) %{
12433 match(Set dst (AddL src1 (RShiftL src2 src3)));
12434
12435 ins_cost(1.9 * INSN_COST);
12436 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12437
12438 ins_encode %{
12439 __ add(as_Register($dst$$reg),
12440 as_Register($src1$$reg),
12441 as_Register($src2$$reg),
12442 Assembler::ASR,
12443 $src3$$constant & 0x3f);
12444 %}
12445
12446 ins_pipe(ialu_reg_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12452 iRegIorL2I src1, iRegIorL2I src2,
12453 immI src3) %{
12454 match(Set dst (AddI src1 (LShiftI src2 src3)));
12455
12456 ins_cost(1.9 * INSN_COST);
12457 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12458
12459 ins_encode %{
12460 __ addw(as_Register($dst$$reg),
12461 as_Register($src1$$reg),
12462 as_Register($src2$$reg),
12463 Assembler::LSL,
12464 $src3$$constant & 0x1f);
12465 %}
12466
12467 ins_pipe(ialu_reg_reg_shift);
12468 %}
12469
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12473 iRegL src1, iRegL src2,
12474 immI src3) %{
12475 match(Set dst (AddL src1 (LShiftL src2 src3)));
12476
12477 ins_cost(1.9 * INSN_COST);
12478 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12479
12480 ins_encode %{
12481 __ add(as_Register($dst$$reg),
12482 as_Register($src1$$reg),
12483 as_Register($src2$$reg),
12484 Assembler::LSL,
12485 $src3$$constant & 0x3f);
12486 %}
12487
12488 ins_pipe(ialu_reg_reg_shift);
12489 %}
12490
12491 // This pattern is automatically generated from aarch64_ad.m4
12492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12493 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12494 iRegIorL2I src1, iRegIorL2I src2,
12495 immI src3) %{
12496 match(Set dst (SubI src1 (URShiftI src2 src3)));
12497
12498 ins_cost(1.9 * INSN_COST);
12499 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12500
12501 ins_encode %{
12502 __ subw(as_Register($dst$$reg),
12503 as_Register($src1$$reg),
12504 as_Register($src2$$reg),
12505 Assembler::LSR,
12506 $src3$$constant & 0x1f);
12507 %}
12508
12509 ins_pipe(ialu_reg_reg_shift);
12510 %}
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12515 iRegL src1, iRegL src2,
12516 immI src3) %{
12517 match(Set dst (SubL src1 (URShiftL src2 src3)));
12518
12519 ins_cost(1.9 * INSN_COST);
12520 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12521
12522 ins_encode %{
12523 __ sub(as_Register($dst$$reg),
12524 as_Register($src1$$reg),
12525 as_Register($src2$$reg),
12526 Assembler::LSR,
12527 $src3$$constant & 0x3f);
12528 %}
12529
12530 ins_pipe(ialu_reg_reg_shift);
12531 %}
12532
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12536 iRegIorL2I src1, iRegIorL2I src2,
12537 immI src3) %{
12538 match(Set dst (SubI src1 (RShiftI src2 src3)));
12539
12540 ins_cost(1.9 * INSN_COST);
12541 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12542
12543 ins_encode %{
12544 __ subw(as_Register($dst$$reg),
12545 as_Register($src1$$reg),
12546 as_Register($src2$$reg),
12547 Assembler::ASR,
12548 $src3$$constant & 0x1f);
12549 %}
12550
12551 ins_pipe(ialu_reg_reg_shift);
12552 %}
12553
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12557 iRegL src1, iRegL src2,
12558 immI src3) %{
12559 match(Set dst (SubL src1 (RShiftL src2 src3)));
12560
12561 ins_cost(1.9 * INSN_COST);
12562 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12563
12564 ins_encode %{
12565 __ sub(as_Register($dst$$reg),
12566 as_Register($src1$$reg),
12567 as_Register($src2$$reg),
12568 Assembler::ASR,
12569 $src3$$constant & 0x3f);
12570 %}
12571
12572 ins_pipe(ialu_reg_reg_shift);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12578 iRegIorL2I src1, iRegIorL2I src2,
12579 immI src3) %{
12580 match(Set dst (SubI src1 (LShiftI src2 src3)));
12581
12582 ins_cost(1.9 * INSN_COST);
12583 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12584
12585 ins_encode %{
12586 __ subw(as_Register($dst$$reg),
12587 as_Register($src1$$reg),
12588 as_Register($src2$$reg),
12589 Assembler::LSL,
12590 $src3$$constant & 0x1f);
12591 %}
12592
12593 ins_pipe(ialu_reg_reg_shift);
12594 %}
12595
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12599 iRegL src1, iRegL src2,
12600 immI src3) %{
12601 match(Set dst (SubL src1 (LShiftL src2 src3)));
12602
12603 ins_cost(1.9 * INSN_COST);
12604 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12605
12606 ins_encode %{
12607 __ sub(as_Register($dst$$reg),
12608 as_Register($src1$$reg),
12609 as_Register($src2$$reg),
12610 Assembler::LSL,
12611 $src3$$constant & 0x3f);
12612 %}
12613
12614 ins_pipe(ialu_reg_reg_shift);
12615 %}
12616
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619
12620 // Shift Left followed by Shift Right.
12621 // This idiom is used by the compiler for the i2b bytecode etc.
12622 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12623 %{
12624 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12625 ins_cost(INSN_COST * 2);
12626 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12627 ins_encode %{
12628 int lshift = $lshift_count$$constant & 63;
12629 int rshift = $rshift_count$$constant & 63;
12630 int s = 63 - lshift;
12631 int r = (rshift - lshift) & 63;
12632 __ sbfm(as_Register($dst$$reg),
12633 as_Register($src$$reg),
12634 r, s);
12635 %}
12636
12637 ins_pipe(ialu_reg_shift);
12638 %}
12639
12640 // This pattern is automatically generated from aarch64_ad.m4
12641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12642
12643 // Shift Left followed by Shift Right.
12644 // This idiom is used by the compiler for the i2b bytecode etc.
12645 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12646 %{
12647 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12648 ins_cost(INSN_COST * 2);
12649 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12650 ins_encode %{
12651 int lshift = $lshift_count$$constant & 31;
12652 int rshift = $rshift_count$$constant & 31;
12653 int s = 31 - lshift;
12654 int r = (rshift - lshift) & 31;
12655 __ sbfmw(as_Register($dst$$reg),
12656 as_Register($src$$reg),
12657 r, s);
12658 %}
12659
12660 ins_pipe(ialu_reg_shift);
12661 %}
12662
12663 // This pattern is automatically generated from aarch64_ad.m4
12664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12665
12666 // Shift Left followed by Shift Right.
12667 // This idiom is used by the compiler for the i2b bytecode etc.
12668 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12669 %{
12670 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12671 ins_cost(INSN_COST * 2);
12672 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12673 ins_encode %{
12674 int lshift = $lshift_count$$constant & 63;
12675 int rshift = $rshift_count$$constant & 63;
12676 int s = 63 - lshift;
12677 int r = (rshift - lshift) & 63;
12678 __ ubfm(as_Register($dst$$reg),
12679 as_Register($src$$reg),
12680 r, s);
12681 %}
12682
12683 ins_pipe(ialu_reg_shift);
12684 %}
12685
12686 // This pattern is automatically generated from aarch64_ad.m4
12687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12688
12689 // Shift Left followed by Shift Right.
12690 // This idiom is used by the compiler for the i2b bytecode etc.
12691 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12692 %{
12693 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12694 ins_cost(INSN_COST * 2);
12695 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12696 ins_encode %{
12697 int lshift = $lshift_count$$constant & 31;
12698 int rshift = $rshift_count$$constant & 31;
12699 int s = 31 - lshift;
12700 int r = (rshift - lshift) & 31;
12701 __ ubfmw(as_Register($dst$$reg),
12702 as_Register($src$$reg),
12703 r, s);
12704 %}
12705
12706 ins_pipe(ialu_reg_shift);
12707 %}
12708
12709 // Bitfield extract with shift & mask
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12714 %{
12715 match(Set dst (AndI (URShiftI src rshift) mask));
12716 // Make sure we are not going to exceed what ubfxw can do.
12717 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12718
12719 ins_cost(INSN_COST);
12720 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12721 ins_encode %{
12722 int rshift = $rshift$$constant & 31;
12723 intptr_t mask = $mask$$constant;
12724 int width = exact_log2(mask+1);
12725 __ ubfxw(as_Register($dst$$reg),
12726 as_Register($src$$reg), rshift, width);
12727 %}
12728 ins_pipe(ialu_reg_shift);
12729 %}
12730
12731 // This pattern is automatically generated from aarch64_ad.m4
12732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12733 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12734 %{
12735 match(Set dst (AndL (URShiftL src rshift) mask));
12736 // Make sure we are not going to exceed what ubfx can do.
12737 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12738
12739 ins_cost(INSN_COST);
12740 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12741 ins_encode %{
12742 int rshift = $rshift$$constant & 63;
12743 intptr_t mask = $mask$$constant;
12744 int width = exact_log2_long(mask+1);
12745 __ ubfx(as_Register($dst$$reg),
12746 as_Register($src$$reg), rshift, width);
12747 %}
12748 ins_pipe(ialu_reg_shift);
12749 %}
12750
12751
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754
12755 // We can use ubfx when extending an And with a mask when we know mask
12756 // is positive. We know that because immI_bitmask guarantees it.
12757 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12758 %{
12759 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12760 // Make sure we are not going to exceed what ubfxw can do.
12761 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12762
12763 ins_cost(INSN_COST * 2);
12764 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12765 ins_encode %{
12766 int rshift = $rshift$$constant & 31;
12767 intptr_t mask = $mask$$constant;
12768 int width = exact_log2(mask+1);
12769 __ ubfx(as_Register($dst$$reg),
12770 as_Register($src$$reg), rshift, width);
12771 %}
12772 ins_pipe(ialu_reg_shift);
12773 %}
12774
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778
12779 // We can use ubfiz when masking by a positive number and then left shifting the result.
12780 // We know that the mask is positive because immI_bitmask guarantees it.
12781 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12782 %{
12783 match(Set dst (LShiftI (AndI src mask) lshift));
12784 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12785
12786 ins_cost(INSN_COST);
12787 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12788 ins_encode %{
12789 int lshift = $lshift$$constant & 31;
12790 intptr_t mask = $mask$$constant;
12791 int width = exact_log2(mask+1);
12792 __ ubfizw(as_Register($dst$$reg),
12793 as_Register($src$$reg), lshift, width);
12794 %}
12795 ins_pipe(ialu_reg_shift);
12796 %}
12797
12798 // This pattern is automatically generated from aarch64_ad.m4
12799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12800
12801 // We can use ubfiz when masking by a positive number and then left shifting the result.
12802 // We know that the mask is positive because immL_bitmask guarantees it.
12803 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12804 %{
12805 match(Set dst (LShiftL (AndL src mask) lshift));
12806 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12807
12808 ins_cost(INSN_COST);
12809 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12810 ins_encode %{
12811 int lshift = $lshift$$constant & 63;
12812 intptr_t mask = $mask$$constant;
12813 int width = exact_log2_long(mask+1);
12814 __ ubfiz(as_Register($dst$$reg),
12815 as_Register($src$$reg), lshift, width);
12816 %}
12817 ins_pipe(ialu_reg_shift);
12818 %}
12819
12820 // This pattern is automatically generated from aarch64_ad.m4
12821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12822
12823 // We can use ubfiz when masking by a positive number and then left shifting the result.
12824 // We know that the mask is positive because immI_bitmask guarantees it.
12825 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12826 %{
12827 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12828 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12829
12830 ins_cost(INSN_COST);
12831 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12832 ins_encode %{
12833 int lshift = $lshift$$constant & 31;
12834 intptr_t mask = $mask$$constant;
12835 int width = exact_log2(mask+1);
12836 __ ubfizw(as_Register($dst$$reg),
12837 as_Register($src$$reg), lshift, width);
12838 %}
12839 ins_pipe(ialu_reg_shift);
12840 %}
12841
12842 // This pattern is automatically generated from aarch64_ad.m4
12843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12844
12845 // We can use ubfiz when masking by a positive number and then left shifting the result.
12846 // We know that the mask is positive because immL_bitmask guarantees it.
12847 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12848 %{
12849 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12850 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12851
12852 ins_cost(INSN_COST);
12853 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12854 ins_encode %{
12855 int lshift = $lshift$$constant & 63;
12856 intptr_t mask = $mask$$constant;
12857 int width = exact_log2_long(mask+1);
12858 __ ubfiz(as_Register($dst$$reg),
12859 as_Register($src$$reg), lshift, width);
12860 %}
12861 ins_pipe(ialu_reg_shift);
12862 %}
12863
12864
12865 // This pattern is automatically generated from aarch64_ad.m4
12866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12867
12868 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12869 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12870 %{
12871 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12872 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12873
12874 ins_cost(INSN_COST);
12875 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12876 ins_encode %{
12877 int lshift = $lshift$$constant & 63;
12878 intptr_t mask = $mask$$constant;
12879 int width = exact_log2(mask+1);
12880 __ ubfiz(as_Register($dst$$reg),
12881 as_Register($src$$reg), lshift, width);
12882 %}
12883 ins_pipe(ialu_reg_shift);
12884 %}
12885
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888
12889 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12890 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12891 %{
12892 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12893 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12894
12895 ins_cost(INSN_COST);
12896 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12897 ins_encode %{
12898 int lshift = $lshift$$constant & 31;
12899 intptr_t mask = $mask$$constant;
12900 int width = exact_log2(mask+1);
12901 __ ubfiz(as_Register($dst$$reg),
12902 as_Register($src$$reg), lshift, width);
12903 %}
12904 ins_pipe(ialu_reg_shift);
12905 %}
12906
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909
12910 // Can skip int2long conversions after AND with small bitmask
12911 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12912 %{
12913 match(Set dst (ConvI2L (AndI src msk)));
12914 ins_cost(INSN_COST);
12915 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12916 ins_encode %{
12917 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12918 %}
12919 ins_pipe(ialu_reg_shift);
12920 %}
12921
12922
12923 // Rotations
12924
12925 // This pattern is automatically generated from aarch64_ad.m4
12926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12927 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12928 %{
12929 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12930 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12931
12932 ins_cost(INSN_COST);
12933 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12934
12935 ins_encode %{
12936 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12937 $rshift$$constant & 63);
12938 %}
12939 ins_pipe(ialu_reg_reg_extr);
12940 %}
12941
12942
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12946 %{
12947 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12948 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12949
12950 ins_cost(INSN_COST);
12951 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12952
12953 ins_encode %{
12954 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12955 $rshift$$constant & 31);
12956 %}
12957 ins_pipe(ialu_reg_reg_extr);
12958 %}
12959
12960
12961 // This pattern is automatically generated from aarch64_ad.m4
12962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12963 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12964 %{
12965 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12966 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12967
12968 ins_cost(INSN_COST);
12969 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12970
12971 ins_encode %{
12972 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12973 $rshift$$constant & 63);
12974 %}
12975 ins_pipe(ialu_reg_reg_extr);
12976 %}
12977
12978
12979 // This pattern is automatically generated from aarch64_ad.m4
12980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12981 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12982 %{
12983 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12984 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12985
12986 ins_cost(INSN_COST);
12987 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12988
12989 ins_encode %{
12990 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12991 $rshift$$constant & 31);
12992 %}
12993 ins_pipe(ialu_reg_reg_extr);
12994 %}
12995
12996 // This pattern is automatically generated from aarch64_ad.m4
12997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12998 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12999 %{
13000 match(Set dst (RotateRight src shift));
13001
13002 ins_cost(INSN_COST);
13003 format %{ "ror $dst, $src, $shift" %}
13004
13005 ins_encode %{
13006 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13007 $shift$$constant & 0x1f);
13008 %}
13009 ins_pipe(ialu_reg_reg_vshift);
13010 %}
13011
13012 // This pattern is automatically generated from aarch64_ad.m4
13013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13014 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13015 %{
13016 match(Set dst (RotateRight src shift));
13017
13018 ins_cost(INSN_COST);
13019 format %{ "ror $dst, $src, $shift" %}
13020
13021 ins_encode %{
13022 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13023 $shift$$constant & 0x3f);
13024 %}
13025 ins_pipe(ialu_reg_reg_vshift);
13026 %}
13027
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13031 %{
13032 match(Set dst (RotateRight src shift));
13033
13034 ins_cost(INSN_COST);
13035 format %{ "ror $dst, $src, $shift" %}
13036
13037 ins_encode %{
13038 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13039 %}
13040 ins_pipe(ialu_reg_reg_vshift);
13041 %}
13042
13043 // This pattern is automatically generated from aarch64_ad.m4
13044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13045 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13046 %{
13047 match(Set dst (RotateRight src shift));
13048
13049 ins_cost(INSN_COST);
13050 format %{ "ror $dst, $src, $shift" %}
13051
13052 ins_encode %{
13053 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13054 %}
13055 ins_pipe(ialu_reg_reg_vshift);
13056 %}
13057
13058 // This pattern is automatically generated from aarch64_ad.m4
13059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13060 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13061 %{
13062 match(Set dst (RotateLeft src shift));
13063
13064 ins_cost(INSN_COST);
13065 format %{ "rol $dst, $src, $shift" %}
13066
13067 ins_encode %{
13068 __ subw(rscratch1, zr, as_Register($shift$$reg));
13069 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13070 %}
13071 ins_pipe(ialu_reg_reg_vshift);
13072 %}
13073
13074 // This pattern is automatically generated from aarch64_ad.m4
13075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13076 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13077 %{
13078 match(Set dst (RotateLeft src shift));
13079
13080 ins_cost(INSN_COST);
13081 format %{ "rol $dst, $src, $shift" %}
13082
13083 ins_encode %{
13084 __ subw(rscratch1, zr, as_Register($shift$$reg));
13085 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13086 %}
13087 ins_pipe(ialu_reg_reg_vshift);
13088 %}
13089
13090
13091 // Add/subtract (extended)
13092
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13096 %{
13097 match(Set dst (AddL src1 (ConvI2L src2)));
13098 ins_cost(INSN_COST);
13099 format %{ "add $dst, $src1, $src2, sxtw" %}
13100
13101 ins_encode %{
13102 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13103 as_Register($src2$$reg), ext::sxtw);
13104 %}
13105 ins_pipe(ialu_reg_reg);
13106 %}
13107
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13111 %{
13112 match(Set dst (SubL src1 (ConvI2L src2)));
13113 ins_cost(INSN_COST);
13114 format %{ "sub $dst, $src1, $src2, sxtw" %}
13115
13116 ins_encode %{
13117 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13118 as_Register($src2$$reg), ext::sxtw);
13119 %}
13120 ins_pipe(ialu_reg_reg);
13121 %}
13122
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13126 %{
13127 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13128 ins_cost(INSN_COST);
13129 format %{ "add $dst, $src1, $src2, sxth" %}
13130
13131 ins_encode %{
13132 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13133 as_Register($src2$$reg), ext::sxth);
13134 %}
13135 ins_pipe(ialu_reg_reg);
13136 %}
13137
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13141 %{
13142 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13143 ins_cost(INSN_COST);
13144 format %{ "add $dst, $src1, $src2, sxtb" %}
13145
13146 ins_encode %{
13147 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13148 as_Register($src2$$reg), ext::sxtb);
13149 %}
13150 ins_pipe(ialu_reg_reg);
13151 %}
13152
13153 // This pattern is automatically generated from aarch64_ad.m4
13154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13155 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13156 %{
13157 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13158 ins_cost(INSN_COST);
13159 format %{ "add $dst, $src1, $src2, uxtb" %}
13160
13161 ins_encode %{
13162 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13163 as_Register($src2$$reg), ext::uxtb);
13164 %}
13165 ins_pipe(ialu_reg_reg);
13166 %}
13167
13168 // This pattern is automatically generated from aarch64_ad.m4
13169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13170 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13171 %{
13172 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13173 ins_cost(INSN_COST);
13174 format %{ "add $dst, $src1, $src2, sxth" %}
13175
13176 ins_encode %{
13177 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13178 as_Register($src2$$reg), ext::sxth);
13179 %}
13180 ins_pipe(ialu_reg_reg);
13181 %}
13182
13183 // This pattern is automatically generated from aarch64_ad.m4
13184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13185 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13186 %{
13187 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13188 ins_cost(INSN_COST);
13189 format %{ "add $dst, $src1, $src2, sxtw" %}
13190
13191 ins_encode %{
13192 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13193 as_Register($src2$$reg), ext::sxtw);
13194 %}
13195 ins_pipe(ialu_reg_reg);
13196 %}
13197
13198 // This pattern is automatically generated from aarch64_ad.m4
13199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13200 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13201 %{
13202 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13203 ins_cost(INSN_COST);
13204 format %{ "add $dst, $src1, $src2, sxtb" %}
13205
13206 ins_encode %{
13207 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13208 as_Register($src2$$reg), ext::sxtb);
13209 %}
13210 ins_pipe(ialu_reg_reg);
13211 %}
13212
13213 // This pattern is automatically generated from aarch64_ad.m4
13214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13215 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13216 %{
13217 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13218 ins_cost(INSN_COST);
13219 format %{ "add $dst, $src1, $src2, uxtb" %}
13220
13221 ins_encode %{
13222 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13223 as_Register($src2$$reg), ext::uxtb);
13224 %}
13225 ins_pipe(ialu_reg_reg);
13226 %}
13227
13228 // This pattern is automatically generated from aarch64_ad.m4
13229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13230 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13231 %{
13232 match(Set dst (AddI src1 (AndI src2 mask)));
13233 ins_cost(INSN_COST);
13234 format %{ "addw $dst, $src1, $src2, uxtb" %}
13235
13236 ins_encode %{
13237 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13238 as_Register($src2$$reg), ext::uxtb);
13239 %}
13240 ins_pipe(ialu_reg_reg);
13241 %}
13242
13243 // This pattern is automatically generated from aarch64_ad.m4
13244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13245 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13246 %{
13247 match(Set dst (AddI src1 (AndI src2 mask)));
13248 ins_cost(INSN_COST);
13249 format %{ "addw $dst, $src1, $src2, uxth" %}
13250
13251 ins_encode %{
13252 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13253 as_Register($src2$$reg), ext::uxth);
13254 %}
13255 ins_pipe(ialu_reg_reg);
13256 %}
13257
13258 // This pattern is automatically generated from aarch64_ad.m4
13259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13260 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13261 %{
13262 match(Set dst (AddL src1 (AndL src2 mask)));
13263 ins_cost(INSN_COST);
13264 format %{ "add $dst, $src1, $src2, uxtb" %}
13265
13266 ins_encode %{
13267 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13268 as_Register($src2$$reg), ext::uxtb);
13269 %}
13270 ins_pipe(ialu_reg_reg);
13271 %}
13272
13273 // This pattern is automatically generated from aarch64_ad.m4
13274 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13275 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13276 %{
13277 match(Set dst (AddL src1 (AndL src2 mask)));
13278 ins_cost(INSN_COST);
13279 format %{ "add $dst, $src1, $src2, uxth" %}
13280
13281 ins_encode %{
13282 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13283 as_Register($src2$$reg), ext::uxth);
13284 %}
13285 ins_pipe(ialu_reg_reg);
13286 %}
13287
13288 // This pattern is automatically generated from aarch64_ad.m4
13289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13290 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13291 %{
13292 match(Set dst (AddL src1 (AndL src2 mask)));
13293 ins_cost(INSN_COST);
13294 format %{ "add $dst, $src1, $src2, uxtw" %}
13295
13296 ins_encode %{
13297 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13298 as_Register($src2$$reg), ext::uxtw);
13299 %}
13300 ins_pipe(ialu_reg_reg);
13301 %}
13302
13303 // This pattern is automatically generated from aarch64_ad.m4
13304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13305 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13306 %{
13307 match(Set dst (SubI src1 (AndI src2 mask)));
13308 ins_cost(INSN_COST);
13309 format %{ "subw $dst, $src1, $src2, uxtb" %}
13310
13311 ins_encode %{
13312 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13313 as_Register($src2$$reg), ext::uxtb);
13314 %}
13315 ins_pipe(ialu_reg_reg);
13316 %}
13317
13318 // This pattern is automatically generated from aarch64_ad.m4
13319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13320 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13321 %{
13322 match(Set dst (SubI src1 (AndI src2 mask)));
13323 ins_cost(INSN_COST);
13324 format %{ "subw $dst, $src1, $src2, uxth" %}
13325
13326 ins_encode %{
13327 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13328 as_Register($src2$$reg), ext::uxth);
13329 %}
13330 ins_pipe(ialu_reg_reg);
13331 %}
13332
13333 // This pattern is automatically generated from aarch64_ad.m4
13334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13335 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13336 %{
13337 match(Set dst (SubL src1 (AndL src2 mask)));
13338 ins_cost(INSN_COST);
13339 format %{ "sub $dst, $src1, $src2, uxtb" %}
13340
13341 ins_encode %{
13342 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13343 as_Register($src2$$reg), ext::uxtb);
13344 %}
13345 ins_pipe(ialu_reg_reg);
13346 %}
13347
13348 // This pattern is automatically generated from aarch64_ad.m4
13349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13350 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13351 %{
13352 match(Set dst (SubL src1 (AndL src2 mask)));
13353 ins_cost(INSN_COST);
13354 format %{ "sub $dst, $src1, $src2, uxth" %}
13355
13356 ins_encode %{
13357 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13358 as_Register($src2$$reg), ext::uxth);
13359 %}
13360 ins_pipe(ialu_reg_reg);
13361 %}
13362
13363 // This pattern is automatically generated from aarch64_ad.m4
13364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13365 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13366 %{
13367 match(Set dst (SubL src1 (AndL src2 mask)));
13368 ins_cost(INSN_COST);
13369 format %{ "sub $dst, $src1, $src2, uxtw" %}
13370
13371 ins_encode %{
13372 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13373 as_Register($src2$$reg), ext::uxtw);
13374 %}
13375 ins_pipe(ialu_reg_reg);
13376 %}
13377
13378
13379 // This pattern is automatically generated from aarch64_ad.m4
13380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13381 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13382 %{
13383 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13384 ins_cost(1.9 * INSN_COST);
13385 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13386
13387 ins_encode %{
13388 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13389 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13390 %}
13391 ins_pipe(ialu_reg_reg_shift);
13392 %}
13393
13394 // This pattern is automatically generated from aarch64_ad.m4
13395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13396 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13397 %{
13398 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13399 ins_cost(1.9 * INSN_COST);
13400 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13401
13402 ins_encode %{
13403 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13404 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13405 %}
13406 ins_pipe(ialu_reg_reg_shift);
13407 %}
13408
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13412 %{
13413 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13414 ins_cost(1.9 * INSN_COST);
13415 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13416
13417 ins_encode %{
13418 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13419 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13420 %}
13421 ins_pipe(ialu_reg_reg_shift);
13422 %}
13423
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13427 %{
13428 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13429 ins_cost(1.9 * INSN_COST);
13430 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13431
13432 ins_encode %{
13433 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13434 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13435 %}
13436 ins_pipe(ialu_reg_reg_shift);
13437 %}
13438
13439 // This pattern is automatically generated from aarch64_ad.m4
13440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13441 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13442 %{
13443 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13444 ins_cost(1.9 * INSN_COST);
13445 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13446
13447 ins_encode %{
13448 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13449 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13450 %}
13451 ins_pipe(ialu_reg_reg_shift);
13452 %}
13453
13454 // This pattern is automatically generated from aarch64_ad.m4
13455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13456 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13457 %{
13458 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13459 ins_cost(1.9 * INSN_COST);
13460 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13461
13462 ins_encode %{
13463 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13464 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13465 %}
13466 ins_pipe(ialu_reg_reg_shift);
13467 %}
13468
13469 // This pattern is automatically generated from aarch64_ad.m4
13470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13471 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13472 %{
13473 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13474 ins_cost(1.9 * INSN_COST);
13475 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13476
13477 ins_encode %{
13478 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13479 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13480 %}
13481 ins_pipe(ialu_reg_reg_shift);
13482 %}
13483
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13487 %{
13488 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13489 ins_cost(1.9 * INSN_COST);
13490 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13491
13492 ins_encode %{
13493 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13494 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13495 %}
13496 ins_pipe(ialu_reg_reg_shift);
13497 %}
13498
13499 // This pattern is automatically generated from aarch64_ad.m4
13500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13501 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13502 %{
13503 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13504 ins_cost(1.9 * INSN_COST);
13505 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13506
13507 ins_encode %{
13508 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13509 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13510 %}
13511 ins_pipe(ialu_reg_reg_shift);
13512 %}
13513
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13517 %{
13518 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13519 ins_cost(1.9 * INSN_COST);
13520 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13521
13522 ins_encode %{
13523 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13524 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13525 %}
13526 ins_pipe(ialu_reg_reg_shift);
13527 %}
13528
13529 // This pattern is automatically generated from aarch64_ad.m4
13530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13531 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13532 %{
13533 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13534 ins_cost(1.9 * INSN_COST);
13535 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13536
13537 ins_encode %{
13538 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13539 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13540 %}
13541 ins_pipe(ialu_reg_reg_shift);
13542 %}
13543
13544 // This pattern is automatically generated from aarch64_ad.m4
13545 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13546 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13547 %{
13548 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13549 ins_cost(1.9 * INSN_COST);
13550 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13551
13552 ins_encode %{
13553 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13554 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13555 %}
13556 ins_pipe(ialu_reg_reg_shift);
13557 %}
13558
13559 // This pattern is automatically generated from aarch64_ad.m4
13560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13561 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13562 %{
13563 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13564 ins_cost(1.9 * INSN_COST);
13565 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13566
13567 ins_encode %{
13568 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13569 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13570 %}
13571 ins_pipe(ialu_reg_reg_shift);
13572 %}
13573
13574 // This pattern is automatically generated from aarch64_ad.m4
13575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13576 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13577 %{
13578 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13579 ins_cost(1.9 * INSN_COST);
13580 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13581
13582 ins_encode %{
13583 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13584 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13585 %}
13586 ins_pipe(ialu_reg_reg_shift);
13587 %}
13588
13589 // This pattern is automatically generated from aarch64_ad.m4
13590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13591 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13592 %{
13593 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13594 ins_cost(1.9 * INSN_COST);
13595 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13596
13597 ins_encode %{
13598 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13599 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13600 %}
13601 ins_pipe(ialu_reg_reg_shift);
13602 %}
13603
13604 // This pattern is automatically generated from aarch64_ad.m4
13605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13606 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13607 %{
13608 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13609 ins_cost(1.9 * INSN_COST);
13610 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13611
13612 ins_encode %{
13613 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13614 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13615 %}
13616 ins_pipe(ialu_reg_reg_shift);
13617 %}
13618
13619 // This pattern is automatically generated from aarch64_ad.m4
13620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13621 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13622 %{
13623 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13624 ins_cost(1.9 * INSN_COST);
13625 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13626
13627 ins_encode %{
13628 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13629 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13630 %}
13631 ins_pipe(ialu_reg_reg_shift);
13632 %}
13633
13634 // This pattern is automatically generated from aarch64_ad.m4
13635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13636 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13637 %{
13638 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13639 ins_cost(1.9 * INSN_COST);
13640 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13641
13642 ins_encode %{
13643 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13644 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13645 %}
13646 ins_pipe(ialu_reg_reg_shift);
13647 %}
13648
13649 // This pattern is automatically generated from aarch64_ad.m4
13650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13651 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13652 %{
13653 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13654 ins_cost(1.9 * INSN_COST);
13655 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13656
13657 ins_encode %{
13658 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13659 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13660 %}
13661 ins_pipe(ialu_reg_reg_shift);
13662 %}
13663
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13667 %{
13668 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13669 ins_cost(1.9 * INSN_COST);
13670 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13671
13672 ins_encode %{
13673 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13674 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13675 %}
13676 ins_pipe(ialu_reg_reg_shift);
13677 %}
13678
13679 // This pattern is automatically generated from aarch64_ad.m4
13680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13681 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13682 %{
13683 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13684 ins_cost(1.9 * INSN_COST);
13685 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13686
13687 ins_encode %{
13688 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13689 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13690 %}
13691 ins_pipe(ialu_reg_reg_shift);
13692 %}
13693
13694 // This pattern is automatically generated from aarch64_ad.m4
13695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13696 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13697 %{
13698 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13699 ins_cost(1.9 * INSN_COST);
13700 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13701
13702 ins_encode %{
13703 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13704 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13705 %}
13706 ins_pipe(ialu_reg_reg_shift);
13707 %}
13708
13709 // This pattern is automatically generated from aarch64_ad.m4
13710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13711 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13712 %{
13713 effect(DEF dst, USE src1, USE src2, USE cr);
13714 ins_cost(INSN_COST * 2);
13715 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13716
13717 ins_encode %{
13718 __ cselw($dst$$Register,
13719 $src1$$Register,
13720 $src2$$Register,
13721 Assembler::LT);
13722 %}
13723 ins_pipe(icond_reg_reg);
13724 %}
13725
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13729 %{
13730 effect(DEF dst, USE src1, USE src2, USE cr);
13731 ins_cost(INSN_COST * 2);
13732 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13733
13734 ins_encode %{
13735 __ cselw($dst$$Register,
13736 $src1$$Register,
13737 $src2$$Register,
13738 Assembler::GT);
13739 %}
13740 ins_pipe(icond_reg_reg);
13741 %}
13742
13743 // This pattern is automatically generated from aarch64_ad.m4
13744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13745 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13746 %{
13747 effect(DEF dst, USE src1, USE cr);
13748 ins_cost(INSN_COST * 2);
13749 format %{ "cselw $dst, $src1, zr lt\t" %}
13750
13751 ins_encode %{
13752 __ cselw($dst$$Register,
13753 $src1$$Register,
13754 zr,
13755 Assembler::LT);
13756 %}
13757 ins_pipe(icond_reg);
13758 %}
13759
13760 // This pattern is automatically generated from aarch64_ad.m4
13761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13762 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13763 %{
13764 effect(DEF dst, USE src1, USE cr);
13765 ins_cost(INSN_COST * 2);
13766 format %{ "cselw $dst, $src1, zr gt\t" %}
13767
13768 ins_encode %{
13769 __ cselw($dst$$Register,
13770 $src1$$Register,
13771 zr,
13772 Assembler::GT);
13773 %}
13774 ins_pipe(icond_reg);
13775 %}
13776
13777 // This pattern is automatically generated from aarch64_ad.m4
13778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13779 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13780 %{
13781 effect(DEF dst, USE src1, USE cr);
13782 ins_cost(INSN_COST * 2);
13783 format %{ "csincw $dst, $src1, zr le\t" %}
13784
13785 ins_encode %{
13786 __ csincw($dst$$Register,
13787 $src1$$Register,
13788 zr,
13789 Assembler::LE);
13790 %}
13791 ins_pipe(icond_reg);
13792 %}
13793
13794 // This pattern is automatically generated from aarch64_ad.m4
13795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13796 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13797 %{
13798 effect(DEF dst, USE src1, USE cr);
13799 ins_cost(INSN_COST * 2);
13800 format %{ "csincw $dst, $src1, zr gt\t" %}
13801
13802 ins_encode %{
13803 __ csincw($dst$$Register,
13804 $src1$$Register,
13805 zr,
13806 Assembler::GT);
13807 %}
13808 ins_pipe(icond_reg);
13809 %}
13810
13811 // This pattern is automatically generated from aarch64_ad.m4
13812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13813 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13814 %{
13815 effect(DEF dst, USE src1, USE cr);
13816 ins_cost(INSN_COST * 2);
13817 format %{ "csinvw $dst, $src1, zr lt\t" %}
13818
13819 ins_encode %{
13820 __ csinvw($dst$$Register,
13821 $src1$$Register,
13822 zr,
13823 Assembler::LT);
13824 %}
13825 ins_pipe(icond_reg);
13826 %}
13827
13828 // This pattern is automatically generated from aarch64_ad.m4
13829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13830 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13831 %{
13832 effect(DEF dst, USE src1, USE cr);
13833 ins_cost(INSN_COST * 2);
13834 format %{ "csinvw $dst, $src1, zr ge\t" %}
13835
13836 ins_encode %{
13837 __ csinvw($dst$$Register,
13838 $src1$$Register,
13839 zr,
13840 Assembler::GE);
13841 %}
13842 ins_pipe(icond_reg);
13843 %}
13844
13845 // This pattern is automatically generated from aarch64_ad.m4
13846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13847 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13848 %{
13849 match(Set dst (MinI src imm));
13850 ins_cost(INSN_COST * 3);
13851 expand %{
13852 rFlagsReg cr;
13853 compI_reg_imm0(cr, src);
13854 cmovI_reg_imm0_lt(dst, src, cr);
13855 %}
13856 %}
13857
13858 // This pattern is automatically generated from aarch64_ad.m4
13859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13860 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13861 %{
13862 match(Set dst (MinI imm src));
13863 ins_cost(INSN_COST * 3);
13864 expand %{
13865 rFlagsReg cr;
13866 compI_reg_imm0(cr, src);
13867 cmovI_reg_imm0_lt(dst, src, cr);
13868 %}
13869 %}
13870
13871 // This pattern is automatically generated from aarch64_ad.m4
13872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13873 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13874 %{
13875 match(Set dst (MinI src imm));
13876 ins_cost(INSN_COST * 3);
13877 expand %{
13878 rFlagsReg cr;
13879 compI_reg_imm0(cr, src);
13880 cmovI_reg_imm1_le(dst, src, cr);
13881 %}
13882 %}
13883
13884 // This pattern is automatically generated from aarch64_ad.m4
13885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13886 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13887 %{
13888 match(Set dst (MinI imm src));
13889 ins_cost(INSN_COST * 3);
13890 expand %{
13891 rFlagsReg cr;
13892 compI_reg_imm0(cr, src);
13893 cmovI_reg_imm1_le(dst, src, cr);
13894 %}
13895 %}
13896
13897 // This pattern is automatically generated from aarch64_ad.m4
13898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13899 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13900 %{
13901 match(Set dst (MinI src imm));
13902 ins_cost(INSN_COST * 3);
13903 expand %{
13904 rFlagsReg cr;
13905 compI_reg_imm0(cr, src);
13906 cmovI_reg_immM1_lt(dst, src, cr);
13907 %}
13908 %}
13909
13910 // This pattern is automatically generated from aarch64_ad.m4
13911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13912 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13913 %{
13914 match(Set dst (MinI imm src));
13915 ins_cost(INSN_COST * 3);
13916 expand %{
13917 rFlagsReg cr;
13918 compI_reg_imm0(cr, src);
13919 cmovI_reg_immM1_lt(dst, src, cr);
13920 %}
13921 %}
13922
13923 // This pattern is automatically generated from aarch64_ad.m4
13924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13925 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13926 %{
13927 match(Set dst (MaxI src imm));
13928 ins_cost(INSN_COST * 3);
13929 expand %{
13930 rFlagsReg cr;
13931 compI_reg_imm0(cr, src);
13932 cmovI_reg_imm0_gt(dst, src, cr);
13933 %}
13934 %}
13935
13936 // This pattern is automatically generated from aarch64_ad.m4
13937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13938 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13939 %{
13940 match(Set dst (MaxI imm src));
13941 ins_cost(INSN_COST * 3);
13942 expand %{
13943 rFlagsReg cr;
13944 compI_reg_imm0(cr, src);
13945 cmovI_reg_imm0_gt(dst, src, cr);
13946 %}
13947 %}
13948
13949 // This pattern is automatically generated from aarch64_ad.m4
13950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13951 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13952 %{
13953 match(Set dst (MaxI src imm));
13954 ins_cost(INSN_COST * 3);
13955 expand %{
13956 rFlagsReg cr;
13957 compI_reg_imm0(cr, src);
13958 cmovI_reg_imm1_gt(dst, src, cr);
13959 %}
13960 %}
13961
13962 // This pattern is automatically generated from aarch64_ad.m4
13963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13964 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13965 %{
13966 match(Set dst (MaxI imm src));
13967 ins_cost(INSN_COST * 3);
13968 expand %{
13969 rFlagsReg cr;
13970 compI_reg_imm0(cr, src);
13971 cmovI_reg_imm1_gt(dst, src, cr);
13972 %}
13973 %}
13974
13975 // This pattern is automatically generated from aarch64_ad.m4
13976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13977 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13978 %{
13979 match(Set dst (MaxI src imm));
13980 ins_cost(INSN_COST * 3);
13981 expand %{
13982 rFlagsReg cr;
13983 compI_reg_imm0(cr, src);
13984 cmovI_reg_immM1_ge(dst, src, cr);
13985 %}
13986 %}
13987
13988 // This pattern is automatically generated from aarch64_ad.m4
13989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13990 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13991 %{
13992 match(Set dst (MaxI imm src));
13993 ins_cost(INSN_COST * 3);
13994 expand %{
13995 rFlagsReg cr;
13996 compI_reg_imm0(cr, src);
13997 cmovI_reg_immM1_ge(dst, src, cr);
13998 %}
13999 %}
14000
14001 // This pattern is automatically generated from aarch64_ad.m4
14002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14003 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
14004 %{
14005 match(Set dst (ReverseI src));
14006 ins_cost(INSN_COST);
14007 format %{ "rbitw $dst, $src" %}
14008 ins_encode %{
14009 __ rbitw($dst$$Register, $src$$Register);
14010 %}
14011 ins_pipe(ialu_reg);
14012 %}
14013
14014 // This pattern is automatically generated from aarch64_ad.m4
14015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14016 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
14017 %{
14018 match(Set dst (ReverseL src));
14019 ins_cost(INSN_COST);
14020 format %{ "rbit $dst, $src" %}
14021 ins_encode %{
14022 __ rbit($dst$$Register, $src$$Register);
14023 %}
14024 ins_pipe(ialu_reg);
14025 %}
14026
14027
14028 // END This section of the file is automatically generated. Do not edit --------------
14029
14030
14031 // ============================================================================
14032 // Floating Point Arithmetic Instructions
14033
14034 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14035 match(Set dst (AddF src1 src2));
14036
14037 ins_cost(INSN_COST * 5);
14038 format %{ "fadds $dst, $src1, $src2" %}
14039
14040 ins_encode %{
14041 __ fadds(as_FloatRegister($dst$$reg),
14042 as_FloatRegister($src1$$reg),
14043 as_FloatRegister($src2$$reg));
14044 %}
14045
14046 ins_pipe(fp_dop_reg_reg_s);
14047 %}
14048
14049 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14050 match(Set dst (AddD src1 src2));
14051
14052 ins_cost(INSN_COST * 5);
14053 format %{ "faddd $dst, $src1, $src2" %}
14054
14055 ins_encode %{
14056 __ faddd(as_FloatRegister($dst$$reg),
14057 as_FloatRegister($src1$$reg),
14058 as_FloatRegister($src2$$reg));
14059 %}
14060
14061 ins_pipe(fp_dop_reg_reg_d);
14062 %}
14063
14064 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14065 match(Set dst (SubF src1 src2));
14066
14067 ins_cost(INSN_COST * 5);
14068 format %{ "fsubs $dst, $src1, $src2" %}
14069
14070 ins_encode %{
14071 __ fsubs(as_FloatRegister($dst$$reg),
14072 as_FloatRegister($src1$$reg),
14073 as_FloatRegister($src2$$reg));
14074 %}
14075
14076 ins_pipe(fp_dop_reg_reg_s);
14077 %}
14078
14079 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14080 match(Set dst (SubD src1 src2));
14081
14082 ins_cost(INSN_COST * 5);
14083 format %{ "fsubd $dst, $src1, $src2" %}
14084
14085 ins_encode %{
14086 __ fsubd(as_FloatRegister($dst$$reg),
14087 as_FloatRegister($src1$$reg),
14088 as_FloatRegister($src2$$reg));
14089 %}
14090
14091 ins_pipe(fp_dop_reg_reg_d);
14092 %}
14093
14094 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14095 match(Set dst (MulF src1 src2));
14096
14097 ins_cost(INSN_COST * 6);
14098 format %{ "fmuls $dst, $src1, $src2" %}
14099
14100 ins_encode %{
14101 __ fmuls(as_FloatRegister($dst$$reg),
14102 as_FloatRegister($src1$$reg),
14103 as_FloatRegister($src2$$reg));
14104 %}
14105
14106 ins_pipe(fp_dop_reg_reg_s);
14107 %}
14108
14109 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14110 match(Set dst (MulD src1 src2));
14111
14112 ins_cost(INSN_COST * 6);
14113 format %{ "fmuld $dst, $src1, $src2" %}
14114
14115 ins_encode %{
14116 __ fmuld(as_FloatRegister($dst$$reg),
14117 as_FloatRegister($src1$$reg),
14118 as_FloatRegister($src2$$reg));
14119 %}
14120
14121 ins_pipe(fp_dop_reg_reg_d);
14122 %}
14123
14124 // src1 * src2 + src3
14125 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14126 predicate(UseFMA);
14127 match(Set dst (FmaF src3 (Binary src1 src2)));
14128
14129 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14130
14131 ins_encode %{
14132 __ fmadds(as_FloatRegister($dst$$reg),
14133 as_FloatRegister($src1$$reg),
14134 as_FloatRegister($src2$$reg),
14135 as_FloatRegister($src3$$reg));
14136 %}
14137
14138 ins_pipe(pipe_class_default);
14139 %}
14140
14141 // src1 * src2 + src3
14142 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14143 predicate(UseFMA);
14144 match(Set dst (FmaD src3 (Binary src1 src2)));
14145
14146 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14147
14148 ins_encode %{
14149 __ fmaddd(as_FloatRegister($dst$$reg),
14150 as_FloatRegister($src1$$reg),
14151 as_FloatRegister($src2$$reg),
14152 as_FloatRegister($src3$$reg));
14153 %}
14154
14155 ins_pipe(pipe_class_default);
14156 %}
14157
14158 // -src1 * src2 + src3
14159 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14160 predicate(UseFMA);
14161 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14162 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14163
14164 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14165
14166 ins_encode %{
14167 __ fmsubs(as_FloatRegister($dst$$reg),
14168 as_FloatRegister($src1$$reg),
14169 as_FloatRegister($src2$$reg),
14170 as_FloatRegister($src3$$reg));
14171 %}
14172
14173 ins_pipe(pipe_class_default);
14174 %}
14175
14176 // -src1 * src2 + src3
14177 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14178 predicate(UseFMA);
14179 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14180 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14181
14182 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14183
14184 ins_encode %{
14185 __ fmsubd(as_FloatRegister($dst$$reg),
14186 as_FloatRegister($src1$$reg),
14187 as_FloatRegister($src2$$reg),
14188 as_FloatRegister($src3$$reg));
14189 %}
14190
14191 ins_pipe(pipe_class_default);
14192 %}
14193
14194 // -src1 * src2 - src3
14195 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14196 predicate(UseFMA);
14197 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14198 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14199
14200 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14201
14202 ins_encode %{
14203 __ fnmadds(as_FloatRegister($dst$$reg),
14204 as_FloatRegister($src1$$reg),
14205 as_FloatRegister($src2$$reg),
14206 as_FloatRegister($src3$$reg));
14207 %}
14208
14209 ins_pipe(pipe_class_default);
14210 %}
14211
14212 // -src1 * src2 - src3
14213 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14214 predicate(UseFMA);
14215 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14216 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14217
14218 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14219
14220 ins_encode %{
14221 __ fnmaddd(as_FloatRegister($dst$$reg),
14222 as_FloatRegister($src1$$reg),
14223 as_FloatRegister($src2$$reg),
14224 as_FloatRegister($src3$$reg));
14225 %}
14226
14227 ins_pipe(pipe_class_default);
14228 %}
14229
14230 // src1 * src2 - src3
14231 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14232 predicate(UseFMA);
14233 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14234
14235 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14236
14237 ins_encode %{
14238 __ fnmsubs(as_FloatRegister($dst$$reg),
14239 as_FloatRegister($src1$$reg),
14240 as_FloatRegister($src2$$reg),
14241 as_FloatRegister($src3$$reg));
14242 %}
14243
14244 ins_pipe(pipe_class_default);
14245 %}
14246
14247 // src1 * src2 - src3
14248 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14249 predicate(UseFMA);
14250 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14251
14252 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14253
14254 ins_encode %{
14255 // n.b. insn name should be fnmsubd
14256 __ fnmsub(as_FloatRegister($dst$$reg),
14257 as_FloatRegister($src1$$reg),
14258 as_FloatRegister($src2$$reg),
14259 as_FloatRegister($src3$$reg));
14260 %}
14261
14262 ins_pipe(pipe_class_default);
14263 %}
14264
14265
14266 // Math.max(FF)F
14267 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14268 match(Set dst (MaxF src1 src2));
14269
14270 format %{ "fmaxs $dst, $src1, $src2" %}
14271 ins_encode %{
14272 __ fmaxs(as_FloatRegister($dst$$reg),
14273 as_FloatRegister($src1$$reg),
14274 as_FloatRegister($src2$$reg));
14275 %}
14276
14277 ins_pipe(fp_dop_reg_reg_s);
14278 %}
14279
14280 // Math.min(FF)F
14281 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14282 match(Set dst (MinF src1 src2));
14283
14284 format %{ "fmins $dst, $src1, $src2" %}
14285 ins_encode %{
14286 __ fmins(as_FloatRegister($dst$$reg),
14287 as_FloatRegister($src1$$reg),
14288 as_FloatRegister($src2$$reg));
14289 %}
14290
14291 ins_pipe(fp_dop_reg_reg_s);
14292 %}
14293
14294 // Math.max(DD)D
14295 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14296 match(Set dst (MaxD src1 src2));
14297
14298 format %{ "fmaxd $dst, $src1, $src2" %}
14299 ins_encode %{
14300 __ fmaxd(as_FloatRegister($dst$$reg),
14301 as_FloatRegister($src1$$reg),
14302 as_FloatRegister($src2$$reg));
14303 %}
14304
14305 ins_pipe(fp_dop_reg_reg_d);
14306 %}
14307
14308 // Math.min(DD)D
14309 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14310 match(Set dst (MinD src1 src2));
14311
14312 format %{ "fmind $dst, $src1, $src2" %}
14313 ins_encode %{
14314 __ fmind(as_FloatRegister($dst$$reg),
14315 as_FloatRegister($src1$$reg),
14316 as_FloatRegister($src2$$reg));
14317 %}
14318
14319 ins_pipe(fp_dop_reg_reg_d);
14320 %}
14321
14322
14323 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14324 match(Set dst (DivF src1 src2));
14325
14326 ins_cost(INSN_COST * 18);
14327 format %{ "fdivs $dst, $src1, $src2" %}
14328
14329 ins_encode %{
14330 __ fdivs(as_FloatRegister($dst$$reg),
14331 as_FloatRegister($src1$$reg),
14332 as_FloatRegister($src2$$reg));
14333 %}
14334
14335 ins_pipe(fp_div_s);
14336 %}
14337
14338 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14339 match(Set dst (DivD src1 src2));
14340
14341 ins_cost(INSN_COST * 32);
14342 format %{ "fdivd $dst, $src1, $src2" %}
14343
14344 ins_encode %{
14345 __ fdivd(as_FloatRegister($dst$$reg),
14346 as_FloatRegister($src1$$reg),
14347 as_FloatRegister($src2$$reg));
14348 %}
14349
14350 ins_pipe(fp_div_d);
14351 %}
14352
14353 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14354 match(Set dst (NegF src));
14355
14356 ins_cost(INSN_COST * 3);
14357 format %{ "fneg $dst, $src" %}
14358
14359 ins_encode %{
14360 __ fnegs(as_FloatRegister($dst$$reg),
14361 as_FloatRegister($src$$reg));
14362 %}
14363
14364 ins_pipe(fp_uop_s);
14365 %}
14366
14367 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14368 match(Set dst (NegD src));
14369
14370 ins_cost(INSN_COST * 3);
14371 format %{ "fnegd $dst, $src" %}
14372
14373 ins_encode %{
14374 __ fnegd(as_FloatRegister($dst$$reg),
14375 as_FloatRegister($src$$reg));
14376 %}
14377
14378 ins_pipe(fp_uop_d);
14379 %}
14380
14381 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14382 %{
14383 match(Set dst (AbsI src));
14384
14385 effect(KILL cr);
14386 ins_cost(INSN_COST * 2);
14387 format %{ "cmpw $src, zr\n\t"
14388 "cnegw $dst, $src, Assembler::LT\t# int abs"
14389 %}
14390
14391 ins_encode %{
14392 __ cmpw(as_Register($src$$reg), zr);
14393 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14394 %}
14395 ins_pipe(pipe_class_default);
14396 %}
14397
14398 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14399 %{
14400 match(Set dst (AbsL src));
14401
14402 effect(KILL cr);
14403 ins_cost(INSN_COST * 2);
14404 format %{ "cmp $src, zr\n\t"
14405 "cneg $dst, $src, Assembler::LT\t# long abs"
14406 %}
14407
14408 ins_encode %{
14409 __ cmp(as_Register($src$$reg), zr);
14410 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14411 %}
14412 ins_pipe(pipe_class_default);
14413 %}
14414
14415 instruct absF_reg(vRegF dst, vRegF src) %{
14416 match(Set dst (AbsF src));
14417
14418 ins_cost(INSN_COST * 3);
14419 format %{ "fabss $dst, $src" %}
14420 ins_encode %{
14421 __ fabss(as_FloatRegister($dst$$reg),
14422 as_FloatRegister($src$$reg));
14423 %}
14424
14425 ins_pipe(fp_uop_s);
14426 %}
14427
14428 instruct absD_reg(vRegD dst, vRegD src) %{
14429 match(Set dst (AbsD src));
14430
14431 ins_cost(INSN_COST * 3);
14432 format %{ "fabsd $dst, $src" %}
14433 ins_encode %{
14434 __ fabsd(as_FloatRegister($dst$$reg),
14435 as_FloatRegister($src$$reg));
14436 %}
14437
14438 ins_pipe(fp_uop_d);
14439 %}
14440
14441 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14442 match(Set dst (AbsF (SubF src1 src2)));
14443
14444 ins_cost(INSN_COST * 3);
14445 format %{ "fabds $dst, $src1, $src2" %}
14446 ins_encode %{
14447 __ fabds(as_FloatRegister($dst$$reg),
14448 as_FloatRegister($src1$$reg),
14449 as_FloatRegister($src2$$reg));
14450 %}
14451
14452 ins_pipe(fp_uop_s);
14453 %}
14454
14455 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14456 match(Set dst (AbsD (SubD src1 src2)));
14457
14458 ins_cost(INSN_COST * 3);
14459 format %{ "fabdd $dst, $src1, $src2" %}
14460 ins_encode %{
14461 __ fabdd(as_FloatRegister($dst$$reg),
14462 as_FloatRegister($src1$$reg),
14463 as_FloatRegister($src2$$reg));
14464 %}
14465
14466 ins_pipe(fp_uop_d);
14467 %}
14468
14469 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14470 match(Set dst (SqrtD src));
14471
14472 ins_cost(INSN_COST * 50);
14473 format %{ "fsqrtd $dst, $src" %}
14474 ins_encode %{
14475 __ fsqrtd(as_FloatRegister($dst$$reg),
14476 as_FloatRegister($src$$reg));
14477 %}
14478
14479 ins_pipe(fp_div_s);
14480 %}
14481
14482 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14483 match(Set dst (SqrtF src));
14484
14485 ins_cost(INSN_COST * 50);
14486 format %{ "fsqrts $dst, $src" %}
14487 ins_encode %{
14488 __ fsqrts(as_FloatRegister($dst$$reg),
14489 as_FloatRegister($src$$reg));
14490 %}
14491
14492 ins_pipe(fp_div_d);
14493 %}
14494
14495 // Math.rint, floor, ceil
14496 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14497 match(Set dst (RoundDoubleMode src rmode));
14498 format %{ "frint $dst, $src, $rmode" %}
14499 ins_encode %{
14500 switch ($rmode$$constant) {
14501 case RoundDoubleModeNode::rmode_rint:
14502 __ frintnd(as_FloatRegister($dst$$reg),
14503 as_FloatRegister($src$$reg));
14504 break;
14505 case RoundDoubleModeNode::rmode_floor:
14506 __ frintmd(as_FloatRegister($dst$$reg),
14507 as_FloatRegister($src$$reg));
14508 break;
14509 case RoundDoubleModeNode::rmode_ceil:
14510 __ frintpd(as_FloatRegister($dst$$reg),
14511 as_FloatRegister($src$$reg));
14512 break;
14513 }
14514 %}
14515 ins_pipe(fp_uop_d);
14516 %}
14517
14518 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14519 match(Set dst (CopySignD src1 (Binary src2 zero)));
14520 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14521 format %{ "CopySignD $dst $src1 $src2" %}
14522 ins_encode %{
14523 FloatRegister dst = as_FloatRegister($dst$$reg),
14524 src1 = as_FloatRegister($src1$$reg),
14525 src2 = as_FloatRegister($src2$$reg),
14526 zero = as_FloatRegister($zero$$reg);
14527 __ fnegd(dst, zero);
14528 __ bsl(dst, __ T8B, src2, src1);
14529 %}
14530 ins_pipe(fp_uop_d);
14531 %}
14532
14533 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14534 match(Set dst (CopySignF src1 src2));
14535 effect(TEMP_DEF dst, USE src1, USE src2);
14536 format %{ "CopySignF $dst $src1 $src2" %}
14537 ins_encode %{
14538 FloatRegister dst = as_FloatRegister($dst$$reg),
14539 src1 = as_FloatRegister($src1$$reg),
14540 src2 = as_FloatRegister($src2$$reg);
14541 __ movi(dst, __ T2S, 0x80, 24);
14542 __ bsl(dst, __ T8B, src2, src1);
14543 %}
14544 ins_pipe(fp_uop_d);
14545 %}
14546
14547 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14548 match(Set dst (SignumD src (Binary zero one)));
14549 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14550 format %{ "signumD $dst, $src" %}
14551 ins_encode %{
14552 FloatRegister src = as_FloatRegister($src$$reg),
14553 dst = as_FloatRegister($dst$$reg),
14554 zero = as_FloatRegister($zero$$reg),
14555 one = as_FloatRegister($one$$reg);
14556 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14557 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14558 // Bit selection instruction gets bit from "one" for each enabled bit in
14559 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14560 // NaN the whole "src" will be copied because "dst" is zero. For all other
14561 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14562 // from "src", and all other bits are copied from 1.0.
14563 __ bsl(dst, __ T8B, one, src);
14564 %}
14565 ins_pipe(fp_uop_d);
14566 %}
14567
14568 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14569 match(Set dst (SignumF src (Binary zero one)));
14570 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14571 format %{ "signumF $dst, $src" %}
14572 ins_encode %{
14573 FloatRegister src = as_FloatRegister($src$$reg),
14574 dst = as_FloatRegister($dst$$reg),
14575 zero = as_FloatRegister($zero$$reg),
14576 one = as_FloatRegister($one$$reg);
14577 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14578 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14579 // Bit selection instruction gets bit from "one" for each enabled bit in
14580 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14581 // NaN the whole "src" will be copied because "dst" is zero. For all other
14582 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14583 // from "src", and all other bits are copied from 1.0.
14584 __ bsl(dst, __ T8B, one, src);
14585 %}
14586 ins_pipe(fp_uop_d);
14587 %}
14588
14589 instruct onspinwait() %{
14590 match(OnSpinWait);
14591 ins_cost(INSN_COST);
14592
14593 format %{ "onspinwait" %}
14594
14595 ins_encode %{
14596 __ spin_wait();
14597 %}
14598 ins_pipe(pipe_class_empty);
14599 %}
14600
14601 // ============================================================================
14602 // Logical Instructions
14603
14604 // Integer Logical Instructions
14605
14606 // And Instructions
14607
14608
14609 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14610 match(Set dst (AndI src1 src2));
14611
14612 format %{ "andw $dst, $src1, $src2\t# int" %}
14613
14614 ins_cost(INSN_COST);
14615 ins_encode %{
14616 __ andw(as_Register($dst$$reg),
14617 as_Register($src1$$reg),
14618 as_Register($src2$$reg));
14619 %}
14620
14621 ins_pipe(ialu_reg_reg);
14622 %}
14623
14624 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14625 match(Set dst (AndI src1 src2));
14626
14627 format %{ "andsw $dst, $src1, $src2\t# int" %}
14628
14629 ins_cost(INSN_COST);
14630 ins_encode %{
14631 __ andw(as_Register($dst$$reg),
14632 as_Register($src1$$reg),
14633 (uint64_t)($src2$$constant));
14634 %}
14635
14636 ins_pipe(ialu_reg_imm);
14637 %}
14638
14639 // Or Instructions
14640
14641 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14642 match(Set dst (OrI src1 src2));
14643
14644 format %{ "orrw $dst, $src1, $src2\t# int" %}
14645
14646 ins_cost(INSN_COST);
14647 ins_encode %{
14648 __ orrw(as_Register($dst$$reg),
14649 as_Register($src1$$reg),
14650 as_Register($src2$$reg));
14651 %}
14652
14653 ins_pipe(ialu_reg_reg);
14654 %}
14655
14656 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14657 match(Set dst (OrI src1 src2));
14658
14659 format %{ "orrw $dst, $src1, $src2\t# int" %}
14660
14661 ins_cost(INSN_COST);
14662 ins_encode %{
14663 __ orrw(as_Register($dst$$reg),
14664 as_Register($src1$$reg),
14665 (uint64_t)($src2$$constant));
14666 %}
14667
14668 ins_pipe(ialu_reg_imm);
14669 %}
14670
14671 // Xor Instructions
14672
14673 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14674 match(Set dst (XorI src1 src2));
14675
14676 format %{ "eorw $dst, $src1, $src2\t# int" %}
14677
14678 ins_cost(INSN_COST);
14679 ins_encode %{
14680 __ eorw(as_Register($dst$$reg),
14681 as_Register($src1$$reg),
14682 as_Register($src2$$reg));
14683 %}
14684
14685 ins_pipe(ialu_reg_reg);
14686 %}
14687
14688 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14689 match(Set dst (XorI src1 src2));
14690
14691 format %{ "eorw $dst, $src1, $src2\t# int" %}
14692
14693 ins_cost(INSN_COST);
14694 ins_encode %{
14695 __ eorw(as_Register($dst$$reg),
14696 as_Register($src1$$reg),
14697 (uint64_t)($src2$$constant));
14698 %}
14699
14700 ins_pipe(ialu_reg_imm);
14701 %}
14702
14703 // Long Logical Instructions
14704 // TODO
14705
14706 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14707 match(Set dst (AndL src1 src2));
14708
14709 format %{ "and $dst, $src1, $src2\t# int" %}
14710
14711 ins_cost(INSN_COST);
14712 ins_encode %{
14713 __ andr(as_Register($dst$$reg),
14714 as_Register($src1$$reg),
14715 as_Register($src2$$reg));
14716 %}
14717
14718 ins_pipe(ialu_reg_reg);
14719 %}
14720
14721 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14722 match(Set dst (AndL src1 src2));
14723
14724 format %{ "and $dst, $src1, $src2\t# int" %}
14725
14726 ins_cost(INSN_COST);
14727 ins_encode %{
14728 __ andr(as_Register($dst$$reg),
14729 as_Register($src1$$reg),
14730 (uint64_t)($src2$$constant));
14731 %}
14732
14733 ins_pipe(ialu_reg_imm);
14734 %}
14735
14736 // Or Instructions
14737
14738 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14739 match(Set dst (OrL src1 src2));
14740
14741 format %{ "orr $dst, $src1, $src2\t# int" %}
14742
14743 ins_cost(INSN_COST);
14744 ins_encode %{
14745 __ orr(as_Register($dst$$reg),
14746 as_Register($src1$$reg),
14747 as_Register($src2$$reg));
14748 %}
14749
14750 ins_pipe(ialu_reg_reg);
14751 %}
14752
14753 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14754 match(Set dst (OrL src1 src2));
14755
14756 format %{ "orr $dst, $src1, $src2\t# int" %}
14757
14758 ins_cost(INSN_COST);
14759 ins_encode %{
14760 __ orr(as_Register($dst$$reg),
14761 as_Register($src1$$reg),
14762 (uint64_t)($src2$$constant));
14763 %}
14764
14765 ins_pipe(ialu_reg_imm);
14766 %}
14767
14768 // Xor Instructions
14769
14770 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14771 match(Set dst (XorL src1 src2));
14772
14773 format %{ "eor $dst, $src1, $src2\t# int" %}
14774
14775 ins_cost(INSN_COST);
14776 ins_encode %{
14777 __ eor(as_Register($dst$$reg),
14778 as_Register($src1$$reg),
14779 as_Register($src2$$reg));
14780 %}
14781
14782 ins_pipe(ialu_reg_reg);
14783 %}
14784
14785 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14786 match(Set dst (XorL src1 src2));
14787
14788 ins_cost(INSN_COST);
14789 format %{ "eor $dst, $src1, $src2\t# int" %}
14790
14791 ins_encode %{
14792 __ eor(as_Register($dst$$reg),
14793 as_Register($src1$$reg),
14794 (uint64_t)($src2$$constant));
14795 %}
14796
14797 ins_pipe(ialu_reg_imm);
14798 %}
14799
14800 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14801 %{
14802 match(Set dst (ConvI2L src));
14803
14804 ins_cost(INSN_COST);
14805 format %{ "sxtw $dst, $src\t# i2l" %}
14806 ins_encode %{
14807 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14808 %}
14809 ins_pipe(ialu_reg_shift);
14810 %}
14811
14812 // this pattern occurs in bigmath arithmetic
14813 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14814 %{
14815 match(Set dst (AndL (ConvI2L src) mask));
14816
14817 ins_cost(INSN_COST);
14818 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14819 ins_encode %{
14820 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14821 %}
14822
14823 ins_pipe(ialu_reg_shift);
14824 %}
14825
14826 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14827 match(Set dst (ConvL2I src));
14828
14829 ins_cost(INSN_COST);
14830 format %{ "movw $dst, $src \t// l2i" %}
14831
14832 ins_encode %{
14833 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14834 %}
14835
14836 ins_pipe(ialu_reg);
14837 %}
14838
14839 instruct convD2F_reg(vRegF dst, vRegD src) %{
14840 match(Set dst (ConvD2F src));
14841
14842 ins_cost(INSN_COST * 5);
14843 format %{ "fcvtd $dst, $src \t// d2f" %}
14844
14845 ins_encode %{
14846 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14847 %}
14848
14849 ins_pipe(fp_d2f);
14850 %}
14851
14852 instruct convF2D_reg(vRegD dst, vRegF src) %{
14853 match(Set dst (ConvF2D src));
14854
14855 ins_cost(INSN_COST * 5);
14856 format %{ "fcvts $dst, $src \t// f2d" %}
14857
14858 ins_encode %{
14859 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14860 %}
14861
14862 ins_pipe(fp_f2d);
14863 %}
14864
14865 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14866 match(Set dst (ConvF2I src));
14867
14868 ins_cost(INSN_COST * 5);
14869 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14870
14871 ins_encode %{
14872 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14873 %}
14874
14875 ins_pipe(fp_f2i);
14876 %}
14877
14878 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14879 match(Set dst (ConvF2L src));
14880
14881 ins_cost(INSN_COST * 5);
14882 format %{ "fcvtzs $dst, $src \t// f2l" %}
14883
14884 ins_encode %{
14885 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14886 %}
14887
14888 ins_pipe(fp_f2l);
14889 %}
14890
14891 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14892 match(Set dst (ConvF2HF src));
14893 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14894 "smov $dst, $tmp\t# move result from $tmp to $dst"
14895 %}
14896 effect(TEMP tmp);
14897 ins_encode %{
14898 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14899 %}
14900 ins_pipe(pipe_slow);
14901 %}
14902
14903 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14904 match(Set dst (ConvHF2F src));
14905 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14906 "fcvt $dst, $tmp\t# convert half to single precision"
14907 %}
14908 effect(TEMP tmp);
14909 ins_encode %{
14910 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14911 %}
14912 ins_pipe(pipe_slow);
14913 %}
14914
14915 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14916 match(Set dst (ConvI2F src));
14917
14918 ins_cost(INSN_COST * 5);
14919 format %{ "scvtfws $dst, $src \t// i2f" %}
14920
14921 ins_encode %{
14922 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14923 %}
14924
14925 ins_pipe(fp_i2f);
14926 %}
14927
14928 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14929 match(Set dst (ConvL2F src));
14930
14931 ins_cost(INSN_COST * 5);
14932 format %{ "scvtfs $dst, $src \t// l2f" %}
14933
14934 ins_encode %{
14935 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14936 %}
14937
14938 ins_pipe(fp_l2f);
14939 %}
14940
14941 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14942 match(Set dst (ConvD2I src));
14943
14944 ins_cost(INSN_COST * 5);
14945 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14946
14947 ins_encode %{
14948 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14949 %}
14950
14951 ins_pipe(fp_d2i);
14952 %}
14953
14954 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14955 match(Set dst (ConvD2L src));
14956
14957 ins_cost(INSN_COST * 5);
14958 format %{ "fcvtzd $dst, $src \t// d2l" %}
14959
14960 ins_encode %{
14961 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14962 %}
14963
14964 ins_pipe(fp_d2l);
14965 %}
14966
14967 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14968 match(Set dst (ConvI2D src));
14969
14970 ins_cost(INSN_COST * 5);
14971 format %{ "scvtfwd $dst, $src \t// i2d" %}
14972
14973 ins_encode %{
14974 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14975 %}
14976
14977 ins_pipe(fp_i2d);
14978 %}
14979
14980 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14981 match(Set dst (ConvL2D src));
14982
14983 ins_cost(INSN_COST * 5);
14984 format %{ "scvtfd $dst, $src \t// l2d" %}
14985
14986 ins_encode %{
14987 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14988 %}
14989
14990 ins_pipe(fp_l2d);
14991 %}
14992
14993 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14994 %{
14995 match(Set dst (RoundD src));
14996 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14997 format %{ "java_round_double $dst,$src"%}
14998 ins_encode %{
14999 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15000 as_FloatRegister($ftmp$$reg));
15001 %}
15002 ins_pipe(pipe_slow);
15003 %}
15004
15005 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15006 %{
15007 match(Set dst (RoundF src));
15008 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15009 format %{ "java_round_float $dst,$src"%}
15010 ins_encode %{
15011 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15012 as_FloatRegister($ftmp$$reg));
15013 %}
15014 ins_pipe(pipe_slow);
15015 %}
15016
15017 // stack <-> reg and reg <-> reg shuffles with no conversion
15018
15019 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15020
15021 match(Set dst (MoveF2I src));
15022
15023 effect(DEF dst, USE src);
15024
15025 ins_cost(4 * INSN_COST);
15026
15027 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15028
15029 ins_encode %{
15030 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15031 %}
15032
15033 ins_pipe(iload_reg_reg);
15034
15035 %}
15036
15037 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15038
15039 match(Set dst (MoveI2F src));
15040
15041 effect(DEF dst, USE src);
15042
15043 ins_cost(4 * INSN_COST);
15044
15045 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15046
15047 ins_encode %{
15048 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15049 %}
15050
15051 ins_pipe(pipe_class_memory);
15052
15053 %}
15054
15055 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15056
15057 match(Set dst (MoveD2L src));
15058
15059 effect(DEF dst, USE src);
15060
15061 ins_cost(4 * INSN_COST);
15062
15063 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15064
15065 ins_encode %{
15066 __ ldr($dst$$Register, Address(sp, $src$$disp));
15067 %}
15068
15069 ins_pipe(iload_reg_reg);
15070
15071 %}
15072
15073 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15074
15075 match(Set dst (MoveL2D src));
15076
15077 effect(DEF dst, USE src);
15078
15079 ins_cost(4 * INSN_COST);
15080
15081 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15082
15083 ins_encode %{
15084 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15085 %}
15086
15087 ins_pipe(pipe_class_memory);
15088
15089 %}
15090
15091 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15092
15093 match(Set dst (MoveF2I src));
15094
15095 effect(DEF dst, USE src);
15096
15097 ins_cost(INSN_COST);
15098
15099 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15100
15101 ins_encode %{
15102 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15103 %}
15104
15105 ins_pipe(pipe_class_memory);
15106
15107 %}
15108
15109 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15110
15111 match(Set dst (MoveI2F src));
15112
15113 effect(DEF dst, USE src);
15114
15115 ins_cost(INSN_COST);
15116
15117 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15118
15119 ins_encode %{
15120 __ strw($src$$Register, Address(sp, $dst$$disp));
15121 %}
15122
15123 ins_pipe(istore_reg_reg);
15124
15125 %}
15126
15127 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15128
15129 match(Set dst (MoveD2L src));
15130
15131 effect(DEF dst, USE src);
15132
15133 ins_cost(INSN_COST);
15134
15135 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15136
15137 ins_encode %{
15138 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15139 %}
15140
15141 ins_pipe(pipe_class_memory);
15142
15143 %}
15144
15145 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15146
15147 match(Set dst (MoveL2D src));
15148
15149 effect(DEF dst, USE src);
15150
15151 ins_cost(INSN_COST);
15152
15153 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15154
15155 ins_encode %{
15156 __ str($src$$Register, Address(sp, $dst$$disp));
15157 %}
15158
15159 ins_pipe(istore_reg_reg);
15160
15161 %}
15162
15163 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15164
15165 match(Set dst (MoveF2I src));
15166
15167 effect(DEF dst, USE src);
15168
15169 ins_cost(INSN_COST);
15170
15171 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15172
15173 ins_encode %{
15174 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15175 %}
15176
15177 ins_pipe(fp_f2i);
15178
15179 %}
15180
15181 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15182
15183 match(Set dst (MoveI2F src));
15184
15185 effect(DEF dst, USE src);
15186
15187 ins_cost(INSN_COST);
15188
15189 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15190
15191 ins_encode %{
15192 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15193 %}
15194
15195 ins_pipe(fp_i2f);
15196
15197 %}
15198
15199 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15200
15201 match(Set dst (MoveD2L src));
15202
15203 effect(DEF dst, USE src);
15204
15205 ins_cost(INSN_COST);
15206
15207 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15208
15209 ins_encode %{
15210 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15211 %}
15212
15213 ins_pipe(fp_d2l);
15214
15215 %}
15216
15217 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15218
15219 match(Set dst (MoveL2D src));
15220
15221 effect(DEF dst, USE src);
15222
15223 ins_cost(INSN_COST);
15224
15225 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15226
15227 ins_encode %{
15228 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15229 %}
15230
15231 ins_pipe(fp_l2d);
15232
15233 %}
15234
15235 // ============================================================================
15236 // clearing of an array
15237
15238 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15239 %{
15240 match(Set dummy (ClearArray cnt base));
15241 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15242
15243 ins_cost(4 * INSN_COST);
15244 format %{ "ClearArray $cnt, $base" %}
15245
15246 ins_encode %{
15247 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15248 if (tpc == NULL) {
15249 ciEnv::current()->record_failure("CodeCache is full");
15250 return;
15251 }
15252 %}
15253
15254 ins_pipe(pipe_class_memory);
15255 %}
15256
15257 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15258 %{
15259 predicate((uint64_t)n->in(2)->get_long()
15260 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15261 match(Set dummy (ClearArray cnt base));
15262 effect(TEMP temp, USE_KILL base, KILL cr);
15263
15264 ins_cost(4 * INSN_COST);
15265 format %{ "ClearArray $cnt, $base" %}
15266
15267 ins_encode %{
15268 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15269 if (tpc == NULL) {
15270 ciEnv::current()->record_failure("CodeCache is full");
15271 return;
15272 }
15273 %}
15274
15275 ins_pipe(pipe_class_memory);
15276 %}
15277
15278 // ============================================================================
15279 // Overflow Math Instructions
15280
15281 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15282 %{
15283 match(Set cr (OverflowAddI op1 op2));
15284
15285 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15286 ins_cost(INSN_COST);
15287 ins_encode %{
15288 __ cmnw($op1$$Register, $op2$$Register);
15289 %}
15290
15291 ins_pipe(icmp_reg_reg);
15292 %}
15293
15294 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15295 %{
15296 match(Set cr (OverflowAddI op1 op2));
15297
15298 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15299 ins_cost(INSN_COST);
15300 ins_encode %{
15301 __ cmnw($op1$$Register, $op2$$constant);
15302 %}
15303
15304 ins_pipe(icmp_reg_imm);
15305 %}
15306
15307 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15308 %{
15309 match(Set cr (OverflowAddL op1 op2));
15310
15311 format %{ "cmn $op1, $op2\t# overflow check long" %}
15312 ins_cost(INSN_COST);
15313 ins_encode %{
15314 __ cmn($op1$$Register, $op2$$Register);
15315 %}
15316
15317 ins_pipe(icmp_reg_reg);
15318 %}
15319
15320 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15321 %{
15322 match(Set cr (OverflowAddL op1 op2));
15323
15324 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15325 ins_cost(INSN_COST);
15326 ins_encode %{
15327 __ adds(zr, $op1$$Register, $op2$$constant);
15328 %}
15329
15330 ins_pipe(icmp_reg_imm);
15331 %}
15332
15333 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15334 %{
15335 match(Set cr (OverflowSubI op1 op2));
15336
15337 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15338 ins_cost(INSN_COST);
15339 ins_encode %{
15340 __ cmpw($op1$$Register, $op2$$Register);
15341 %}
15342
15343 ins_pipe(icmp_reg_reg);
15344 %}
15345
15346 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15347 %{
15348 match(Set cr (OverflowSubI op1 op2));
15349
15350 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15351 ins_cost(INSN_COST);
15352 ins_encode %{
15353 __ cmpw($op1$$Register, $op2$$constant);
15354 %}
15355
15356 ins_pipe(icmp_reg_imm);
15357 %}
15358
15359 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15360 %{
15361 match(Set cr (OverflowSubL op1 op2));
15362
15363 format %{ "cmp $op1, $op2\t# overflow check long" %}
15364 ins_cost(INSN_COST);
15365 ins_encode %{
15366 __ cmp($op1$$Register, $op2$$Register);
15367 %}
15368
15369 ins_pipe(icmp_reg_reg);
15370 %}
15371
15372 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15373 %{
15374 match(Set cr (OverflowSubL op1 op2));
15375
15376 format %{ "cmp $op1, $op2\t# overflow check long" %}
15377 ins_cost(INSN_COST);
15378 ins_encode %{
15379 __ subs(zr, $op1$$Register, $op2$$constant);
15380 %}
15381
15382 ins_pipe(icmp_reg_imm);
15383 %}
15384
15385 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15386 %{
15387 match(Set cr (OverflowSubI zero op1));
15388
15389 format %{ "cmpw zr, $op1\t# overflow check int" %}
15390 ins_cost(INSN_COST);
15391 ins_encode %{
15392 __ cmpw(zr, $op1$$Register);
15393 %}
15394
15395 ins_pipe(icmp_reg_imm);
15396 %}
15397
15398 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15399 %{
15400 match(Set cr (OverflowSubL zero op1));
15401
15402 format %{ "cmp zr, $op1\t# overflow check long" %}
15403 ins_cost(INSN_COST);
15404 ins_encode %{
15405 __ cmp(zr, $op1$$Register);
15406 %}
15407
15408 ins_pipe(icmp_reg_imm);
15409 %}
15410
15411 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15412 %{
15413 match(Set cr (OverflowMulI op1 op2));
15414
15415 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15416 "cmp rscratch1, rscratch1, sxtw\n\t"
15417 "movw rscratch1, #0x80000000\n\t"
15418 "cselw rscratch1, rscratch1, zr, NE\n\t"
15419 "cmpw rscratch1, #1" %}
15420 ins_cost(5 * INSN_COST);
15421 ins_encode %{
15422 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15423 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15424 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15425 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15426 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15427 %}
15428
15429 ins_pipe(pipe_slow);
15430 %}
15431
15432 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15433 %{
15434 match(If cmp (OverflowMulI op1 op2));
15435 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15436 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15437 effect(USE labl, KILL cr);
15438
15439 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15440 "cmp rscratch1, rscratch1, sxtw\n\t"
15441 "b$cmp $labl" %}
15442 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15443 ins_encode %{
15444 Label* L = $labl$$label;
15445 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15446 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15447 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15448 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15449 %}
15450
15451 ins_pipe(pipe_serial);
15452 %}
15453
15454 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15455 %{
15456 match(Set cr (OverflowMulL op1 op2));
15457
15458 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15459 "smulh rscratch2, $op1, $op2\n\t"
15460 "cmp rscratch2, rscratch1, ASR #63\n\t"
15461 "movw rscratch1, #0x80000000\n\t"
15462 "cselw rscratch1, rscratch1, zr, NE\n\t"
15463 "cmpw rscratch1, #1" %}
15464 ins_cost(6 * INSN_COST);
15465 ins_encode %{
15466 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15467 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15468 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15469 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15470 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15471 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15472 %}
15473
15474 ins_pipe(pipe_slow);
15475 %}
15476
15477 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15478 %{
15479 match(If cmp (OverflowMulL op1 op2));
15480 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15481 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15482 effect(USE labl, KILL cr);
15483
15484 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15485 "smulh rscratch2, $op1, $op2\n\t"
15486 "cmp rscratch2, rscratch1, ASR #63\n\t"
15487 "b$cmp $labl" %}
15488 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15489 ins_encode %{
15490 Label* L = $labl$$label;
15491 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15492 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15493 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15494 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15495 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15496 %}
15497
15498 ins_pipe(pipe_serial);
15499 %}
15500
15501 // ============================================================================
15502 // Compare Instructions
15503
15504 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15505 %{
15506 match(Set cr (CmpI op1 op2));
15507
15508 effect(DEF cr, USE op1, USE op2);
15509
15510 ins_cost(INSN_COST);
15511 format %{ "cmpw $op1, $op2" %}
15512
15513 ins_encode(aarch64_enc_cmpw(op1, op2));
15514
15515 ins_pipe(icmp_reg_reg);
15516 %}
15517
15518 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15519 %{
15520 match(Set cr (CmpI op1 zero));
15521
15522 effect(DEF cr, USE op1);
15523
15524 ins_cost(INSN_COST);
15525 format %{ "cmpw $op1, 0" %}
15526
15527 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15528
15529 ins_pipe(icmp_reg_imm);
15530 %}
15531
15532 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15533 %{
15534 match(Set cr (CmpI op1 op2));
15535
15536 effect(DEF cr, USE op1);
15537
15538 ins_cost(INSN_COST);
15539 format %{ "cmpw $op1, $op2" %}
15540
15541 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15542
15543 ins_pipe(icmp_reg_imm);
15544 %}
15545
15546 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15547 %{
15548 match(Set cr (CmpI op1 op2));
15549
15550 effect(DEF cr, USE op1);
15551
15552 ins_cost(INSN_COST * 2);
15553 format %{ "cmpw $op1, $op2" %}
15554
15555 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15556
15557 ins_pipe(icmp_reg_imm);
15558 %}
15559
15560 // Unsigned compare Instructions; really, same as signed compare
15561 // except it should only be used to feed an If or a CMovI which takes a
15562 // cmpOpU.
15563
15564 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15565 %{
15566 match(Set cr (CmpU op1 op2));
15567
15568 effect(DEF cr, USE op1, USE op2);
15569
15570 ins_cost(INSN_COST);
15571 format %{ "cmpw $op1, $op2\t# unsigned" %}
15572
15573 ins_encode(aarch64_enc_cmpw(op1, op2));
15574
15575 ins_pipe(icmp_reg_reg);
15576 %}
15577
15578 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15579 %{
15580 match(Set cr (CmpU op1 zero));
15581
15582 effect(DEF cr, USE op1);
15583
15584 ins_cost(INSN_COST);
15585 format %{ "cmpw $op1, #0\t# unsigned" %}
15586
15587 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15588
15589 ins_pipe(icmp_reg_imm);
15590 %}
15591
15592 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15593 %{
15594 match(Set cr (CmpU op1 op2));
15595
15596 effect(DEF cr, USE op1);
15597
15598 ins_cost(INSN_COST);
15599 format %{ "cmpw $op1, $op2\t# unsigned" %}
15600
15601 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15602
15603 ins_pipe(icmp_reg_imm);
15604 %}
15605
15606 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15607 %{
15608 match(Set cr (CmpU op1 op2));
15609
15610 effect(DEF cr, USE op1);
15611
15612 ins_cost(INSN_COST * 2);
15613 format %{ "cmpw $op1, $op2\t# unsigned" %}
15614
15615 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15616
15617 ins_pipe(icmp_reg_imm);
15618 %}
15619
15620 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15621 %{
15622 match(Set cr (CmpL op1 op2));
15623
15624 effect(DEF cr, USE op1, USE op2);
15625
15626 ins_cost(INSN_COST);
15627 format %{ "cmp $op1, $op2" %}
15628
15629 ins_encode(aarch64_enc_cmp(op1, op2));
15630
15631 ins_pipe(icmp_reg_reg);
15632 %}
15633
15634 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15635 %{
15636 match(Set cr (CmpL op1 zero));
15637
15638 effect(DEF cr, USE op1);
15639
15640 ins_cost(INSN_COST);
15641 format %{ "tst $op1" %}
15642
15643 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15644
15645 ins_pipe(icmp_reg_imm);
15646 %}
15647
15648 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15649 %{
15650 match(Set cr (CmpL op1 op2));
15651
15652 effect(DEF cr, USE op1);
15653
15654 ins_cost(INSN_COST);
15655 format %{ "cmp $op1, $op2" %}
15656
15657 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15658
15659 ins_pipe(icmp_reg_imm);
15660 %}
15661
15662 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15663 %{
15664 match(Set cr (CmpL op1 op2));
15665
15666 effect(DEF cr, USE op1);
15667
15668 ins_cost(INSN_COST * 2);
15669 format %{ "cmp $op1, $op2" %}
15670
15671 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15672
15673 ins_pipe(icmp_reg_imm);
15674 %}
15675
15676 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15677 %{
15678 match(Set cr (CmpUL op1 op2));
15679
15680 effect(DEF cr, USE op1, USE op2);
15681
15682 ins_cost(INSN_COST);
15683 format %{ "cmp $op1, $op2" %}
15684
15685 ins_encode(aarch64_enc_cmp(op1, op2));
15686
15687 ins_pipe(icmp_reg_reg);
15688 %}
15689
15690 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15691 %{
15692 match(Set cr (CmpUL op1 zero));
15693
15694 effect(DEF cr, USE op1);
15695
15696 ins_cost(INSN_COST);
15697 format %{ "tst $op1" %}
15698
15699 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15700
15701 ins_pipe(icmp_reg_imm);
15702 %}
15703
15704 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15705 %{
15706 match(Set cr (CmpUL op1 op2));
15707
15708 effect(DEF cr, USE op1);
15709
15710 ins_cost(INSN_COST);
15711 format %{ "cmp $op1, $op2" %}
15712
15713 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15714
15715 ins_pipe(icmp_reg_imm);
15716 %}
15717
15718 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15719 %{
15720 match(Set cr (CmpUL op1 op2));
15721
15722 effect(DEF cr, USE op1);
15723
15724 ins_cost(INSN_COST * 2);
15725 format %{ "cmp $op1, $op2" %}
15726
15727 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15728
15729 ins_pipe(icmp_reg_imm);
15730 %}
15731
15732 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15733 %{
15734 match(Set cr (CmpP op1 op2));
15735
15736 effect(DEF cr, USE op1, USE op2);
15737
15738 ins_cost(INSN_COST);
15739 format %{ "cmp $op1, $op2\t // ptr" %}
15740
15741 ins_encode(aarch64_enc_cmpp(op1, op2));
15742
15743 ins_pipe(icmp_reg_reg);
15744 %}
15745
15746 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15747 %{
15748 match(Set cr (CmpN op1 op2));
15749
15750 effect(DEF cr, USE op1, USE op2);
15751
15752 ins_cost(INSN_COST);
15753 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15754
15755 ins_encode(aarch64_enc_cmpn(op1, op2));
15756
15757 ins_pipe(icmp_reg_reg);
15758 %}
15759
15760 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15761 %{
15762 match(Set cr (CmpP op1 zero));
15763
15764 effect(DEF cr, USE op1, USE zero);
15765
15766 ins_cost(INSN_COST);
15767 format %{ "cmp $op1, 0\t // ptr" %}
15768
15769 ins_encode(aarch64_enc_testp(op1));
15770
15771 ins_pipe(icmp_reg_imm);
15772 %}
15773
15774 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15775 %{
15776 match(Set cr (CmpN op1 zero));
15777
15778 effect(DEF cr, USE op1, USE zero);
15779
15780 ins_cost(INSN_COST);
15781 format %{ "cmp $op1, 0\t // compressed ptr" %}
15782
15783 ins_encode(aarch64_enc_testn(op1));
15784
15785 ins_pipe(icmp_reg_imm);
15786 %}
15787
15788 // FP comparisons
15789 //
15790 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15791 // using normal cmpOp. See declaration of rFlagsReg for details.
15792
15793 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15794 %{
15795 match(Set cr (CmpF src1 src2));
15796
15797 ins_cost(3 * INSN_COST);
15798 format %{ "fcmps $src1, $src2" %}
15799
15800 ins_encode %{
15801 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15802 %}
15803
15804 ins_pipe(pipe_class_compare);
15805 %}
15806
15807 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15808 %{
15809 match(Set cr (CmpF src1 src2));
15810
15811 ins_cost(3 * INSN_COST);
15812 format %{ "fcmps $src1, 0.0" %}
15813
15814 ins_encode %{
15815 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15816 %}
15817
15818 ins_pipe(pipe_class_compare);
15819 %}
15820 // FROM HERE
15821
15822 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15823 %{
15824 match(Set cr (CmpD src1 src2));
15825
15826 ins_cost(3 * INSN_COST);
15827 format %{ "fcmpd $src1, $src2" %}
15828
15829 ins_encode %{
15830 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15831 %}
15832
15833 ins_pipe(pipe_class_compare);
15834 %}
15835
15836 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15837 %{
15838 match(Set cr (CmpD src1 src2));
15839
15840 ins_cost(3 * INSN_COST);
15841 format %{ "fcmpd $src1, 0.0" %}
15842
15843 ins_encode %{
15844 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15845 %}
15846
15847 ins_pipe(pipe_class_compare);
15848 %}
15849
15850 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15851 %{
15852 match(Set dst (CmpF3 src1 src2));
15853 effect(KILL cr);
15854
15855 ins_cost(5 * INSN_COST);
15856 format %{ "fcmps $src1, $src2\n\t"
15857 "csinvw($dst, zr, zr, eq\n\t"
15858 "csnegw($dst, $dst, $dst, lt)"
15859 %}
15860
15861 ins_encode %{
15862 Label done;
15863 FloatRegister s1 = as_FloatRegister($src1$$reg);
15864 FloatRegister s2 = as_FloatRegister($src2$$reg);
15865 Register d = as_Register($dst$$reg);
15866 __ fcmps(s1, s2);
15867 // installs 0 if EQ else -1
15868 __ csinvw(d, zr, zr, Assembler::EQ);
15869 // keeps -1 if less or unordered else installs 1
15870 __ csnegw(d, d, d, Assembler::LT);
15871 __ bind(done);
15872 %}
15873
15874 ins_pipe(pipe_class_default);
15875
15876 %}
15877
15878 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15879 %{
15880 match(Set dst (CmpD3 src1 src2));
15881 effect(KILL cr);
15882
15883 ins_cost(5 * INSN_COST);
15884 format %{ "fcmpd $src1, $src2\n\t"
15885 "csinvw($dst, zr, zr, eq\n\t"
15886 "csnegw($dst, $dst, $dst, lt)"
15887 %}
15888
15889 ins_encode %{
15890 Label done;
15891 FloatRegister s1 = as_FloatRegister($src1$$reg);
15892 FloatRegister s2 = as_FloatRegister($src2$$reg);
15893 Register d = as_Register($dst$$reg);
15894 __ fcmpd(s1, s2);
15895 // installs 0 if EQ else -1
15896 __ csinvw(d, zr, zr, Assembler::EQ);
15897 // keeps -1 if less or unordered else installs 1
15898 __ csnegw(d, d, d, Assembler::LT);
15899 __ bind(done);
15900 %}
15901 ins_pipe(pipe_class_default);
15902
15903 %}
15904
15905 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15906 %{
15907 match(Set dst (CmpF3 src1 zero));
15908 effect(KILL cr);
15909
15910 ins_cost(5 * INSN_COST);
15911 format %{ "fcmps $src1, 0.0\n\t"
15912 "csinvw($dst, zr, zr, eq\n\t"
15913 "csnegw($dst, $dst, $dst, lt)"
15914 %}
15915
15916 ins_encode %{
15917 Label done;
15918 FloatRegister s1 = as_FloatRegister($src1$$reg);
15919 Register d = as_Register($dst$$reg);
15920 __ fcmps(s1, 0.0);
15921 // installs 0 if EQ else -1
15922 __ csinvw(d, zr, zr, Assembler::EQ);
15923 // keeps -1 if less or unordered else installs 1
15924 __ csnegw(d, d, d, Assembler::LT);
15925 __ bind(done);
15926 %}
15927
15928 ins_pipe(pipe_class_default);
15929
15930 %}
15931
15932 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15933 %{
15934 match(Set dst (CmpD3 src1 zero));
15935 effect(KILL cr);
15936
15937 ins_cost(5 * INSN_COST);
15938 format %{ "fcmpd $src1, 0.0\n\t"
15939 "csinvw($dst, zr, zr, eq\n\t"
15940 "csnegw($dst, $dst, $dst, lt)"
15941 %}
15942
15943 ins_encode %{
15944 Label done;
15945 FloatRegister s1 = as_FloatRegister($src1$$reg);
15946 Register d = as_Register($dst$$reg);
15947 __ fcmpd(s1, 0.0);
15948 // installs 0 if EQ else -1
15949 __ csinvw(d, zr, zr, Assembler::EQ);
15950 // keeps -1 if less or unordered else installs 1
15951 __ csnegw(d, d, d, Assembler::LT);
15952 __ bind(done);
15953 %}
15954 ins_pipe(pipe_class_default);
15955
15956 %}
15957
15958 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15959 %{
15960 match(Set dst (CmpLTMask p q));
15961 effect(KILL cr);
15962
15963 ins_cost(3 * INSN_COST);
15964
15965 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15966 "csetw $dst, lt\n\t"
15967 "subw $dst, zr, $dst"
15968 %}
15969
15970 ins_encode %{
15971 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15972 __ csetw(as_Register($dst$$reg), Assembler::LT);
15973 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15974 %}
15975
15976 ins_pipe(ialu_reg_reg);
15977 %}
15978
15979 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15980 %{
15981 match(Set dst (CmpLTMask src zero));
15982 effect(KILL cr);
15983
15984 ins_cost(INSN_COST);
15985
15986 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15987
15988 ins_encode %{
15989 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15990 %}
15991
15992 ins_pipe(ialu_reg_shift);
15993 %}
15994
15995 // ============================================================================
15996 // Max and Min
15997
15998 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15999
16000 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16001 %{
16002 effect(DEF cr, USE src);
16003 ins_cost(INSN_COST);
16004 format %{ "cmpw $src, 0" %}
16005
16006 ins_encode %{
16007 __ cmpw($src$$Register, 0);
16008 %}
16009 ins_pipe(icmp_reg_imm);
16010 %}
16011
16012 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16013 %{
16014 match(Set dst (MinI src1 src2));
16015 ins_cost(INSN_COST * 3);
16016
16017 expand %{
16018 rFlagsReg cr;
16019 compI_reg_reg(cr, src1, src2);
16020 cmovI_reg_reg_lt(dst, src1, src2, cr);
16021 %}
16022 %}
16023
16024 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16025 %{
16026 match(Set dst (MaxI src1 src2));
16027 ins_cost(INSN_COST * 3);
16028
16029 expand %{
16030 rFlagsReg cr;
16031 compI_reg_reg(cr, src1, src2);
16032 cmovI_reg_reg_gt(dst, src1, src2, cr);
16033 %}
16034 %}
16035
16036
16037 // ============================================================================
16038 // Branch Instructions
16039
16040 // Direct Branch.
16041 instruct branch(label lbl)
16042 %{
16043 match(Goto);
16044
16045 effect(USE lbl);
16046
16047 ins_cost(BRANCH_COST);
16048 format %{ "b $lbl" %}
16049
16050 ins_encode(aarch64_enc_b(lbl));
16051
16052 ins_pipe(pipe_branch);
16053 %}
16054
16055 // Conditional Near Branch
16056 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16057 %{
16058 // Same match rule as `branchConFar'.
16059 match(If cmp cr);
16060
16061 effect(USE lbl);
16062
16063 ins_cost(BRANCH_COST);
16064 // If set to 1 this indicates that the current instruction is a
16065 // short variant of a long branch. This avoids using this
16066 // instruction in first-pass matching. It will then only be used in
16067 // the `Shorten_branches' pass.
16068 // ins_short_branch(1);
16069 format %{ "b$cmp $lbl" %}
16070
16071 ins_encode(aarch64_enc_br_con(cmp, lbl));
16072
16073 ins_pipe(pipe_branch_cond);
16074 %}
16075
16076 // Conditional Near Branch Unsigned
16077 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16078 %{
16079 // Same match rule as `branchConFar'.
16080 match(If cmp cr);
16081
16082 effect(USE lbl);
16083
16084 ins_cost(BRANCH_COST);
16085 // If set to 1 this indicates that the current instruction is a
16086 // short variant of a long branch. This avoids using this
16087 // instruction in first-pass matching. It will then only be used in
16088 // the `Shorten_branches' pass.
16089 // ins_short_branch(1);
16090 format %{ "b$cmp $lbl\t# unsigned" %}
16091
16092 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16093
16094 ins_pipe(pipe_branch_cond);
16095 %}
16096
16097 // Make use of CBZ and CBNZ. These instructions, as well as being
16098 // shorter than (cmp; branch), have the additional benefit of not
16099 // killing the flags.
16100
16101 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16102 match(If cmp (CmpI op1 op2));
16103 effect(USE labl);
16104
16105 ins_cost(BRANCH_COST);
16106 format %{ "cbw$cmp $op1, $labl" %}
16107 ins_encode %{
16108 Label* L = $labl$$label;
16109 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16110 if (cond == Assembler::EQ)
16111 __ cbzw($op1$$Register, *L);
16112 else
16113 __ cbnzw($op1$$Register, *L);
16114 %}
16115 ins_pipe(pipe_cmp_branch);
16116 %}
16117
16118 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16119 match(If cmp (CmpL op1 op2));
16120 effect(USE labl);
16121
16122 ins_cost(BRANCH_COST);
16123 format %{ "cb$cmp $op1, $labl" %}
16124 ins_encode %{
16125 Label* L = $labl$$label;
16126 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16127 if (cond == Assembler::EQ)
16128 __ cbz($op1$$Register, *L);
16129 else
16130 __ cbnz($op1$$Register, *L);
16131 %}
16132 ins_pipe(pipe_cmp_branch);
16133 %}
16134
16135 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16136 match(If cmp (CmpP op1 op2));
16137 effect(USE labl);
16138
16139 ins_cost(BRANCH_COST);
16140 format %{ "cb$cmp $op1, $labl" %}
16141 ins_encode %{
16142 Label* L = $labl$$label;
16143 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16144 if (cond == Assembler::EQ)
16145 __ cbz($op1$$Register, *L);
16146 else
16147 __ cbnz($op1$$Register, *L);
16148 %}
16149 ins_pipe(pipe_cmp_branch);
16150 %}
16151
16152 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16153 match(If cmp (CmpN op1 op2));
16154 effect(USE labl);
16155
16156 ins_cost(BRANCH_COST);
16157 format %{ "cbw$cmp $op1, $labl" %}
16158 ins_encode %{
16159 Label* L = $labl$$label;
16160 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16161 if (cond == Assembler::EQ)
16162 __ cbzw($op1$$Register, *L);
16163 else
16164 __ cbnzw($op1$$Register, *L);
16165 %}
16166 ins_pipe(pipe_cmp_branch);
16167 %}
16168
16169 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16170 match(If cmp (CmpP (DecodeN oop) zero));
16171 effect(USE labl);
16172
16173 ins_cost(BRANCH_COST);
16174 format %{ "cb$cmp $oop, $labl" %}
16175 ins_encode %{
16176 Label* L = $labl$$label;
16177 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16178 if (cond == Assembler::EQ)
16179 __ cbzw($oop$$Register, *L);
16180 else
16181 __ cbnzw($oop$$Register, *L);
16182 %}
16183 ins_pipe(pipe_cmp_branch);
16184 %}
16185
16186 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16187 match(If cmp (CmpU op1 op2));
16188 effect(USE labl);
16189
16190 ins_cost(BRANCH_COST);
16191 format %{ "cbw$cmp $op1, $labl" %}
16192 ins_encode %{
16193 Label* L = $labl$$label;
16194 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16195 if (cond == Assembler::EQ || cond == Assembler::LS)
16196 __ cbzw($op1$$Register, *L);
16197 else
16198 __ cbnzw($op1$$Register, *L);
16199 %}
16200 ins_pipe(pipe_cmp_branch);
16201 %}
16202
16203 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16204 match(If cmp (CmpUL op1 op2));
16205 effect(USE labl);
16206
16207 ins_cost(BRANCH_COST);
16208 format %{ "cb$cmp $op1, $labl" %}
16209 ins_encode %{
16210 Label* L = $labl$$label;
16211 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16212 if (cond == Assembler::EQ || cond == Assembler::LS)
16213 __ cbz($op1$$Register, *L);
16214 else
16215 __ cbnz($op1$$Register, *L);
16216 %}
16217 ins_pipe(pipe_cmp_branch);
16218 %}
16219
16220 // Test bit and Branch
16221
16222 // Patterns for short (< 32KiB) variants
16223 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16224 match(If cmp (CmpL op1 op2));
16225 effect(USE labl);
16226
16227 ins_cost(BRANCH_COST);
16228 format %{ "cb$cmp $op1, $labl # long" %}
16229 ins_encode %{
16230 Label* L = $labl$$label;
16231 Assembler::Condition cond =
16232 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16233 __ tbr(cond, $op1$$Register, 63, *L);
16234 %}
16235 ins_pipe(pipe_cmp_branch);
16236 ins_short_branch(1);
16237 %}
16238
16239 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16240 match(If cmp (CmpI op1 op2));
16241 effect(USE labl);
16242
16243 ins_cost(BRANCH_COST);
16244 format %{ "cb$cmp $op1, $labl # int" %}
16245 ins_encode %{
16246 Label* L = $labl$$label;
16247 Assembler::Condition cond =
16248 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16249 __ tbr(cond, $op1$$Register, 31, *L);
16250 %}
16251 ins_pipe(pipe_cmp_branch);
16252 ins_short_branch(1);
16253 %}
16254
16255 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16256 match(If cmp (CmpL (AndL op1 op2) op3));
16257 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16258 effect(USE labl);
16259
16260 ins_cost(BRANCH_COST);
16261 format %{ "tb$cmp $op1, $op2, $labl" %}
16262 ins_encode %{
16263 Label* L = $labl$$label;
16264 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16265 int bit = exact_log2_long($op2$$constant);
16266 __ tbr(cond, $op1$$Register, bit, *L);
16267 %}
16268 ins_pipe(pipe_cmp_branch);
16269 ins_short_branch(1);
16270 %}
16271
16272 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16273 match(If cmp (CmpI (AndI op1 op2) op3));
16274 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16275 effect(USE labl);
16276
16277 ins_cost(BRANCH_COST);
16278 format %{ "tb$cmp $op1, $op2, $labl" %}
16279 ins_encode %{
16280 Label* L = $labl$$label;
16281 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16282 int bit = exact_log2((juint)$op2$$constant);
16283 __ tbr(cond, $op1$$Register, bit, *L);
16284 %}
16285 ins_pipe(pipe_cmp_branch);
16286 ins_short_branch(1);
16287 %}
16288
16289 // And far variants
16290 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16291 match(If cmp (CmpL op1 op2));
16292 effect(USE labl);
16293
16294 ins_cost(BRANCH_COST);
16295 format %{ "cb$cmp $op1, $labl # long" %}
16296 ins_encode %{
16297 Label* L = $labl$$label;
16298 Assembler::Condition cond =
16299 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16300 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16301 %}
16302 ins_pipe(pipe_cmp_branch);
16303 %}
16304
16305 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16306 match(If cmp (CmpI op1 op2));
16307 effect(USE labl);
16308
16309 ins_cost(BRANCH_COST);
16310 format %{ "cb$cmp $op1, $labl # int" %}
16311 ins_encode %{
16312 Label* L = $labl$$label;
16313 Assembler::Condition cond =
16314 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16315 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16316 %}
16317 ins_pipe(pipe_cmp_branch);
16318 %}
16319
16320 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16321 match(If cmp (CmpL (AndL op1 op2) op3));
16322 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16323 effect(USE labl);
16324
16325 ins_cost(BRANCH_COST);
16326 format %{ "tb$cmp $op1, $op2, $labl" %}
16327 ins_encode %{
16328 Label* L = $labl$$label;
16329 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16330 int bit = exact_log2_long($op2$$constant);
16331 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16332 %}
16333 ins_pipe(pipe_cmp_branch);
16334 %}
16335
16336 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16337 match(If cmp (CmpI (AndI op1 op2) op3));
16338 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16339 effect(USE labl);
16340
16341 ins_cost(BRANCH_COST);
16342 format %{ "tb$cmp $op1, $op2, $labl" %}
16343 ins_encode %{
16344 Label* L = $labl$$label;
16345 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16346 int bit = exact_log2((juint)$op2$$constant);
16347 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16348 %}
16349 ins_pipe(pipe_cmp_branch);
16350 %}
16351
16352 // Test bits
16353
16354 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16355 match(Set cr (CmpL (AndL op1 op2) op3));
16356 predicate(Assembler::operand_valid_for_logical_immediate
16357 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16358
16359 ins_cost(INSN_COST);
16360 format %{ "tst $op1, $op2 # long" %}
16361 ins_encode %{
16362 __ tst($op1$$Register, $op2$$constant);
16363 %}
16364 ins_pipe(ialu_reg_reg);
16365 %}
16366
16367 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16368 match(Set cr (CmpI (AndI op1 op2) op3));
16369 predicate(Assembler::operand_valid_for_logical_immediate
16370 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16371
16372 ins_cost(INSN_COST);
16373 format %{ "tst $op1, $op2 # int" %}
16374 ins_encode %{
16375 __ tstw($op1$$Register, $op2$$constant);
16376 %}
16377 ins_pipe(ialu_reg_reg);
16378 %}
16379
16380 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16381 match(Set cr (CmpL (AndL op1 op2) op3));
16382
16383 ins_cost(INSN_COST);
16384 format %{ "tst $op1, $op2 # long" %}
16385 ins_encode %{
16386 __ tst($op1$$Register, $op2$$Register);
16387 %}
16388 ins_pipe(ialu_reg_reg);
16389 %}
16390
16391 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16392 match(Set cr (CmpI (AndI op1 op2) op3));
16393
16394 ins_cost(INSN_COST);
16395 format %{ "tstw $op1, $op2 # int" %}
16396 ins_encode %{
16397 __ tstw($op1$$Register, $op2$$Register);
16398 %}
16399 ins_pipe(ialu_reg_reg);
16400 %}
16401
16402
16403 // Conditional Far Branch
16404 // Conditional Far Branch Unsigned
16405 // TODO: fixme
16406
16407 // counted loop end branch near
16408 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16409 %{
16410 match(CountedLoopEnd cmp cr);
16411
16412 effect(USE lbl);
16413
16414 ins_cost(BRANCH_COST);
16415 // short variant.
16416 // ins_short_branch(1);
16417 format %{ "b$cmp $lbl \t// counted loop end" %}
16418
16419 ins_encode(aarch64_enc_br_con(cmp, lbl));
16420
16421 ins_pipe(pipe_branch);
16422 %}
16423
16424 // counted loop end branch far
16425 // TODO: fixme
16426
16427 // ============================================================================
16428 // inlined locking and unlocking
16429
16430 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16431 %{
16432 predicate(LockingMode != LM_LIGHTWEIGHT);
16433 match(Set cr (FastLock object box));
16434 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16435
16436 ins_cost(5 * INSN_COST);
16437 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16438
16439 ins_encode %{
16440 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16441 %}
16442
16443 ins_pipe(pipe_serial);
16444 %}
16445
16446 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16447 %{
16448 predicate(LockingMode != LM_LIGHTWEIGHT);
16449 match(Set cr (FastUnlock object box));
16450 effect(TEMP tmp, TEMP tmp2);
16451
16452 ins_cost(5 * INSN_COST);
16453 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16454
16455 ins_encode %{
16456 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16457 %}
16458
16459 ins_pipe(pipe_serial);
16460 %}
16461
16462 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16463 %{
16464 predicate(LockingMode == LM_LIGHTWEIGHT);
16465 match(Set cr (FastLock object box));
16466 effect(TEMP tmp, TEMP tmp2);
16467
16468 ins_cost(5 * INSN_COST);
16469 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16470
16471 ins_encode %{
16472 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16473 %}
16474
16475 ins_pipe(pipe_serial);
16476 %}
16477
16478 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16479 %{
16480 predicate(LockingMode == LM_LIGHTWEIGHT);
16481 match(Set cr (FastUnlock object box));
16482 effect(TEMP tmp, TEMP tmp2);
16483
16484 ins_cost(5 * INSN_COST);
16485 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16486
16487 ins_encode %{
16488 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16489 %}
16490
16491 ins_pipe(pipe_serial);
16492 %}
16493
16494 // ============================================================================
16495 // Safepoint Instructions
16496
16497 // TODO
16498 // provide a near and far version of this code
16499
16500 instruct safePoint(rFlagsReg cr, iRegP poll)
16501 %{
16502 match(SafePoint poll);
16503 effect(KILL cr);
16504
16505 format %{
16506 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16507 %}
16508 ins_encode %{
16509 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16510 %}
16511 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16512 %}
16513
16514
16515 // ============================================================================
16516 // Procedure Call/Return Instructions
16517
16518 // Call Java Static Instruction
16519
16520 instruct CallStaticJavaDirect(method meth)
16521 %{
16522 match(CallStaticJava);
16523
16524 effect(USE meth);
16525
16526 ins_cost(CALL_COST);
16527
16528 format %{ "call,static $meth \t// ==> " %}
16529
16530 ins_encode(aarch64_enc_java_static_call(meth),
16531 aarch64_enc_call_epilog);
16532
16533 ins_pipe(pipe_class_call);
16534 %}
16535
16536 // TO HERE
16537
16538 // Call Java Dynamic Instruction
16539 instruct CallDynamicJavaDirect(method meth)
16540 %{
16541 match(CallDynamicJava);
16542
16543 effect(USE meth);
16544
16545 ins_cost(CALL_COST);
16546
16547 format %{ "CALL,dynamic $meth \t// ==> " %}
16548
16549 ins_encode(aarch64_enc_java_dynamic_call(meth),
16550 aarch64_enc_call_epilog);
16551
16552 ins_pipe(pipe_class_call);
16553 %}
16554
16555 // Call Runtime Instruction
16556
16557 instruct CallRuntimeDirect(method meth)
16558 %{
16559 match(CallRuntime);
16560
16561 effect(USE meth);
16562
16563 ins_cost(CALL_COST);
16564
16565 format %{ "CALL, runtime $meth" %}
16566
16567 ins_encode( aarch64_enc_java_to_runtime(meth) );
16568
16569 ins_pipe(pipe_class_call);
16570 %}
16571
16572 // Call Runtime Instruction
16573
16574 instruct CallLeafDirect(method meth)
16575 %{
16576 match(CallLeaf);
16577
16578 effect(USE meth);
16579
16580 ins_cost(CALL_COST);
16581
16582 format %{ "CALL, runtime leaf $meth" %}
16583
16584 ins_encode( aarch64_enc_java_to_runtime(meth) );
16585
16586 ins_pipe(pipe_class_call);
16587 %}
16588
16589 // Call Runtime Instruction
16590
16591 instruct CallLeafNoFPDirect(method meth)
16592 %{
16593 match(CallLeafNoFP);
16594
16595 effect(USE meth);
16596
16597 ins_cost(CALL_COST);
16598
16599 format %{ "CALL, runtime leaf nofp $meth" %}
16600
16601 ins_encode( aarch64_enc_java_to_runtime(meth) );
16602
16603 ins_pipe(pipe_class_call);
16604 %}
16605
16606 // Tail Call; Jump from runtime stub to Java code.
16607 // Also known as an 'interprocedural jump'.
16608 // Target of jump will eventually return to caller.
16609 // TailJump below removes the return address.
16610 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16611 %{
16612 match(TailCall jump_target method_ptr);
16613
16614 ins_cost(CALL_COST);
16615
16616 format %{ "br $jump_target\t# $method_ptr holds method" %}
16617
16618 ins_encode(aarch64_enc_tail_call(jump_target));
16619
16620 ins_pipe(pipe_class_call);
16621 %}
16622
16623 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16624 %{
16625 match(TailJump jump_target ex_oop);
16626
16627 ins_cost(CALL_COST);
16628
16629 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16630
16631 ins_encode(aarch64_enc_tail_jmp(jump_target));
16632
16633 ins_pipe(pipe_class_call);
16634 %}
16635
16636 // Create exception oop: created by stack-crawling runtime code.
16637 // Created exception is now available to this handler, and is setup
16638 // just prior to jumping to this handler. No code emitted.
16639 // TODO check
16640 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16641 instruct CreateException(iRegP_R0 ex_oop)
16642 %{
16643 match(Set ex_oop (CreateEx));
16644
16645 format %{ " -- \t// exception oop; no code emitted" %}
16646
16647 size(0);
16648
16649 ins_encode( /*empty*/ );
16650
16651 ins_pipe(pipe_class_empty);
16652 %}
16653
16654 // Rethrow exception: The exception oop will come in the first
16655 // argument position. Then JUMP (not call) to the rethrow stub code.
16656 instruct RethrowException() %{
16657 match(Rethrow);
16658 ins_cost(CALL_COST);
16659
16660 format %{ "b rethrow_stub" %}
16661
16662 ins_encode( aarch64_enc_rethrow() );
16663
16664 ins_pipe(pipe_class_call);
16665 %}
16666
16667
16668 // Return Instruction
16669 // epilog node loads ret address into lr as part of frame pop
16670 instruct Ret()
16671 %{
16672 match(Return);
16673
16674 format %{ "ret\t// return register" %}
16675
16676 ins_encode( aarch64_enc_ret() );
16677
16678 ins_pipe(pipe_branch);
16679 %}
16680
16681 // Die now.
16682 instruct ShouldNotReachHere() %{
16683 match(Halt);
16684
16685 ins_cost(CALL_COST);
16686 format %{ "ShouldNotReachHere" %}
16687
16688 ins_encode %{
16689 if (is_reachable()) {
16690 __ stop(_halt_reason);
16691 }
16692 %}
16693
16694 ins_pipe(pipe_class_default);
16695 %}
16696
16697 // ============================================================================
16698 // Partial Subtype Check
16699 //
16700 // superklass array for an instance of the superklass. Set a hidden
16701 // internal cache on a hit (cache is checked with exposed code in
16702 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16703 // encoding ALSO sets flags.
16704
16705 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16706 %{
16707 match(Set result (PartialSubtypeCheck sub super));
16708 effect(KILL cr, KILL temp);
16709
16710 ins_cost(1100); // slightly larger than the next version
16711 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16712
16713 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16714
16715 opcode(0x1); // Force zero of result reg on hit
16716
16717 ins_pipe(pipe_class_memory);
16718 %}
16719
16720 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16721 %{
16722 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16723 effect(KILL temp, KILL result);
16724
16725 ins_cost(1100); // slightly larger than the next version
16726 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16727
16728 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16729
16730 opcode(0x0); // Don't zero result reg on hit
16731
16732 ins_pipe(pipe_class_memory);
16733 %}
16734
16735 // Intrisics for String.compareTo()
16736
16737 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16738 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16739 %{
16740 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16741 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16742 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16743
16744 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16745 ins_encode %{
16746 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16747 __ string_compare($str1$$Register, $str2$$Register,
16748 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16749 $tmp1$$Register, $tmp2$$Register,
16750 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16751 %}
16752 ins_pipe(pipe_class_memory);
16753 %}
16754
16755 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16756 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16757 %{
16758 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16759 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16760 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16761
16762 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16763 ins_encode %{
16764 __ string_compare($str1$$Register, $str2$$Register,
16765 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16766 $tmp1$$Register, $tmp2$$Register,
16767 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16768 %}
16769 ins_pipe(pipe_class_memory);
16770 %}
16771
16772 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16773 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16774 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16775 %{
16776 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16777 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16778 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16779 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16780
16781 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16782 ins_encode %{
16783 __ string_compare($str1$$Register, $str2$$Register,
16784 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16785 $tmp1$$Register, $tmp2$$Register,
16786 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16787 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16788 %}
16789 ins_pipe(pipe_class_memory);
16790 %}
16791
16792 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16793 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16794 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16795 %{
16796 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16797 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16798 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16799 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16800
16801 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16802 ins_encode %{
16803 __ string_compare($str1$$Register, $str2$$Register,
16804 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16805 $tmp1$$Register, $tmp2$$Register,
16806 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16807 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16808 %}
16809 ins_pipe(pipe_class_memory);
16810 %}
16811
16812 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16813 // these string_compare variants as NEON register type for convenience so that the prototype of
16814 // string_compare can be shared with all variants.
16815
16816 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16817 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16818 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16819 pRegGov_P1 pgtmp2, rFlagsReg cr)
16820 %{
16821 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16822 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16823 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16824 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16825
16826 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16827 ins_encode %{
16828 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16829 __ string_compare($str1$$Register, $str2$$Register,
16830 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16831 $tmp1$$Register, $tmp2$$Register,
16832 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16833 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16834 StrIntrinsicNode::LL);
16835 %}
16836 ins_pipe(pipe_class_memory);
16837 %}
16838
16839 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16840 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16841 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16842 pRegGov_P1 pgtmp2, rFlagsReg cr)
16843 %{
16844 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16845 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16846 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16847 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16848
16849 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16850 ins_encode %{
16851 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16852 __ string_compare($str1$$Register, $str2$$Register,
16853 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16854 $tmp1$$Register, $tmp2$$Register,
16855 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16856 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16857 StrIntrinsicNode::LU);
16858 %}
16859 ins_pipe(pipe_class_memory);
16860 %}
16861
16862 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16863 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16864 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16865 pRegGov_P1 pgtmp2, rFlagsReg cr)
16866 %{
16867 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16868 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16869 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16870 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16871
16872 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16873 ins_encode %{
16874 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16875 __ string_compare($str1$$Register, $str2$$Register,
16876 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16877 $tmp1$$Register, $tmp2$$Register,
16878 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16879 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16880 StrIntrinsicNode::UL);
16881 %}
16882 ins_pipe(pipe_class_memory);
16883 %}
16884
16885 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16886 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16887 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16888 pRegGov_P1 pgtmp2, rFlagsReg cr)
16889 %{
16890 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16891 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16892 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16893 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16894
16895 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16896 ins_encode %{
16897 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16898 __ string_compare($str1$$Register, $str2$$Register,
16899 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16900 $tmp1$$Register, $tmp2$$Register,
16901 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16902 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16903 StrIntrinsicNode::UU);
16904 %}
16905 ins_pipe(pipe_class_memory);
16906 %}
16907
16908 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16909 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16910 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16911 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16912 %{
16913 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16914 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16915 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16916 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16917 TEMP vtmp0, TEMP vtmp1, KILL cr);
16918 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16919 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16920
16921 ins_encode %{
16922 __ string_indexof($str1$$Register, $str2$$Register,
16923 $cnt1$$Register, $cnt2$$Register,
16924 $tmp1$$Register, $tmp2$$Register,
16925 $tmp3$$Register, $tmp4$$Register,
16926 $tmp5$$Register, $tmp6$$Register,
16927 -1, $result$$Register, StrIntrinsicNode::UU);
16928 %}
16929 ins_pipe(pipe_class_memory);
16930 %}
16931
16932 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16933 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16934 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16935 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16936 %{
16937 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16938 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16939 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16940 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16941 TEMP vtmp0, TEMP vtmp1, KILL cr);
16942 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16943 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16944
16945 ins_encode %{
16946 __ string_indexof($str1$$Register, $str2$$Register,
16947 $cnt1$$Register, $cnt2$$Register,
16948 $tmp1$$Register, $tmp2$$Register,
16949 $tmp3$$Register, $tmp4$$Register,
16950 $tmp5$$Register, $tmp6$$Register,
16951 -1, $result$$Register, StrIntrinsicNode::LL);
16952 %}
16953 ins_pipe(pipe_class_memory);
16954 %}
16955
16956 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16957 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16958 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16959 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16960 %{
16961 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16962 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16963 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16964 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16965 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16966 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16967 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16968
16969 ins_encode %{
16970 __ string_indexof($str1$$Register, $str2$$Register,
16971 $cnt1$$Register, $cnt2$$Register,
16972 $tmp1$$Register, $tmp2$$Register,
16973 $tmp3$$Register, $tmp4$$Register,
16974 $tmp5$$Register, $tmp6$$Register,
16975 -1, $result$$Register, StrIntrinsicNode::UL);
16976 %}
16977 ins_pipe(pipe_class_memory);
16978 %}
16979
16980 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16981 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16982 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16983 %{
16984 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16985 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16986 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16987 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16988 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16989 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16990
16991 ins_encode %{
16992 int icnt2 = (int)$int_cnt2$$constant;
16993 __ string_indexof($str1$$Register, $str2$$Register,
16994 $cnt1$$Register, zr,
16995 $tmp1$$Register, $tmp2$$Register,
16996 $tmp3$$Register, $tmp4$$Register, zr, zr,
16997 icnt2, $result$$Register, StrIntrinsicNode::UU);
16998 %}
16999 ins_pipe(pipe_class_memory);
17000 %}
17001
17002 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17003 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17004 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17005 %{
17006 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17007 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17008 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17009 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17010 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17011 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17012
17013 ins_encode %{
17014 int icnt2 = (int)$int_cnt2$$constant;
17015 __ string_indexof($str1$$Register, $str2$$Register,
17016 $cnt1$$Register, zr,
17017 $tmp1$$Register, $tmp2$$Register,
17018 $tmp3$$Register, $tmp4$$Register, zr, zr,
17019 icnt2, $result$$Register, StrIntrinsicNode::LL);
17020 %}
17021 ins_pipe(pipe_class_memory);
17022 %}
17023
17024 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17025 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17026 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17027 %{
17028 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17029 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17030 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17031 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17032 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17033 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17034
17035 ins_encode %{
17036 int icnt2 = (int)$int_cnt2$$constant;
17037 __ string_indexof($str1$$Register, $str2$$Register,
17038 $cnt1$$Register, zr,
17039 $tmp1$$Register, $tmp2$$Register,
17040 $tmp3$$Register, $tmp4$$Register, zr, zr,
17041 icnt2, $result$$Register, StrIntrinsicNode::UL);
17042 %}
17043 ins_pipe(pipe_class_memory);
17044 %}
17045
17046 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17047 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17048 iRegINoSp tmp3, rFlagsReg cr)
17049 %{
17050 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17051 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17052 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17053 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17054
17055 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17056
17057 ins_encode %{
17058 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17059 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17060 $tmp3$$Register);
17061 %}
17062 ins_pipe(pipe_class_memory);
17063 %}
17064
17065 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17066 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17067 iRegINoSp tmp3, rFlagsReg cr)
17068 %{
17069 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17070 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17071 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17072 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17073
17074 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17075
17076 ins_encode %{
17077 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17078 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17079 $tmp3$$Register);
17080 %}
17081 ins_pipe(pipe_class_memory);
17082 %}
17083
17084 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17085 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17086 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17087 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17088 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17089 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17090 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17091 ins_encode %{
17092 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17093 $result$$Register, $ztmp1$$FloatRegister,
17094 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17095 $ptmp$$PRegister, true /* isL */);
17096 %}
17097 ins_pipe(pipe_class_memory);
17098 %}
17099
17100 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17101 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17102 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17103 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17104 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17105 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17106 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17107 ins_encode %{
17108 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17109 $result$$Register, $ztmp1$$FloatRegister,
17110 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17111 $ptmp$$PRegister, false /* isL */);
17112 %}
17113 ins_pipe(pipe_class_memory);
17114 %}
17115
17116 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17117 iRegI_R0 result, rFlagsReg cr)
17118 %{
17119 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17120 match(Set result (StrEquals (Binary str1 str2) cnt));
17121 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17122
17123 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17124 ins_encode %{
17125 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17126 __ string_equals($str1$$Register, $str2$$Register,
17127 $result$$Register, $cnt$$Register, 1);
17128 %}
17129 ins_pipe(pipe_class_memory);
17130 %}
17131
17132 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17133 iRegI_R0 result, rFlagsReg cr)
17134 %{
17135 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17136 match(Set result (StrEquals (Binary str1 str2) cnt));
17137 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17138
17139 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17140 ins_encode %{
17141 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17142 __ string_equals($str1$$Register, $str2$$Register,
17143 $result$$Register, $cnt$$Register, 2);
17144 %}
17145 ins_pipe(pipe_class_memory);
17146 %}
17147
17148 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17149 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17150 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17151 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17152 iRegP_R10 tmp, rFlagsReg cr)
17153 %{
17154 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17155 match(Set result (AryEq ary1 ary2));
17156 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17157 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17158 TEMP vtmp6, TEMP vtmp7, KILL cr);
17159
17160 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17161 ins_encode %{
17162 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17163 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17164 $result$$Register, $tmp$$Register, 1);
17165 if (tpc == NULL) {
17166 ciEnv::current()->record_failure("CodeCache is full");
17167 return;
17168 }
17169 %}
17170 ins_pipe(pipe_class_memory);
17171 %}
17172
17173 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17174 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17175 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17176 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17177 iRegP_R10 tmp, rFlagsReg cr)
17178 %{
17179 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17180 match(Set result (AryEq ary1 ary2));
17181 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17182 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17183 TEMP vtmp6, TEMP vtmp7, KILL cr);
17184
17185 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17186 ins_encode %{
17187 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17188 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17189 $result$$Register, $tmp$$Register, 2);
17190 if (tpc == NULL) {
17191 ciEnv::current()->record_failure("CodeCache is full");
17192 return;
17193 }
17194 %}
17195 ins_pipe(pipe_class_memory);
17196 %}
17197
17198 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17199 %{
17200 match(Set result (CountPositives ary1 len));
17201 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17202 format %{ "count positives byte[] $ary1,$len -> $result" %}
17203 ins_encode %{
17204 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17205 if (tpc == NULL) {
17206 ciEnv::current()->record_failure("CodeCache is full");
17207 return;
17208 }
17209 %}
17210 ins_pipe( pipe_slow );
17211 %}
17212
17213 // fast char[] to byte[] compression
17214 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17215 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17216 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17217 iRegI_R0 result, rFlagsReg cr)
17218 %{
17219 match(Set result (StrCompressedCopy src (Binary dst len)));
17220 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17221 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17222
17223 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17224 ins_encode %{
17225 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17226 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17227 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17228 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17229 %}
17230 ins_pipe(pipe_slow);
17231 %}
17232
17233 // fast byte[] to char[] inflation
17234 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17235 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17236 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17237 %{
17238 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17239 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17240 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17241 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17242
17243 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17244 ins_encode %{
17245 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17246 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17247 $vtmp2$$FloatRegister, $tmp$$Register);
17248 if (tpc == NULL) {
17249 ciEnv::current()->record_failure("CodeCache is full");
17250 return;
17251 }
17252 %}
17253 ins_pipe(pipe_class_memory);
17254 %}
17255
17256 // encode char[] to byte[] in ISO_8859_1
17257 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17258 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17259 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17260 iRegI_R0 result, rFlagsReg cr)
17261 %{
17262 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17263 match(Set result (EncodeISOArray src (Binary dst len)));
17264 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17265 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17266
17267 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17268 ins_encode %{
17269 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17270 $result$$Register, false,
17271 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17272 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17273 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17274 %}
17275 ins_pipe(pipe_class_memory);
17276 %}
17277
17278 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17279 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17280 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17281 iRegI_R0 result, rFlagsReg cr)
17282 %{
17283 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17284 match(Set result (EncodeISOArray src (Binary dst len)));
17285 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17286 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17287
17288 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17289 ins_encode %{
17290 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17291 $result$$Register, true,
17292 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17293 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17294 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17295 %}
17296 ins_pipe(pipe_class_memory);
17297 %}
17298
17299 //----------------------------- CompressBits/ExpandBits ------------------------
17300
17301 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17302 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17303 match(Set dst (CompressBits src mask));
17304 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17305 format %{ "mov $tsrc, $src\n\t"
17306 "mov $tmask, $mask\n\t"
17307 "bext $tdst, $tsrc, $tmask\n\t"
17308 "mov $dst, $tdst"
17309 %}
17310 ins_encode %{
17311 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17312 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17313 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17314 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17315 %}
17316 ins_pipe(pipe_slow);
17317 %}
17318
17319 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17320 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17321 match(Set dst (CompressBits (LoadI mem) mask));
17322 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17323 format %{ "ldrs $tsrc, $mem\n\t"
17324 "ldrs $tmask, $mask\n\t"
17325 "bext $tdst, $tsrc, $tmask\n\t"
17326 "mov $dst, $tdst"
17327 %}
17328 ins_encode %{
17329 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17330 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17331 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17332 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17333 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17334 %}
17335 ins_pipe(pipe_slow);
17336 %}
17337
17338 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17339 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17340 match(Set dst (CompressBits src mask));
17341 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17342 format %{ "mov $tsrc, $src\n\t"
17343 "mov $tmask, $mask\n\t"
17344 "bext $tdst, $tsrc, $tmask\n\t"
17345 "mov $dst, $tdst"
17346 %}
17347 ins_encode %{
17348 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17349 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17350 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17351 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17352 %}
17353 ins_pipe(pipe_slow);
17354 %}
17355
17356 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17357 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17358 match(Set dst (CompressBits (LoadL mem) mask));
17359 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17360 format %{ "ldrd $tsrc, $mem\n\t"
17361 "ldrd $tmask, $mask\n\t"
17362 "bext $tdst, $tsrc, $tmask\n\t"
17363 "mov $dst, $tdst"
17364 %}
17365 ins_encode %{
17366 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17367 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17368 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17369 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17370 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17371 %}
17372 ins_pipe(pipe_slow);
17373 %}
17374
17375 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17376 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17377 match(Set dst (ExpandBits src mask));
17378 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17379 format %{ "mov $tsrc, $src\n\t"
17380 "mov $tmask, $mask\n\t"
17381 "bdep $tdst, $tsrc, $tmask\n\t"
17382 "mov $dst, $tdst"
17383 %}
17384 ins_encode %{
17385 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17386 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17387 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17388 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17389 %}
17390 ins_pipe(pipe_slow);
17391 %}
17392
17393 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17394 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17395 match(Set dst (ExpandBits (LoadI mem) mask));
17396 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17397 format %{ "ldrs $tsrc, $mem\n\t"
17398 "ldrs $tmask, $mask\n\t"
17399 "bdep $tdst, $tsrc, $tmask\n\t"
17400 "mov $dst, $tdst"
17401 %}
17402 ins_encode %{
17403 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17404 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17405 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17406 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17407 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17408 %}
17409 ins_pipe(pipe_slow);
17410 %}
17411
17412 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17413 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17414 match(Set dst (ExpandBits src mask));
17415 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17416 format %{ "mov $tsrc, $src\n\t"
17417 "mov $tmask, $mask\n\t"
17418 "bdep $tdst, $tsrc, $tmask\n\t"
17419 "mov $dst, $tdst"
17420 %}
17421 ins_encode %{
17422 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17423 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17424 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17425 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17426 %}
17427 ins_pipe(pipe_slow);
17428 %}
17429
17430
17431 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17432 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17433 match(Set dst (ExpandBits (LoadL mem) mask));
17434 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17435 format %{ "ldrd $tsrc, $mem\n\t"
17436 "ldrd $tmask, $mask\n\t"
17437 "bdep $tdst, $tsrc, $tmask\n\t"
17438 "mov $dst, $tdst"
17439 %}
17440 ins_encode %{
17441 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17442 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17443 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17444 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17445 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17446 %}
17447 ins_pipe(pipe_slow);
17448 %}
17449
17450 // ============================================================================
17451 // This name is KNOWN by the ADLC and cannot be changed.
17452 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17453 // for this guy.
17454 instruct tlsLoadP(thread_RegP dst)
17455 %{
17456 match(Set dst (ThreadLocal));
17457
17458 ins_cost(0);
17459
17460 format %{ " -- \t// $dst=Thread::current(), empty" %}
17461
17462 size(0);
17463
17464 ins_encode( /*empty*/ );
17465
17466 ins_pipe(pipe_class_empty);
17467 %}
17468
17469 //----------PEEPHOLE RULES-----------------------------------------------------
17470 // These must follow all instruction definitions as they use the names
17471 // defined in the instructions definitions.
17472 //
17473 // peepmatch ( root_instr_name [preceding_instruction]* );
17474 //
17475 // peepconstraint %{
17476 // (instruction_number.operand_name relational_op instruction_number.operand_name
17477 // [, ...] );
17478 // // instruction numbers are zero-based using left to right order in peepmatch
17479 //
17480 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17481 // // provide an instruction_number.operand_name for each operand that appears
17482 // // in the replacement instruction's match rule
17483 //
17484 // ---------VM FLAGS---------------------------------------------------------
17485 //
17486 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17487 //
17488 // Each peephole rule is given an identifying number starting with zero and
17489 // increasing by one in the order seen by the parser. An individual peephole
17490 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17491 // on the command-line.
17492 //
17493 // ---------CURRENT LIMITATIONS----------------------------------------------
17494 //
17495 // Only match adjacent instructions in same basic block
17496 // Only equality constraints
17497 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17498 // Only one replacement instruction
17499 //
17500 // ---------EXAMPLE----------------------------------------------------------
17501 //
17502 // // pertinent parts of existing instructions in architecture description
17503 // instruct movI(iRegINoSp dst, iRegI src)
17504 // %{
17505 // match(Set dst (CopyI src));
17506 // %}
17507 //
17508 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17509 // %{
17510 // match(Set dst (AddI dst src));
17511 // effect(KILL cr);
17512 // %}
17513 //
17514 // // Change (inc mov) to lea
17515 // peephole %{
17516 // // increment preceded by register-register move
17517 // peepmatch ( incI_iReg movI );
17518 // // require that the destination register of the increment
17519 // // match the destination register of the move
17520 // peepconstraint ( 0.dst == 1.dst );
17521 // // construct a replacement instruction that sets
17522 // // the destination to ( move's source register + one )
17523 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17524 // %}
17525 //
17526
17527 // Implementation no longer uses movX instructions since
17528 // machine-independent system no longer uses CopyX nodes.
17529 //
17530 // peephole
17531 // %{
17532 // peepmatch (incI_iReg movI);
17533 // peepconstraint (0.dst == 1.dst);
17534 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17535 // %}
17536
17537 // peephole
17538 // %{
17539 // peepmatch (decI_iReg movI);
17540 // peepconstraint (0.dst == 1.dst);
17541 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17542 // %}
17543
17544 // peephole
17545 // %{
17546 // peepmatch (addI_iReg_imm movI);
17547 // peepconstraint (0.dst == 1.dst);
17548 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17549 // %}
17550
17551 // peephole
17552 // %{
17553 // peepmatch (incL_iReg movL);
17554 // peepconstraint (0.dst == 1.dst);
17555 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17556 // %}
17557
17558 // peephole
17559 // %{
17560 // peepmatch (decL_iReg movL);
17561 // peepconstraint (0.dst == 1.dst);
17562 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17563 // %}
17564
17565 // peephole
17566 // %{
17567 // peepmatch (addL_iReg_imm movL);
17568 // peepconstraint (0.dst == 1.dst);
17569 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17570 // %}
17571
17572 // peephole
17573 // %{
17574 // peepmatch (addP_iReg_imm movP);
17575 // peepconstraint (0.dst == 1.dst);
17576 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17577 // %}
17578
17579 // // Change load of spilled value to only a spill
17580 // instruct storeI(memory mem, iRegI src)
17581 // %{
17582 // match(Set mem (StoreI mem src));
17583 // %}
17584 //
17585 // instruct loadI(iRegINoSp dst, memory mem)
17586 // %{
17587 // match(Set dst (LoadI mem));
17588 // %}
17589 //
17590
17591 //----------SMARTSPILL RULES---------------------------------------------------
17592 // These must follow all instruction definitions as they use the names
17593 // defined in the instructions definitions.
17594
17595 // Local Variables:
17596 // mode: c++
17597 // End: