1 //
2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 P0,
464 P1,
465 P2,
466 P3,
467 P4,
468 P5,
469 P6,
470 P7,
471
472 P8,
473 P9,
474 P10,
475 P11,
476 P12,
477 P13,
478 P14,
479 P15,
480 );
481
482 alloc_class chunk3(RFLAGS);
483
484 //----------Architecture Description Register Classes--------------------------
485 // Several register classes are automatically defined based upon information in
486 // this architecture description.
487 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
488 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
489 //
490
491 // Class for all 32 bit general purpose registers
492 reg_class all_reg32(
493 R0,
494 R1,
495 R2,
496 R3,
497 R4,
498 R5,
499 R6,
500 R7,
501 R10,
502 R11,
503 R12,
504 R13,
505 R14,
506 R15,
507 R16,
508 R17,
509 R18,
510 R19,
511 R20,
512 R21,
513 R22,
514 R23,
515 R24,
516 R25,
517 R26,
518 R27,
519 R28,
520 R29,
521 R30,
522 R31
523 );
524
525
526 // Class for all 32 bit integer registers (excluding SP which
527 // will never be used as an integer register)
528 reg_class any_reg32 %{
529 return _ANY_REG32_mask;
530 %}
531
532 // Singleton class for R0 int register
533 reg_class int_r0_reg(R0);
534
535 // Singleton class for R2 int register
536 reg_class int_r2_reg(R2);
537
538 // Singleton class for R3 int register
539 reg_class int_r3_reg(R3);
540
541 // Singleton class for R4 int register
542 reg_class int_r4_reg(R4);
543
544 // Singleton class for R31 int register
545 reg_class int_r31_reg(R31);
546
547 // Class for all 64 bit general purpose registers
548 reg_class all_reg(
549 R0, R0_H,
550 R1, R1_H,
551 R2, R2_H,
552 R3, R3_H,
553 R4, R4_H,
554 R5, R5_H,
555 R6, R6_H,
556 R7, R7_H,
557 R10, R10_H,
558 R11, R11_H,
559 R12, R12_H,
560 R13, R13_H,
561 R14, R14_H,
562 R15, R15_H,
563 R16, R16_H,
564 R17, R17_H,
565 R18, R18_H,
566 R19, R19_H,
567 R20, R20_H,
568 R21, R21_H,
569 R22, R22_H,
570 R23, R23_H,
571 R24, R24_H,
572 R25, R25_H,
573 R26, R26_H,
574 R27, R27_H,
575 R28, R28_H,
576 R29, R29_H,
577 R30, R30_H,
578 R31, R31_H
579 );
580
581 // Class for all long integer registers (including SP)
582 reg_class any_reg %{
583 return _ANY_REG_mask;
584 %}
585
586 // Class for non-allocatable 32 bit registers
587 reg_class non_allocatable_reg32(
588 #ifdef R18_RESERVED
589 // See comment in register_aarch64.hpp
590 R18, // tls on Windows
591 #endif
592 R28, // thread
593 R30, // lr
594 R31 // sp
595 );
596
597 // Class for non-allocatable 64 bit registers
598 reg_class non_allocatable_reg(
599 #ifdef R18_RESERVED
600 // See comment in register_aarch64.hpp
601 R18, R18_H, // tls on Windows, platform register on macOS
602 #endif
603 R28, R28_H, // thread
604 R30, R30_H, // lr
605 R31, R31_H // sp
606 );
607
608 // Class for all non-special integer registers
609 reg_class no_special_reg32 %{
610 return _NO_SPECIAL_REG32_mask;
611 %}
612
613 // Class for all non-special long integer registers
614 reg_class no_special_reg %{
615 return _NO_SPECIAL_REG_mask;
616 %}
617
618 // Class for 64 bit register r0
619 reg_class r0_reg(
620 R0, R0_H
621 );
622
623 // Class for 64 bit register r1
624 reg_class r1_reg(
625 R1, R1_H
626 );
627
628 // Class for 64 bit register r2
629 reg_class r2_reg(
630 R2, R2_H
631 );
632
633 // Class for 64 bit register r3
634 reg_class r3_reg(
635 R3, R3_H
636 );
637
638 // Class for 64 bit register r4
639 reg_class r4_reg(
640 R4, R4_H
641 );
642
643 // Class for 64 bit register r5
644 reg_class r5_reg(
645 R5, R5_H
646 );
647
648 // Class for 64 bit register r10
649 reg_class r10_reg(
650 R10, R10_H
651 );
652
653 // Class for 64 bit register r11
654 reg_class r11_reg(
655 R11, R11_H
656 );
657
658 // Class for method register
659 reg_class method_reg(
660 R12, R12_H
661 );
662
663 // Class for heapbase register
664 reg_class heapbase_reg(
665 R27, R27_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all float registers
699 reg_class float_reg(
700 V0,
701 V1,
702 V2,
703 V3,
704 V4,
705 V5,
706 V6,
707 V7,
708 V8,
709 V9,
710 V10,
711 V11,
712 V12,
713 V13,
714 V14,
715 V15,
716 V16,
717 V17,
718 V18,
719 V19,
720 V20,
721 V21,
722 V22,
723 V23,
724 V24,
725 V25,
726 V26,
727 V27,
728 V28,
729 V29,
730 V30,
731 V31
732 );
733
734 // Double precision float registers have virtual `high halves' that
735 // are needed by the allocator.
736 // Class for all double registers
737 reg_class double_reg(
738 V0, V0_H,
739 V1, V1_H,
740 V2, V2_H,
741 V3, V3_H,
742 V4, V4_H,
743 V5, V5_H,
744 V6, V6_H,
745 V7, V7_H,
746 V8, V8_H,
747 V9, V9_H,
748 V10, V10_H,
749 V11, V11_H,
750 V12, V12_H,
751 V13, V13_H,
752 V14, V14_H,
753 V15, V15_H,
754 V16, V16_H,
755 V17, V17_H,
756 V18, V18_H,
757 V19, V19_H,
758 V20, V20_H,
759 V21, V21_H,
760 V22, V22_H,
761 V23, V23_H,
762 V24, V24_H,
763 V25, V25_H,
764 V26, V26_H,
765 V27, V27_H,
766 V28, V28_H,
767 V29, V29_H,
768 V30, V30_H,
769 V31, V31_H
770 );
771
772 // Class for all SVE vector registers.
773 reg_class vectora_reg (
774 V0, V0_H, V0_J, V0_K,
775 V1, V1_H, V1_J, V1_K,
776 V2, V2_H, V2_J, V2_K,
777 V3, V3_H, V3_J, V3_K,
778 V4, V4_H, V4_J, V4_K,
779 V5, V5_H, V5_J, V5_K,
780 V6, V6_H, V6_J, V6_K,
781 V7, V7_H, V7_J, V7_K,
782 V8, V8_H, V8_J, V8_K,
783 V9, V9_H, V9_J, V9_K,
784 V10, V10_H, V10_J, V10_K,
785 V11, V11_H, V11_J, V11_K,
786 V12, V12_H, V12_J, V12_K,
787 V13, V13_H, V13_J, V13_K,
788 V14, V14_H, V14_J, V14_K,
789 V15, V15_H, V15_J, V15_K,
790 V16, V16_H, V16_J, V16_K,
791 V17, V17_H, V17_J, V17_K,
792 V18, V18_H, V18_J, V18_K,
793 V19, V19_H, V19_J, V19_K,
794 V20, V20_H, V20_J, V20_K,
795 V21, V21_H, V21_J, V21_K,
796 V22, V22_H, V22_J, V22_K,
797 V23, V23_H, V23_J, V23_K,
798 V24, V24_H, V24_J, V24_K,
799 V25, V25_H, V25_J, V25_K,
800 V26, V26_H, V26_J, V26_K,
801 V27, V27_H, V27_J, V27_K,
802 V28, V28_H, V28_J, V28_K,
803 V29, V29_H, V29_J, V29_K,
804 V30, V30_H, V30_J, V30_K,
805 V31, V31_H, V31_J, V31_K,
806 );
807
808 // Class for all 64bit vector registers
809 reg_class vectord_reg(
810 V0, V0_H,
811 V1, V1_H,
812 V2, V2_H,
813 V3, V3_H,
814 V4, V4_H,
815 V5, V5_H,
816 V6, V6_H,
817 V7, V7_H,
818 V8, V8_H,
819 V9, V9_H,
820 V10, V10_H,
821 V11, V11_H,
822 V12, V12_H,
823 V13, V13_H,
824 V14, V14_H,
825 V15, V15_H,
826 V16, V16_H,
827 V17, V17_H,
828 V18, V18_H,
829 V19, V19_H,
830 V20, V20_H,
831 V21, V21_H,
832 V22, V22_H,
833 V23, V23_H,
834 V24, V24_H,
835 V25, V25_H,
836 V26, V26_H,
837 V27, V27_H,
838 V28, V28_H,
839 V29, V29_H,
840 V30, V30_H,
841 V31, V31_H
842 );
843
844 // Class for all 128bit vector registers
845 reg_class vectorx_reg(
846 V0, V0_H, V0_J, V0_K,
847 V1, V1_H, V1_J, V1_K,
848 V2, V2_H, V2_J, V2_K,
849 V3, V3_H, V3_J, V3_K,
850 V4, V4_H, V4_J, V4_K,
851 V5, V5_H, V5_J, V5_K,
852 V6, V6_H, V6_J, V6_K,
853 V7, V7_H, V7_J, V7_K,
854 V8, V8_H, V8_J, V8_K,
855 V9, V9_H, V9_J, V9_K,
856 V10, V10_H, V10_J, V10_K,
857 V11, V11_H, V11_J, V11_K,
858 V12, V12_H, V12_J, V12_K,
859 V13, V13_H, V13_J, V13_K,
860 V14, V14_H, V14_J, V14_K,
861 V15, V15_H, V15_J, V15_K,
862 V16, V16_H, V16_J, V16_K,
863 V17, V17_H, V17_J, V17_K,
864 V18, V18_H, V18_J, V18_K,
865 V19, V19_H, V19_J, V19_K,
866 V20, V20_H, V20_J, V20_K,
867 V21, V21_H, V21_J, V21_K,
868 V22, V22_H, V22_J, V22_K,
869 V23, V23_H, V23_J, V23_K,
870 V24, V24_H, V24_J, V24_K,
871 V25, V25_H, V25_J, V25_K,
872 V26, V26_H, V26_J, V26_K,
873 V27, V27_H, V27_J, V27_K,
874 V28, V28_H, V28_J, V28_K,
875 V29, V29_H, V29_J, V29_K,
876 V30, V30_H, V30_J, V30_K,
877 V31, V31_H, V31_J, V31_K
878 );
879
880 // Class for 128 bit register v0
881 reg_class v0_reg(
882 V0, V0_H
883 );
884
885 // Class for 128 bit register v1
886 reg_class v1_reg(
887 V1, V1_H
888 );
889
890 // Class for 128 bit register v2
891 reg_class v2_reg(
892 V2, V2_H
893 );
894
895 // Class for 128 bit register v3
896 reg_class v3_reg(
897 V3, V3_H
898 );
899
900 // Class for 128 bit register v4
901 reg_class v4_reg(
902 V4, V4_H
903 );
904
905 // Class for 128 bit register v5
906 reg_class v5_reg(
907 V5, V5_H
908 );
909
910 // Class for 128 bit register v6
911 reg_class v6_reg(
912 V6, V6_H
913 );
914
915 // Class for 128 bit register v7
916 reg_class v7_reg(
917 V7, V7_H
918 );
919
920 // Class for 128 bit register v8
921 reg_class v8_reg(
922 V8, V8_H
923 );
924
925 // Class for 128 bit register v9
926 reg_class v9_reg(
927 V9, V9_H
928 );
929
930 // Class for 128 bit register v10
931 reg_class v10_reg(
932 V10, V10_H
933 );
934
935 // Class for 128 bit register v11
936 reg_class v11_reg(
937 V11, V11_H
938 );
939
940 // Class for 128 bit register v12
941 reg_class v12_reg(
942 V12, V12_H
943 );
944
945 // Class for 128 bit register v13
946 reg_class v13_reg(
947 V13, V13_H
948 );
949
950 // Class for 128 bit register v14
951 reg_class v14_reg(
952 V14, V14_H
953 );
954
955 // Class for 128 bit register v15
956 reg_class v15_reg(
957 V15, V15_H
958 );
959
960 // Class for 128 bit register v16
961 reg_class v16_reg(
962 V16, V16_H
963 );
964
965 // Class for 128 bit register v17
966 reg_class v17_reg(
967 V17, V17_H
968 );
969
970 // Class for 128 bit register v18
971 reg_class v18_reg(
972 V18, V18_H
973 );
974
975 // Class for 128 bit register v19
976 reg_class v19_reg(
977 V19, V19_H
978 );
979
980 // Class for 128 bit register v20
981 reg_class v20_reg(
982 V20, V20_H
983 );
984
985 // Class for 128 bit register v21
986 reg_class v21_reg(
987 V21, V21_H
988 );
989
990 // Class for 128 bit register v22
991 reg_class v22_reg(
992 V22, V22_H
993 );
994
995 // Class for 128 bit register v23
996 reg_class v23_reg(
997 V23, V23_H
998 );
999
1000 // Class for 128 bit register v24
1001 reg_class v24_reg(
1002 V24, V24_H
1003 );
1004
1005 // Class for 128 bit register v25
1006 reg_class v25_reg(
1007 V25, V25_H
1008 );
1009
1010 // Class for 128 bit register v26
1011 reg_class v26_reg(
1012 V26, V26_H
1013 );
1014
1015 // Class for 128 bit register v27
1016 reg_class v27_reg(
1017 V27, V27_H
1018 );
1019
1020 // Class for 128 bit register v28
1021 reg_class v28_reg(
1022 V28, V28_H
1023 );
1024
1025 // Class for 128 bit register v29
1026 reg_class v29_reg(
1027 V29, V29_H
1028 );
1029
1030 // Class for 128 bit register v30
1031 reg_class v30_reg(
1032 V30, V30_H
1033 );
1034
1035 // Class for 128 bit register v31
1036 reg_class v31_reg(
1037 V31, V31_H
1038 );
1039
1040 // Class for all SVE predicate registers.
1041 reg_class pr_reg (
1042 P0,
1043 P1,
1044 P2,
1045 P3,
1046 P4,
1047 P5,
1048 P6,
1049 // P7, non-allocatable, preserved with all elements preset to TRUE.
1050 P8,
1051 P9,
1052 P10,
1053 P11,
1054 P12,
1055 P13,
1056 P14,
1057 P15
1058 );
1059
1060 // Class for SVE governing predicate registers, which are used
1061 // to determine the active elements of a predicated instruction.
1062 reg_class gov_pr (
1063 P0,
1064 P1,
1065 P2,
1066 P3,
1067 P4,
1068 P5,
1069 P6,
1070 // P7, non-allocatable, preserved with all elements preset to TRUE.
1071 );
1072
1073 reg_class p0_reg(P0);
1074 reg_class p1_reg(P1);
1075
1076 // Singleton class for condition codes
1077 reg_class int_flags(RFLAGS);
1078
1079 %}
1080
1081 //----------DEFINITION BLOCK---------------------------------------------------
1082 // Define name --> value mappings to inform the ADLC of an integer valued name
1083 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1084 // Format:
1085 // int_def <name> ( <int_value>, <expression>);
1086 // Generated Code in ad_<arch>.hpp
1087 // #define <name> (<expression>)
1088 // // value == <int_value>
1089 // Generated code in ad_<arch>.cpp adlc_verification()
1090 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1091 //
1092
1093 // we follow the ppc-aix port in using a simple cost model which ranks
1094 // register operations as cheap, memory ops as more expensive and
1095 // branches as most expensive. the first two have a low as well as a
1096 // normal cost. huge cost appears to be a way of saying don't do
1097 // something
1098
1099 definitions %{
1100 // The default cost (of a register move instruction).
1101 int_def INSN_COST ( 100, 100);
1102 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1103 int_def CALL_COST ( 200, 2 * INSN_COST);
1104 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1105 %}
1106
1107
1108 //----------SOURCE BLOCK-------------------------------------------------------
1109 // This is a block of C++ code which provides values, functions, and
1110 // definitions necessary in the rest of the architecture description
1111
1112 source_hpp %{
1113
1114 #include "asm/macroAssembler.hpp"
1115 #include "gc/shared/barrierSetAssembler.hpp"
1116 #include "gc/shared/cardTable.hpp"
1117 #include "gc/shared/cardTableBarrierSet.hpp"
1118 #include "gc/shared/collectedHeap.hpp"
1119 #include "opto/addnode.hpp"
1120 #include "opto/convertnode.hpp"
1121 #include "runtime/objectMonitor.hpp"
1122
1123 extern RegMask _ANY_REG32_mask;
1124 extern RegMask _ANY_REG_mask;
1125 extern RegMask _PTR_REG_mask;
1126 extern RegMask _NO_SPECIAL_REG32_mask;
1127 extern RegMask _NO_SPECIAL_REG_mask;
1128 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1129
1130 class CallStubImpl {
1131
1132 //--------------------------------------------------------------
1133 //---< Used for optimization in Compile::shorten_branches >---
1134 //--------------------------------------------------------------
1135
1136 public:
1137 // Size of call trampoline stub.
1138 static uint size_call_trampoline() {
1139 return 0; // no call trampolines on this platform
1140 }
1141
1142 // number of relocations needed by a call trampoline stub
1143 static uint reloc_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146 };
1147
1148 class HandlerImpl {
1149
1150 public:
1151
1152 static int emit_exception_handler(CodeBuffer &cbuf);
1153 static int emit_deopt_handler(CodeBuffer& cbuf);
1154
1155 static uint size_exception_handler() {
1156 return MacroAssembler::far_codestub_branch_size();
1157 }
1158
1159 static uint size_deopt_handler() {
1160 // count one adr and one far branch instruction
1161 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1162 }
1163 };
1164
1165 class Node::PD {
1166 public:
1167 enum NodeFlags {
1168 _last_flag = Node::_last_flag
1169 };
1170 };
1171
1172 bool is_CAS(int opcode, bool maybe_volatile);
1173
1174 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1175
1176 bool unnecessary_acquire(const Node *barrier);
1177 bool needs_acquiring_load(const Node *load);
1178
1179 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1180
1181 bool unnecessary_release(const Node *barrier);
1182 bool unnecessary_volatile(const Node *barrier);
1183 bool needs_releasing_store(const Node *store);
1184
1185 // predicate controlling translation of CompareAndSwapX
1186 bool needs_acquiring_load_exclusive(const Node *load);
1187
1188 // predicate controlling addressing modes
1189 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1190 %}
1191
1192 source %{
1193
1194 // Derived RegMask with conditionally allocatable registers
1195
1196 void PhaseOutput::pd_perform_mach_node_analysis() {
1197 }
1198
1199 int MachNode::pd_alignment_required() const {
1200 return 1;
1201 }
1202
1203 int MachNode::compute_padding(int current_offset) const {
1204 return 0;
1205 }
1206
1207 RegMask _ANY_REG32_mask;
1208 RegMask _ANY_REG_mask;
1209 RegMask _PTR_REG_mask;
1210 RegMask _NO_SPECIAL_REG32_mask;
1211 RegMask _NO_SPECIAL_REG_mask;
1212 RegMask _NO_SPECIAL_PTR_REG_mask;
1213
1214 void reg_mask_init() {
1215 // We derive below RegMask(s) from the ones which are auto-generated from
1216 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1217 // registers conditionally reserved.
1218
1219 _ANY_REG32_mask = _ALL_REG32_mask;
1220 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1221
1222 _ANY_REG_mask = _ALL_REG_mask;
1223
1224 _PTR_REG_mask = _ALL_REG_mask;
1225
1226 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1227 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1228
1229 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1230 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1231
1232 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 // r27 is not allocatable when compressed oops is on and heapbase is not
1236 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1237 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1238 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1239 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1240 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1241 }
1242
1243 // r29 is not allocatable when PreserveFramePointer is on
1244 if (PreserveFramePointer) {
1245 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1246 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1247 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1248 }
1249 }
1250
1251 // Optimizaton of volatile gets and puts
1252 // -------------------------------------
1253 //
1254 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1255 // use to implement volatile reads and writes. For a volatile read
1256 // we simply need
1257 //
1258 // ldar<x>
1259 //
1260 // and for a volatile write we need
1261 //
1262 // stlr<x>
1263 //
1264 // Alternatively, we can implement them by pairing a normal
1265 // load/store with a memory barrier. For a volatile read we need
1266 //
1267 // ldr<x>
1268 // dmb ishld
1269 //
1270 // for a volatile write
1271 //
1272 // dmb ish
1273 // str<x>
1274 // dmb ish
1275 //
1276 // We can also use ldaxr and stlxr to implement compare and swap CAS
1277 // sequences. These are normally translated to an instruction
1278 // sequence like the following
1279 //
1280 // dmb ish
1281 // retry:
1282 // ldxr<x> rval raddr
1283 // cmp rval rold
1284 // b.ne done
1285 // stlxr<x> rval, rnew, rold
1286 // cbnz rval retry
1287 // done:
1288 // cset r0, eq
1289 // dmb ishld
1290 //
1291 // Note that the exclusive store is already using an stlxr
1292 // instruction. That is required to ensure visibility to other
1293 // threads of the exclusive write (assuming it succeeds) before that
1294 // of any subsequent writes.
1295 //
1296 // The following instruction sequence is an improvement on the above
1297 //
1298 // retry:
1299 // ldaxr<x> rval raddr
1300 // cmp rval rold
1301 // b.ne done
1302 // stlxr<x> rval, rnew, rold
1303 // cbnz rval retry
1304 // done:
1305 // cset r0, eq
1306 //
1307 // We don't need the leading dmb ish since the stlxr guarantees
1308 // visibility of prior writes in the case that the swap is
1309 // successful. Crucially we don't have to worry about the case where
1310 // the swap is not successful since no valid program should be
1311 // relying on visibility of prior changes by the attempting thread
1312 // in the case where the CAS fails.
1313 //
1314 // Similarly, we don't need the trailing dmb ishld if we substitute
1315 // an ldaxr instruction since that will provide all the guarantees we
1316 // require regarding observation of changes made by other threads
1317 // before any change to the CAS address observed by the load.
1318 //
1319 // In order to generate the desired instruction sequence we need to
1320 // be able to identify specific 'signature' ideal graph node
1321 // sequences which i) occur as a translation of a volatile reads or
1322 // writes or CAS operations and ii) do not occur through any other
1323 // translation or graph transformation. We can then provide
1324 // alternative aldc matching rules which translate these node
1325 // sequences to the desired machine code sequences. Selection of the
1326 // alternative rules can be implemented by predicates which identify
1327 // the relevant node sequences.
1328 //
1329 // The ideal graph generator translates a volatile read to the node
1330 // sequence
1331 //
1332 // LoadX[mo_acquire]
1333 // MemBarAcquire
1334 //
1335 // As a special case when using the compressed oops optimization we
1336 // may also see this variant
1337 //
1338 // LoadN[mo_acquire]
1339 // DecodeN
1340 // MemBarAcquire
1341 //
1342 // A volatile write is translated to the node sequence
1343 //
1344 // MemBarRelease
1345 // StoreX[mo_release] {CardMark}-optional
1346 // MemBarVolatile
1347 //
1348 // n.b. the above node patterns are generated with a strict
1349 // 'signature' configuration of input and output dependencies (see
1350 // the predicates below for exact details). The card mark may be as
1351 // simple as a few extra nodes or, in a few GC configurations, may
1352 // include more complex control flow between the leading and
1353 // trailing memory barriers. However, whatever the card mark
1354 // configuration these signatures are unique to translated volatile
1355 // reads/stores -- they will not appear as a result of any other
1356 // bytecode translation or inlining nor as a consequence of
1357 // optimizing transforms.
1358 //
1359 // We also want to catch inlined unsafe volatile gets and puts and
1360 // be able to implement them using either ldar<x>/stlr<x> or some
1361 // combination of ldr<x>/stlr<x> and dmb instructions.
1362 //
1363 // Inlined unsafe volatiles puts manifest as a minor variant of the
1364 // normal volatile put node sequence containing an extra cpuorder
1365 // membar
1366 //
1367 // MemBarRelease
1368 // MemBarCPUOrder
1369 // StoreX[mo_release] {CardMark}-optional
1370 // MemBarCPUOrder
1371 // MemBarVolatile
1372 //
1373 // n.b. as an aside, a cpuorder membar is not itself subject to
1374 // matching and translation by adlc rules. However, the rule
1375 // predicates need to detect its presence in order to correctly
1376 // select the desired adlc rules.
1377 //
1378 // Inlined unsafe volatile gets manifest as a slightly different
1379 // node sequence to a normal volatile get because of the
1380 // introduction of some CPUOrder memory barriers to bracket the
1381 // Load. However, but the same basic skeleton of a LoadX feeding a
1382 // MemBarAcquire, possibly through an optional DecodeN, is still
1383 // present
1384 //
1385 // MemBarCPUOrder
1386 // || \\
1387 // MemBarCPUOrder LoadX[mo_acquire]
1388 // || |
1389 // || {DecodeN} optional
1390 // || /
1391 // MemBarAcquire
1392 //
1393 // In this case the acquire membar does not directly depend on the
1394 // load. However, we can be sure that the load is generated from an
1395 // inlined unsafe volatile get if we see it dependent on this unique
1396 // sequence of membar nodes. Similarly, given an acquire membar we
1397 // can know that it was added because of an inlined unsafe volatile
1398 // get if it is fed and feeds a cpuorder membar and if its feed
1399 // membar also feeds an acquiring load.
1400 //
1401 // Finally an inlined (Unsafe) CAS operation is translated to the
1402 // following ideal graph
1403 //
1404 // MemBarRelease
1405 // MemBarCPUOrder
1406 // CompareAndSwapX {CardMark}-optional
1407 // MemBarCPUOrder
1408 // MemBarAcquire
1409 //
1410 // So, where we can identify these volatile read and write
1411 // signatures we can choose to plant either of the above two code
1412 // sequences. For a volatile read we can simply plant a normal
1413 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1414 // also choose to inhibit translation of the MemBarAcquire and
1415 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1416 //
1417 // When we recognise a volatile store signature we can choose to
1418 // plant at a dmb ish as a translation for the MemBarRelease, a
1419 // normal str<x> and then a dmb ish for the MemBarVolatile.
1420 // Alternatively, we can inhibit translation of the MemBarRelease
1421 // and MemBarVolatile and instead plant a simple stlr<x>
1422 // instruction.
1423 //
1424 // when we recognise a CAS signature we can choose to plant a dmb
1425 // ish as a translation for the MemBarRelease, the conventional
1426 // macro-instruction sequence for the CompareAndSwap node (which
1427 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1428 // Alternatively, we can elide generation of the dmb instructions
1429 // and plant the alternative CompareAndSwap macro-instruction
1430 // sequence (which uses ldaxr<x>).
1431 //
1432 // Of course, the above only applies when we see these signature
1433 // configurations. We still want to plant dmb instructions in any
1434 // other cases where we may see a MemBarAcquire, MemBarRelease or
1435 // MemBarVolatile. For example, at the end of a constructor which
1436 // writes final/volatile fields we will see a MemBarRelease
1437 // instruction and this needs a 'dmb ish' lest we risk the
1438 // constructed object being visible without making the
1439 // final/volatile field writes visible.
1440 //
1441 // n.b. the translation rules below which rely on detection of the
1442 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1443 // If we see anything other than the signature configurations we
1444 // always just translate the loads and stores to ldr<x> and str<x>
1445 // and translate acquire, release and volatile membars to the
1446 // relevant dmb instructions.
1447 //
1448
1449 // is_CAS(int opcode, bool maybe_volatile)
1450 //
1451 // return true if opcode is one of the possible CompareAndSwapX
1452 // values otherwise false.
1453
1454 bool is_CAS(int opcode, bool maybe_volatile)
1455 {
1456 switch(opcode) {
1457 // We handle these
1458 case Op_CompareAndSwapI:
1459 case Op_CompareAndSwapL:
1460 case Op_CompareAndSwapP:
1461 case Op_CompareAndSwapN:
1462 case Op_ShenandoahCompareAndSwapP:
1463 case Op_ShenandoahCompareAndSwapN:
1464 case Op_CompareAndSwapB:
1465 case Op_CompareAndSwapS:
1466 case Op_GetAndSetI:
1467 case Op_GetAndSetL:
1468 case Op_GetAndSetP:
1469 case Op_GetAndSetN:
1470 case Op_GetAndAddI:
1471 case Op_GetAndAddL:
1472 return true;
1473 case Op_CompareAndExchangeI:
1474 case Op_CompareAndExchangeN:
1475 case Op_CompareAndExchangeB:
1476 case Op_CompareAndExchangeS:
1477 case Op_CompareAndExchangeL:
1478 case Op_CompareAndExchangeP:
1479 case Op_WeakCompareAndSwapB:
1480 case Op_WeakCompareAndSwapS:
1481 case Op_WeakCompareAndSwapI:
1482 case Op_WeakCompareAndSwapL:
1483 case Op_WeakCompareAndSwapP:
1484 case Op_WeakCompareAndSwapN:
1485 case Op_ShenandoahWeakCompareAndSwapP:
1486 case Op_ShenandoahWeakCompareAndSwapN:
1487 case Op_ShenandoahCompareAndExchangeP:
1488 case Op_ShenandoahCompareAndExchangeN:
1489 return maybe_volatile;
1490 default:
1491 return false;
1492 }
1493 }
1494
1495 // helper to determine the maximum number of Phi nodes we may need to
1496 // traverse when searching from a card mark membar for the merge mem
1497 // feeding a trailing membar or vice versa
1498
1499 // predicates controlling emit of ldr<x>/ldar<x>
1500
1501 bool unnecessary_acquire(const Node *barrier)
1502 {
1503 assert(barrier->is_MemBar(), "expecting a membar");
1504
1505 MemBarNode* mb = barrier->as_MemBar();
1506
1507 if (mb->trailing_load()) {
1508 return true;
1509 }
1510
1511 if (mb->trailing_load_store()) {
1512 Node* load_store = mb->in(MemBarNode::Precedent);
1513 assert(load_store->is_LoadStore(), "unexpected graph shape");
1514 return is_CAS(load_store->Opcode(), true);
1515 }
1516
1517 return false;
1518 }
1519
1520 bool needs_acquiring_load(const Node *n)
1521 {
1522 assert(n->is_Load(), "expecting a load");
1523 LoadNode *ld = n->as_Load();
1524 return ld->is_acquire();
1525 }
1526
1527 bool unnecessary_release(const Node *n)
1528 {
1529 assert((n->is_MemBar() &&
1530 n->Opcode() == Op_MemBarRelease),
1531 "expecting a release membar");
1532
1533 MemBarNode *barrier = n->as_MemBar();
1534 if (!barrier->leading()) {
1535 return false;
1536 } else {
1537 Node* trailing = barrier->trailing_membar();
1538 MemBarNode* trailing_mb = trailing->as_MemBar();
1539 assert(trailing_mb->trailing(), "Not a trailing membar?");
1540 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1541
1542 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1543 if (mem->is_Store()) {
1544 assert(mem->as_Store()->is_release(), "");
1545 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1546 return true;
1547 } else {
1548 assert(mem->is_LoadStore(), "");
1549 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1550 return is_CAS(mem->Opcode(), true);
1551 }
1552 }
1553 return false;
1554 }
1555
1556 bool unnecessary_volatile(const Node *n)
1557 {
1558 // assert n->is_MemBar();
1559 MemBarNode *mbvol = n->as_MemBar();
1560
1561 bool release = mbvol->trailing_store();
1562 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1563 #ifdef ASSERT
1564 if (release) {
1565 Node* leading = mbvol->leading_membar();
1566 assert(leading->Opcode() == Op_MemBarRelease, "");
1567 assert(leading->as_MemBar()->leading_store(), "");
1568 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1569 }
1570 #endif
1571
1572 return release;
1573 }
1574
1575 // predicates controlling emit of str<x>/stlr<x>
1576
1577 bool needs_releasing_store(const Node *n)
1578 {
1579 // assert n->is_Store();
1580 StoreNode *st = n->as_Store();
1581 return st->trailing_membar() != NULL;
1582 }
1583
1584 // predicate controlling translation of CAS
1585 //
1586 // returns true if CAS needs to use an acquiring load otherwise false
1587
1588 bool needs_acquiring_load_exclusive(const Node *n)
1589 {
1590 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1591 LoadStoreNode* ldst = n->as_LoadStore();
1592 if (is_CAS(n->Opcode(), false)) {
1593 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1594 } else {
1595 return ldst->trailing_membar() != NULL;
1596 }
1597
1598 // so we can just return true here
1599 return true;
1600 }
1601
1602 #define __ _masm.
1603
1604 // advance declarations for helper functions to convert register
1605 // indices to register objects
1606
1607 // the ad file has to provide implementations of certain methods
1608 // expected by the generic code
1609 //
1610 // REQUIRED FUNCTIONALITY
1611
1612 //=============================================================================
1613
1614 // !!!!! Special hack to get all types of calls to specify the byte offset
1615 // from the start of the call to the point where the return address
1616 // will point.
1617
1618 int MachCallStaticJavaNode::ret_addr_offset()
1619 {
1620 // call should be a simple bl
1621 int off = 4;
1622 return off;
1623 }
1624
1625 int MachCallDynamicJavaNode::ret_addr_offset()
1626 {
1627 return 16; // movz, movk, movk, bl
1628 }
1629
1630 int MachCallRuntimeNode::ret_addr_offset() {
1631 // for generated stubs the call will be
1632 // bl(addr)
1633 // or with far branches
1634 // bl(trampoline_stub)
1635 // for real runtime callouts it will be six instructions
1636 // see aarch64_enc_java_to_runtime
1637 // adr(rscratch2, retaddr)
1638 // lea(rscratch1, RuntimeAddress(addr)
1639 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1640 // blr(rscratch1)
1641 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1642 if (cb) {
1643 return 1 * NativeInstruction::instruction_size;
1644 } else if (_entry_point == NULL) {
1645 // See CallLeafNoFPIndirect
1646 return 1 * NativeInstruction::instruction_size;
1647 } else {
1648 return 6 * NativeInstruction::instruction_size;
1649 }
1650 }
1651
1652 //=============================================================================
1653
1654 #ifndef PRODUCT
1655 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1656 st->print("BREAKPOINT");
1657 }
1658 #endif
1659
1660 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1661 C2_MacroAssembler _masm(&cbuf);
1662 __ brk(0);
1663 }
1664
1665 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1666 return MachNode::size(ra_);
1667 }
1668
1669 //=============================================================================
1670
1671 #ifndef PRODUCT
1672 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1673 st->print("nop \t# %d bytes pad for loops and calls", _count);
1674 }
1675 #endif
1676
1677 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1678 C2_MacroAssembler _masm(&cbuf);
1679 for (int i = 0; i < _count; i++) {
1680 __ nop();
1681 }
1682 }
1683
1684 uint MachNopNode::size(PhaseRegAlloc*) const {
1685 return _count * NativeInstruction::instruction_size;
1686 }
1687
1688 //=============================================================================
1689 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1690
1691 int ConstantTable::calculate_table_base_offset() const {
1692 return 0; // absolute addressing, no offset
1693 }
1694
1695 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1696 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1697 ShouldNotReachHere();
1698 }
1699
1700 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1701 // Empty encoding
1702 }
1703
1704 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1705 return 0;
1706 }
1707
1708 #ifndef PRODUCT
1709 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1710 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1711 }
1712 #endif
1713
1714 #ifndef PRODUCT
1715 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1716 Compile* C = ra_->C;
1717
1718 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1719
1720 if (C->output()->need_stack_bang(framesize))
1721 st->print("# stack bang size=%d\n\t", framesize);
1722
1723 if (VM_Version::use_rop_protection()) {
1724 st->print("ldr zr, [lr]\n\t");
1725 st->print("pacia lr, rfp\n\t");
1726 }
1727 if (framesize < ((1 << 9) + 2 * wordSize)) {
1728 st->print("sub sp, sp, #%d\n\t", framesize);
1729 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1730 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1731 } else {
1732 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1733 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1734 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1735 st->print("sub sp, sp, rscratch1");
1736 }
1737 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1738 st->print("\n\t");
1739 st->print("ldr rscratch1, [guard]\n\t");
1740 st->print("dmb ishld\n\t");
1741 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1742 st->print("cmp rscratch1, rscratch2\n\t");
1743 st->print("b.eq skip");
1744 st->print("\n\t");
1745 st->print("blr #nmethod_entry_barrier_stub\n\t");
1746 st->print("b skip\n\t");
1747 st->print("guard: int\n\t");
1748 st->print("\n\t");
1749 st->print("skip:\n\t");
1750 }
1751 }
1752 #endif
1753
1754 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1755 Compile* C = ra_->C;
1756 C2_MacroAssembler _masm(&cbuf);
1757
1758 // insert a nop at the start of the prolog so we can patch in a
1759 // branch if we need to invalidate the method later
1760 __ nop();
1761
1762 __ verified_entry(C, 0);
1763
1764 if (C->stub_function() == NULL) {
1765 __ entry_barrier();
1766 }
1767
1768 if (!Compile::current()->output()->in_scratch_emit_size()) {
1769 __ bind(*_verified_entry);
1770 }
1771
1772 if (VerifyStackAtCalls) {
1773 Unimplemented();
1774 }
1775
1776 C->output()->set_frame_complete(cbuf.insts_size());
1777
1778 if (C->has_mach_constant_base_node()) {
1779 // NOTE: We set the table base offset here because users might be
1780 // emitted before MachConstantBaseNode.
1781 ConstantTable& constant_table = C->output()->constant_table();
1782 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1783 }
1784 }
1785
1786 int MachPrologNode::reloc() const
1787 {
1788 return 0;
1789 }
1790
1791 //=============================================================================
1792
1793 #ifndef PRODUCT
1794 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1795 Compile* C = ra_->C;
1796 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1797
1798 st->print("# pop frame %d\n\t",framesize);
1799
1800 if (framesize == 0) {
1801 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1802 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1803 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1804 st->print("add sp, sp, #%d\n\t", framesize);
1805 } else {
1806 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1807 st->print("add sp, sp, rscratch1\n\t");
1808 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1809 }
1810 if (VM_Version::use_rop_protection()) {
1811 st->print("autia lr, rfp\n\t");
1812 st->print("ldr zr, [lr]\n\t");
1813 }
1814
1815 if (do_polling() && C->is_method_compilation()) {
1816 st->print("# test polling word\n\t");
1817 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1818 st->print("cmp sp, rscratch1\n\t");
1819 st->print("bhi #slow_path");
1820 }
1821 }
1822 #endif
1823
1824 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1825 Compile* C = ra_->C;
1826 C2_MacroAssembler _masm(&cbuf);
1827 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1828
1829 __ remove_frame(framesize, C->needs_stack_repair());
1830
1831 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1832 __ reserved_stack_check();
1833 }
1834
1835 if (do_polling() && C->is_method_compilation()) {
1836 Label dummy_label;
1837 Label* code_stub = &dummy_label;
1838 if (!C->output()->in_scratch_emit_size()) {
1839 code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
1840 }
1841 __ relocate(relocInfo::poll_return_type);
1842 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1843 }
1844 }
1845
1846 int MachEpilogNode::reloc() const {
1847 // Return number of relocatable values contained in this instruction.
1848 return 1; // 1 for polling page.
1849 }
1850
1851 const Pipeline * MachEpilogNode::pipeline() const {
1852 return MachNode::pipeline_class();
1853 }
1854
1855 //=============================================================================
1856
1857 // Figure out which register class each belongs in: rc_int, rc_float or
1858 // rc_stack.
1859 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1860
1861 static enum RC rc_class(OptoReg::Name reg) {
1862
1863 if (reg == OptoReg::Bad) {
1864 return rc_bad;
1865 }
1866
1867 // we have 32 int registers * 2 halves
1868 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1869
1870 if (reg < slots_of_int_registers) {
1871 return rc_int;
1872 }
1873
1874 // we have 32 float register * 8 halves
1875 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1876 if (reg < slots_of_int_registers + slots_of_float_registers) {
1877 return rc_float;
1878 }
1879
1880 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1881 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1882 return rc_predicate;
1883 }
1884
1885 // Between predicate regs & stack is the flags.
1886 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1887
1888 return rc_stack;
1889 }
1890
1891 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1892 Compile* C = ra_->C;
1893
1894 // Get registers to move.
1895 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1896 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1897 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1898 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1899
1900 enum RC src_hi_rc = rc_class(src_hi);
1901 enum RC src_lo_rc = rc_class(src_lo);
1902 enum RC dst_hi_rc = rc_class(dst_hi);
1903 enum RC dst_lo_rc = rc_class(dst_lo);
1904
1905 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1906
1907 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1908 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1909 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1910 "expected aligned-adjacent pairs");
1911 }
1912
1913 if (src_lo == dst_lo && src_hi == dst_hi) {
1914 return 0; // Self copy, no move.
1915 }
1916
1917 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1918 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1919 int src_offset = ra_->reg2offset(src_lo);
1920 int dst_offset = ra_->reg2offset(dst_lo);
1921
1922 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1923 uint ireg = ideal_reg();
1924 if (ireg == Op_VecA && cbuf) {
1925 C2_MacroAssembler _masm(cbuf);
1926 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1927 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1928 // stack->stack
1929 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1930 sve_vector_reg_size_in_bytes);
1931 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1932 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1933 sve_vector_reg_size_in_bytes);
1934 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1935 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1936 sve_vector_reg_size_in_bytes);
1937 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1938 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1939 as_FloatRegister(Matcher::_regEncode[src_lo]),
1940 as_FloatRegister(Matcher::_regEncode[src_lo]));
1941 } else {
1942 ShouldNotReachHere();
1943 }
1944 } else if (cbuf) {
1945 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1946 C2_MacroAssembler _masm(cbuf);
1947 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1948 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1949 // stack->stack
1950 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1951 if (ireg == Op_VecD) {
1952 __ unspill(rscratch1, true, src_offset);
1953 __ spill(rscratch1, true, dst_offset);
1954 } else {
1955 __ spill_copy128(src_offset, dst_offset);
1956 }
1957 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1958 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1959 ireg == Op_VecD ? __ T8B : __ T16B,
1960 as_FloatRegister(Matcher::_regEncode[src_lo]));
1961 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1962 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1963 ireg == Op_VecD ? __ D : __ Q,
1964 ra_->reg2offset(dst_lo));
1965 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1966 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1967 ireg == Op_VecD ? __ D : __ Q,
1968 ra_->reg2offset(src_lo));
1969 } else {
1970 ShouldNotReachHere();
1971 }
1972 }
1973 } else if (cbuf) {
1974 C2_MacroAssembler _masm(cbuf);
1975 switch (src_lo_rc) {
1976 case rc_int:
1977 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1978 if (is64) {
1979 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1980 as_Register(Matcher::_regEncode[src_lo]));
1981 } else {
1982 C2_MacroAssembler _masm(cbuf);
1983 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1984 as_Register(Matcher::_regEncode[src_lo]));
1985 }
1986 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1987 if (is64) {
1988 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1989 as_Register(Matcher::_regEncode[src_lo]));
1990 } else {
1991 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1992 as_Register(Matcher::_regEncode[src_lo]));
1993 }
1994 } else { // gpr --> stack spill
1995 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1996 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1997 }
1998 break;
1999 case rc_float:
2000 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2001 if (is64) {
2002 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2003 as_FloatRegister(Matcher::_regEncode[src_lo]));
2004 } else {
2005 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2006 as_FloatRegister(Matcher::_regEncode[src_lo]));
2007 }
2008 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2009 if (is64) {
2010 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2011 as_FloatRegister(Matcher::_regEncode[src_lo]));
2012 } else {
2013 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2014 as_FloatRegister(Matcher::_regEncode[src_lo]));
2015 }
2016 } else { // fpr --> stack spill
2017 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2018 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2019 is64 ? __ D : __ S, dst_offset);
2020 }
2021 break;
2022 case rc_stack:
2023 if (dst_lo_rc == rc_int) { // stack --> gpr load
2024 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2025 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2026 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2027 is64 ? __ D : __ S, src_offset);
2028 } else if (dst_lo_rc == rc_predicate) {
2029 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2030 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2031 } else { // stack --> stack copy
2032 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2033 if (ideal_reg() == Op_RegVectMask) {
2034 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2035 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2036 } else {
2037 __ unspill(rscratch1, is64, src_offset);
2038 __ spill(rscratch1, is64, dst_offset);
2039 }
2040 }
2041 break;
2042 case rc_predicate:
2043 if (dst_lo_rc == rc_predicate) {
2044 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2045 } else if (dst_lo_rc == rc_stack) {
2046 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2047 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2048 } else {
2049 assert(false, "bad src and dst rc_class combination.");
2050 ShouldNotReachHere();
2051 }
2052 break;
2053 default:
2054 assert(false, "bad rc_class for spill");
2055 ShouldNotReachHere();
2056 }
2057 }
2058
2059 if (st) {
2060 st->print("spill ");
2061 if (src_lo_rc == rc_stack) {
2062 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2063 } else {
2064 st->print("%s -> ", Matcher::regName[src_lo]);
2065 }
2066 if (dst_lo_rc == rc_stack) {
2067 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2068 } else {
2069 st->print("%s", Matcher::regName[dst_lo]);
2070 }
2071 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2072 int vsize = 0;
2073 switch (ideal_reg()) {
2074 case Op_VecD:
2075 vsize = 64;
2076 break;
2077 case Op_VecX:
2078 vsize = 128;
2079 break;
2080 case Op_VecA:
2081 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2082 break;
2083 default:
2084 assert(false, "bad register type for spill");
2085 ShouldNotReachHere();
2086 }
2087 st->print("\t# vector spill size = %d", vsize);
2088 } else if (ideal_reg() == Op_RegVectMask) {
2089 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2090 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2091 st->print("\t# predicate spill size = %d", vsize);
2092 } else {
2093 st->print("\t# spill size = %d", is64 ? 64 : 32);
2094 }
2095 }
2096
2097 return 0;
2098
2099 }
2100
2101 #ifndef PRODUCT
2102 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2103 if (!ra_)
2104 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2105 else
2106 implementation(NULL, ra_, false, st);
2107 }
2108 #endif
2109
2110 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2111 implementation(&cbuf, ra_, false, NULL);
2112 }
2113
2114 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2115 return MachNode::size(ra_);
2116 }
2117
2118 //=============================================================================
2119
2120 #ifndef PRODUCT
2121 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2122 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2123 int reg = ra_->get_reg_first(this);
2124 st->print("add %s, rsp, #%d]\t# box lock",
2125 Matcher::regName[reg], offset);
2126 }
2127 #endif
2128
2129 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2130 C2_MacroAssembler _masm(&cbuf);
2131
2132 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2133 int reg = ra_->get_encode(this);
2134
2135 // This add will handle any 24-bit signed offset. 24 bits allows an
2136 // 8 megabyte stack frame.
2137 __ add(as_Register(reg), sp, offset);
2138 }
2139
2140 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2141 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2142 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2143
2144 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2145 return NativeInstruction::instruction_size;
2146 } else {
2147 return 2 * NativeInstruction::instruction_size;
2148 }
2149 }
2150
2151 ///=============================================================================
2152 #ifndef PRODUCT
2153 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2154 {
2155 st->print_cr("# MachVEPNode");
2156 if (!_verified) {
2157 st->print_cr("\t load_class");
2158 } else {
2159 st->print_cr("\t unpack_inline_arg");
2160 }
2161 }
2162 #endif
2163
2164 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2165 {
2166 C2_MacroAssembler _masm(&cbuf);
2167
2168 if (!_verified) {
2169 Label skip;
2170 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2171 __ br(Assembler::EQ, skip);
2172 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2173 __ bind(skip);
2174
2175 } else {
2176 // insert a nop at the start of the prolog so we can patch in a
2177 // branch if we need to invalidate the method later
2178 __ nop();
2179
2180 // TODO 8284443 Avoid creation of temporary frame
2181 if (ra_->C->stub_function() == NULL) {
2182 __ verified_entry(ra_->C, 0);
2183 __ entry_barrier();
2184 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2185 __ remove_frame(framesize, false);
2186 }
2187 // Unpack inline type args passed as oop and then jump to
2188 // the verified entry point (skipping the unverified entry).
2189 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2190 // Emit code for verified entry and save increment for stack repair on return
2191 __ verified_entry(ra_->C, sp_inc);
2192 if (Compile::current()->output()->in_scratch_emit_size()) {
2193 Label dummy_verified_entry;
2194 __ b(dummy_verified_entry);
2195 } else {
2196 __ b(*_verified_entry);
2197 }
2198 }
2199 }
2200
2201 //=============================================================================
2202 #ifndef PRODUCT
2203 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2204 {
2205 st->print_cr("# MachUEPNode");
2206 if (UseCompressedClassPointers) {
2207 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2208 if (CompressedKlassPointers::shift() != 0) {
2209 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2210 }
2211 } else {
2212 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2213 }
2214 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2215 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2216 }
2217 #endif
2218
2219 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2220 {
2221 // This is the unverified entry point.
2222 C2_MacroAssembler _masm(&cbuf);
2223 Label skip;
2224
2225 // UseCompressedClassPointers logic are inside cmp_klass
2226 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2227
2228 // TODO
2229 // can we avoid this skip and still use a reloc?
2230 __ br(Assembler::EQ, skip);
2231 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2232 __ bind(skip);
2233 }
2234
2235 // REQUIRED EMIT CODE
2236
2237 //=============================================================================
2238
2239 // Emit exception handler code.
2240 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2241 {
2242 // mov rscratch1 #exception_blob_entry_point
2243 // br rscratch1
2244 // Note that the code buffer's insts_mark is always relative to insts.
2245 // That's why we must use the macroassembler to generate a handler.
2246 C2_MacroAssembler _masm(&cbuf);
2247 address base = __ start_a_stub(size_exception_handler());
2248 if (base == NULL) {
2249 ciEnv::current()->record_failure("CodeCache is full");
2250 return 0; // CodeBuffer::expand failed
2251 }
2252 int offset = __ offset();
2253 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2254 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2255 __ end_a_stub();
2256 return offset;
2257 }
2258
2259 // Emit deopt handler code.
2260 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2261 {
2262 // Note that the code buffer's insts_mark is always relative to insts.
2263 // That's why we must use the macroassembler to generate a handler.
2264 C2_MacroAssembler _masm(&cbuf);
2265 address base = __ start_a_stub(size_deopt_handler());
2266 if (base == NULL) {
2267 ciEnv::current()->record_failure("CodeCache is full");
2268 return 0; // CodeBuffer::expand failed
2269 }
2270 int offset = __ offset();
2271
2272 __ adr(lr, __ pc());
2273 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2274
2275 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2276 __ end_a_stub();
2277 return offset;
2278 }
2279
2280 // REQUIRED MATCHER CODE
2281
2282 //=============================================================================
2283
2284 const bool Matcher::match_rule_supported(int opcode) {
2285 if (!has_match_rule(opcode))
2286 return false;
2287
2288 bool ret_value = true;
2289 switch (opcode) {
2290 case Op_OnSpinWait:
2291 return VM_Version::supports_on_spin_wait();
2292 case Op_CacheWB:
2293 case Op_CacheWBPreSync:
2294 case Op_CacheWBPostSync:
2295 if (!VM_Version::supports_data_cache_line_flush()) {
2296 ret_value = false;
2297 }
2298 break;
2299 }
2300
2301 return ret_value; // Per default match rules are supported.
2302 }
2303
2304 const RegMask* Matcher::predicate_reg_mask(void) {
2305 return &_PR_REG_mask;
2306 }
2307
2308 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2309 return new TypeVectMask(elemTy, length);
2310 }
2311
2312 // Vector calling convention not yet implemented.
2313 const bool Matcher::supports_vector_calling_convention(void) {
2314 return false;
2315 }
2316
2317 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2318 Unimplemented();
2319 return OptoRegPair(0, 0);
2320 }
2321
2322 // Is this branch offset short enough that a short branch can be used?
2323 //
2324 // NOTE: If the platform does not provide any short branch variants, then
2325 // this method should return false for offset 0.
2326 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2327 // The passed offset is relative to address of the branch.
2328
2329 return (-32768 <= offset && offset < 32768);
2330 }
2331
2332 // Vector width in bytes.
2333 const int Matcher::vector_width_in_bytes(BasicType bt) {
2334 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2335 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2336 // Minimum 2 values in vector
2337 if (size < 2*type2aelembytes(bt)) size = 0;
2338 // But never < 4
2339 if (size < 4) size = 0;
2340 return size;
2341 }
2342
2343 // Limits on vector size (number of elements) loaded into vector.
2344 const int Matcher::max_vector_size(const BasicType bt) {
2345 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2346 }
2347
2348 const int Matcher::min_vector_size(const BasicType bt) {
2349 int max_size = max_vector_size(bt);
2350 // Limit the min vector size to 8 bytes.
2351 int size = 8 / type2aelembytes(bt);
2352 if (bt == T_BYTE) {
2353 // To support vector api shuffle/rearrange.
2354 size = 4;
2355 } else if (bt == T_BOOLEAN) {
2356 // To support vector api load/store mask.
2357 size = 2;
2358 }
2359 if (size < 2) size = 2;
2360 return MIN2(size, max_size);
2361 }
2362
2363 // Actual max scalable vector register length.
2364 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2365 return Matcher::max_vector_size(bt);
2366 }
2367
2368 // Vector ideal reg.
2369 const uint Matcher::vector_ideal_reg(int len) {
2370 if (UseSVE > 0 && 16 < len && len <= 256) {
2371 return Op_VecA;
2372 }
2373 switch(len) {
2374 // For 16-bit/32-bit mask vector, reuse VecD.
2375 case 2:
2376 case 4:
2377 case 8: return Op_VecD;
2378 case 16: return Op_VecX;
2379 }
2380 ShouldNotReachHere();
2381 return 0;
2382 }
2383
2384 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2385 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2386 switch (ideal_reg) {
2387 case Op_VecA: return new vecAOper();
2388 case Op_VecD: return new vecDOper();
2389 case Op_VecX: return new vecXOper();
2390 }
2391 ShouldNotReachHere();
2392 return NULL;
2393 }
2394
2395 bool Matcher::is_reg2reg_move(MachNode* m) {
2396 return false;
2397 }
2398
2399 bool Matcher::is_generic_vector(MachOper* opnd) {
2400 return opnd->opcode() == VREG;
2401 }
2402
2403 // Return whether or not this register is ever used as an argument.
2404 // This function is used on startup to build the trampoline stubs in
2405 // generateOptoStub. Registers not mentioned will be killed by the VM
2406 // call in the trampoline, and arguments in those registers not be
2407 // available to the callee.
2408 bool Matcher::can_be_java_arg(int reg)
2409 {
2410 return
2411 reg == R0_num || reg == R0_H_num ||
2412 reg == R1_num || reg == R1_H_num ||
2413 reg == R2_num || reg == R2_H_num ||
2414 reg == R3_num || reg == R3_H_num ||
2415 reg == R4_num || reg == R4_H_num ||
2416 reg == R5_num || reg == R5_H_num ||
2417 reg == R6_num || reg == R6_H_num ||
2418 reg == R7_num || reg == R7_H_num ||
2419 reg == V0_num || reg == V0_H_num ||
2420 reg == V1_num || reg == V1_H_num ||
2421 reg == V2_num || reg == V2_H_num ||
2422 reg == V3_num || reg == V3_H_num ||
2423 reg == V4_num || reg == V4_H_num ||
2424 reg == V5_num || reg == V5_H_num ||
2425 reg == V6_num || reg == V6_H_num ||
2426 reg == V7_num || reg == V7_H_num;
2427 }
2428
2429 bool Matcher::is_spillable_arg(int reg)
2430 {
2431 return can_be_java_arg(reg);
2432 }
2433
2434 uint Matcher::int_pressure_limit()
2435 {
2436 // JDK-8183543: When taking the number of available registers as int
2437 // register pressure threshold, the jtreg test:
2438 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2439 // failed due to C2 compilation failure with
2440 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2441 //
2442 // A derived pointer is live at CallNode and then is flagged by RA
2443 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2444 // derived pointers and lastly fail to spill after reaching maximum
2445 // number of iterations. Lowering the default pressure threshold to
2446 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2447 // a high register pressure area of the code so that split_DEF can
2448 // generate DefinitionSpillCopy for the derived pointer.
2449 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2450 if (!PreserveFramePointer) {
2451 // When PreserveFramePointer is off, frame pointer is allocatable,
2452 // but different from other SOC registers, it is excluded from
2453 // fatproj's mask because its save type is No-Save. Decrease 1 to
2454 // ensure high pressure at fatproj when PreserveFramePointer is off.
2455 // See check_pressure_at_fatproj().
2456 default_int_pressure_threshold--;
2457 }
2458 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2459 }
2460
2461 uint Matcher::float_pressure_limit()
2462 {
2463 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2464 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2465 }
2466
2467 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2468 return false;
2469 }
2470
2471 RegMask Matcher::divI_proj_mask() {
2472 ShouldNotReachHere();
2473 return RegMask();
2474 }
2475
2476 // Register for MODI projection of divmodI.
2477 RegMask Matcher::modI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for DIVL projection of divmodL.
2483 RegMask Matcher::divL_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for MODL projection of divmodL.
2489 RegMask Matcher::modL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2495 return FP_REG_mask();
2496 }
2497
2498 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2499 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2500 Node* u = addp->fast_out(i);
2501 if (u->is_LoadStore()) {
2502 // On AArch64, LoadStoreNodes (i.e. compare and swap
2503 // instructions) only take register indirect as an operand, so
2504 // any attempt to use an AddPNode as an input to a LoadStoreNode
2505 // must fail.
2506 return false;
2507 }
2508 if (u->is_Mem()) {
2509 int opsize = u->as_Mem()->memory_size();
2510 assert(opsize > 0, "unexpected memory operand size");
2511 if (u->as_Mem()->memory_size() != (1<<shift)) {
2512 return false;
2513 }
2514 }
2515 }
2516 return true;
2517 }
2518
2519 // Binary src (Replicate con)
2520 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2521 if (n == NULL || m == NULL) {
2522 return false;
2523 }
2524
2525 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2526 return false;
2527 }
2528
2529 Node* imm_node = m->in(1);
2530 if (!imm_node->is_Con()) {
2531 return false;
2532 }
2533
2534 const Type* t = imm_node->bottom_type();
2535 if (!(t->isa_int() || t->isa_long())) {
2536 return false;
2537 }
2538
2539 switch (n->Opcode()) {
2540 case Op_AndV:
2541 case Op_OrV:
2542 case Op_XorV: {
2543 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2544 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2545 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2546 }
2547 case Op_AddVB:
2548 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2549 case Op_AddVS:
2550 case Op_AddVI:
2551 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2552 case Op_AddVL:
2553 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2554 default:
2555 return false;
2556 }
2557 }
2558
2559 // (XorV src (Replicate m1))
2560 // (XorVMask src (MaskAll m1))
2561 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2562 if (n != NULL && m != NULL) {
2563 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2564 VectorNode::is_all_ones_vector(m);
2565 }
2566 return false;
2567 }
2568
2569 // Should the matcher clone input 'm' of node 'n'?
2570 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2571 if (is_vshift_con_pattern(n, m) ||
2572 is_vector_bitwise_not_pattern(n, m) ||
2573 is_valid_sve_arith_imm_pattern(n, m)) {
2574 mstack.push(m, Visit);
2575 return true;
2576 }
2577 return false;
2578 }
2579
2580 // Should the Matcher clone shifts on addressing modes, expecting them
2581 // to be subsumed into complex addressing expressions or compute them
2582 // into registers?
2583 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2584 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2585 return true;
2586 }
2587
2588 Node *off = m->in(AddPNode::Offset);
2589 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2590 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2591 // Are there other uses besides address expressions?
2592 !is_visited(off)) {
2593 address_visited.set(off->_idx); // Flag as address_visited
2594 mstack.push(off->in(2), Visit);
2595 Node *conv = off->in(1);
2596 if (conv->Opcode() == Op_ConvI2L &&
2597 // Are there other uses besides address expressions?
2598 !is_visited(conv)) {
2599 address_visited.set(conv->_idx); // Flag as address_visited
2600 mstack.push(conv->in(1), Pre_Visit);
2601 } else {
2602 mstack.push(conv, Pre_Visit);
2603 }
2604 address_visited.test_set(m->_idx); // Flag as address_visited
2605 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2606 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2607 return true;
2608 } else if (off->Opcode() == Op_ConvI2L &&
2609 // Are there other uses besides address expressions?
2610 !is_visited(off)) {
2611 address_visited.test_set(m->_idx); // Flag as address_visited
2612 address_visited.set(off->_idx); // Flag as address_visited
2613 mstack.push(off->in(1), Pre_Visit);
2614 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2615 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2616 return true;
2617 }
2618 return false;
2619 }
2620
2621 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2622 C2_MacroAssembler _masm(&cbuf); \
2623 { \
2624 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2625 guarantee(DISP == 0, "mode not permitted for volatile"); \
2626 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2627 __ INSN(REG, as_Register(BASE)); \
2628 }
2629
2630
2631 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2632 {
2633 Address::extend scale;
2634
2635 // Hooboy, this is fugly. We need a way to communicate to the
2636 // encoder that the index needs to be sign extended, so we have to
2637 // enumerate all the cases.
2638 switch (opcode) {
2639 case INDINDEXSCALEDI2L:
2640 case INDINDEXSCALEDI2LN:
2641 case INDINDEXI2L:
2642 case INDINDEXI2LN:
2643 scale = Address::sxtw(size);
2644 break;
2645 default:
2646 scale = Address::lsl(size);
2647 }
2648
2649 if (index == -1) {
2650 return Address(base, disp);
2651 } else {
2652 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2653 return Address(base, as_Register(index), scale);
2654 }
2655 }
2656
2657
2658 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2659 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2660 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2661 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2662 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2663
2664 // Used for all non-volatile memory accesses. The use of
2665 // $mem->opcode() to discover whether this pattern uses sign-extended
2666 // offsets is something of a kludge.
2667 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2668 Register reg, int opcode,
2669 Register base, int index, int scale, int disp,
2670 int size_in_memory)
2671 {
2672 Address addr = mem2address(opcode, base, index, scale, disp);
2673 if (addr.getMode() == Address::base_plus_offset) {
2674 /* If we get an out-of-range offset it is a bug in the compiler,
2675 so we assert here. */
2676 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2677 "c2 compiler bug");
2678 /* Fix up any out-of-range offsets. */
2679 assert_different_registers(rscratch1, base);
2680 assert_different_registers(rscratch1, reg);
2681 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2682 }
2683 (masm.*insn)(reg, addr);
2684 }
2685
2686 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2687 FloatRegister reg, int opcode,
2688 Register base, int index, int size, int disp,
2689 int size_in_memory)
2690 {
2691 Address::extend scale;
2692
2693 switch (opcode) {
2694 case INDINDEXSCALEDI2L:
2695 case INDINDEXSCALEDI2LN:
2696 scale = Address::sxtw(size);
2697 break;
2698 default:
2699 scale = Address::lsl(size);
2700 }
2701
2702 if (index == -1) {
2703 /* If we get an out-of-range offset it is a bug in the compiler,
2704 so we assert here. */
2705 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2706 /* Fix up any out-of-range offsets. */
2707 assert_different_registers(rscratch1, base);
2708 Address addr = Address(base, disp);
2709 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2710 (masm.*insn)(reg, addr);
2711 } else {
2712 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2713 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2714 }
2715 }
2716
2717 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2718 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2719 int opcode, Register base, int index, int size, int disp)
2720 {
2721 if (index == -1) {
2722 (masm.*insn)(reg, T, Address(base, disp));
2723 } else {
2724 assert(disp == 0, "unsupported address mode");
2725 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2726 }
2727 }
2728
2729 %}
2730
2731
2732
2733 //----------ENCODING BLOCK-----------------------------------------------------
2734 // This block specifies the encoding classes used by the compiler to
2735 // output byte streams. Encoding classes are parameterized macros
2736 // used by Machine Instruction Nodes in order to generate the bit
2737 // encoding of the instruction. Operands specify their base encoding
2738 // interface with the interface keyword. There are currently
2739 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2740 // COND_INTER. REG_INTER causes an operand to generate a function
2741 // which returns its register number when queried. CONST_INTER causes
2742 // an operand to generate a function which returns the value of the
2743 // constant when queried. MEMORY_INTER causes an operand to generate
2744 // four functions which return the Base Register, the Index Register,
2745 // the Scale Value, and the Offset Value of the operand when queried.
2746 // COND_INTER causes an operand to generate six functions which return
2747 // the encoding code (ie - encoding bits for the instruction)
2748 // associated with each basic boolean condition for a conditional
2749 // instruction.
2750 //
2751 // Instructions specify two basic values for encoding. Again, a
2752 // function is available to check if the constant displacement is an
2753 // oop. They use the ins_encode keyword to specify their encoding
2754 // classes (which must be a sequence of enc_class names, and their
2755 // parameters, specified in the encoding block), and they use the
2756 // opcode keyword to specify, in order, their primary, secondary, and
2757 // tertiary opcode. Only the opcode sections which a particular
2758 // instruction needs for encoding need to be specified.
2759 encode %{
2760 // Build emit functions for each basic byte or larger field in the
2761 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2762 // from C++ code in the enc_class source block. Emit functions will
2763 // live in the main source block for now. In future, we can
2764 // generalize this by adding a syntax that specifies the sizes of
2765 // fields in an order, so that the adlc can build the emit functions
2766 // automagically
2767
2768 // catch all for unimplemented encodings
2769 enc_class enc_unimplemented %{
2770 C2_MacroAssembler _masm(&cbuf);
2771 __ unimplemented("C2 catch all");
2772 %}
2773
2774 // BEGIN Non-volatile memory access
2775
2776 // This encoding class is generated automatically from ad_encode.m4.
2777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2778 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2779 Register dst_reg = as_Register($dst$$reg);
2780 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2781 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2782 %}
2783
2784 // This encoding class is generated automatically from ad_encode.m4.
2785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2786 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2787 Register dst_reg = as_Register($dst$$reg);
2788 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2789 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2790 %}
2791
2792 // This encoding class is generated automatically from ad_encode.m4.
2793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2794 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2795 Register dst_reg = as_Register($dst$$reg);
2796 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2797 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2798 %}
2799
2800 // This encoding class is generated automatically from ad_encode.m4.
2801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2802 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2803 Register dst_reg = as_Register($dst$$reg);
2804 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2805 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2806 %}
2807
2808 // This encoding class is generated automatically from ad_encode.m4.
2809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2810 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2811 Register dst_reg = as_Register($dst$$reg);
2812 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2813 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2814 %}
2815
2816 // This encoding class is generated automatically from ad_encode.m4.
2817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2818 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2819 Register dst_reg = as_Register($dst$$reg);
2820 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2821 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2822 %}
2823
2824 // This encoding class is generated automatically from ad_encode.m4.
2825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2826 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2827 Register dst_reg = as_Register($dst$$reg);
2828 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2829 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2830 %}
2831
2832 // This encoding class is generated automatically from ad_encode.m4.
2833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2834 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2835 Register dst_reg = as_Register($dst$$reg);
2836 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2837 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2838 %}
2839
2840 // This encoding class is generated automatically from ad_encode.m4.
2841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2842 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2843 Register dst_reg = as_Register($dst$$reg);
2844 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2845 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2846 %}
2847
2848 // This encoding class is generated automatically from ad_encode.m4.
2849 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2850 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2851 Register dst_reg = as_Register($dst$$reg);
2852 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2853 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2854 %}
2855
2856 // This encoding class is generated automatically from ad_encode.m4.
2857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2858 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2859 Register dst_reg = as_Register($dst$$reg);
2860 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2861 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2862 %}
2863
2864 // This encoding class is generated automatically from ad_encode.m4.
2865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2866 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2867 Register dst_reg = as_Register($dst$$reg);
2868 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2869 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2870 %}
2871
2872 // This encoding class is generated automatically from ad_encode.m4.
2873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2874 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2875 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2876 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2877 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2878 %}
2879
2880 // This encoding class is generated automatically from ad_encode.m4.
2881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2882 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2883 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2884 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2885 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2886 %}
2887
2888 // This encoding class is generated automatically from ad_encode.m4.
2889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2890 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2891 Register src_reg = as_Register($src$$reg);
2892 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2893 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2894 %}
2895
2896 // This encoding class is generated automatically from ad_encode.m4.
2897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2898 enc_class aarch64_enc_strb0(memory1 mem) %{
2899 C2_MacroAssembler _masm(&cbuf);
2900 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2901 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2902 %}
2903
2904 // This encoding class is generated automatically from ad_encode.m4.
2905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2906 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2907 Register src_reg = as_Register($src$$reg);
2908 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2909 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2910 %}
2911
2912 // This encoding class is generated automatically from ad_encode.m4.
2913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2914 enc_class aarch64_enc_strh0(memory2 mem) %{
2915 C2_MacroAssembler _masm(&cbuf);
2916 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2917 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2918 %}
2919
2920 // This encoding class is generated automatically from ad_encode.m4.
2921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2922 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2923 Register src_reg = as_Register($src$$reg);
2924 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2925 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2926 %}
2927
2928 // This encoding class is generated automatically from ad_encode.m4.
2929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2930 enc_class aarch64_enc_strw0(memory4 mem) %{
2931 C2_MacroAssembler _masm(&cbuf);
2932 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2933 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2934 %}
2935
2936 // This encoding class is generated automatically from ad_encode.m4.
2937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2938 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2939 Register src_reg = as_Register($src$$reg);
2940 // we sometimes get asked to store the stack pointer into the
2941 // current thread -- we cannot do that directly on AArch64
2942 if (src_reg == r31_sp) {
2943 C2_MacroAssembler _masm(&cbuf);
2944 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2945 __ mov(rscratch2, sp);
2946 src_reg = rscratch2;
2947 }
2948 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2949 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2950 %}
2951
2952 // This encoding class is generated automatically from ad_encode.m4.
2953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2954 enc_class aarch64_enc_str0(memory8 mem) %{
2955 C2_MacroAssembler _masm(&cbuf);
2956 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2963 FloatRegister src_reg = as_FloatRegister($src$$reg);
2964 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2971 FloatRegister src_reg = as_FloatRegister($src$$reg);
2972 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2973 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2974 %}
2975
2976 // This encoding class is generated automatically from ad_encode.m4.
2977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2978 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2979 C2_MacroAssembler _masm(&cbuf);
2980 __ membar(Assembler::StoreStore);
2981 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2983 %}
2984
2985 // END Non-volatile memory access
2986
2987 // Vector loads and stores
2988 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
2989 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2990 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
2991 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2992 %}
2993
2994 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
2995 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2996 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2997 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2998 %}
2999
3000 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3001 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3002 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3003 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3004 %}
3005
3006 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3007 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3008 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3009 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3010 %}
3011
3012 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3013 FloatRegister src_reg = as_FloatRegister($src$$reg);
3014 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3015 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3016 %}
3017
3018 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3019 FloatRegister src_reg = as_FloatRegister($src$$reg);
3020 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3021 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3022 %}
3023
3024 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3025 FloatRegister src_reg = as_FloatRegister($src$$reg);
3026 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3031 FloatRegister src_reg = as_FloatRegister($src$$reg);
3032 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 // volatile loads and stores
3037
3038 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3039 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3040 rscratch1, stlrb);
3041 %}
3042
3043 enc_class aarch64_enc_stlrb0(memory mem) %{
3044 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3045 rscratch1, stlrb);
3046 %}
3047
3048 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3049 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3050 rscratch1, stlrh);
3051 %}
3052
3053 enc_class aarch64_enc_stlrh0(memory mem) %{
3054 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3055 rscratch1, stlrh);
3056 %}
3057
3058 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3059 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3060 rscratch1, stlrw);
3061 %}
3062
3063 enc_class aarch64_enc_stlrw0(memory mem) %{
3064 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3065 rscratch1, stlrw);
3066 %}
3067
3068 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3069 Register dst_reg = as_Register($dst$$reg);
3070 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3071 rscratch1, ldarb);
3072 __ sxtbw(dst_reg, dst_reg);
3073 %}
3074
3075 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3076 Register dst_reg = as_Register($dst$$reg);
3077 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3078 rscratch1, ldarb);
3079 __ sxtb(dst_reg, dst_reg);
3080 %}
3081
3082 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3083 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3084 rscratch1, ldarb);
3085 %}
3086
3087 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3088 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3089 rscratch1, ldarb);
3090 %}
3091
3092 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3093 Register dst_reg = as_Register($dst$$reg);
3094 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3095 rscratch1, ldarh);
3096 __ sxthw(dst_reg, dst_reg);
3097 %}
3098
3099 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3100 Register dst_reg = as_Register($dst$$reg);
3101 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3102 rscratch1, ldarh);
3103 __ sxth(dst_reg, dst_reg);
3104 %}
3105
3106 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3107 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3108 rscratch1, ldarh);
3109 %}
3110
3111 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3112 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3113 rscratch1, ldarh);
3114 %}
3115
3116 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3117 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3118 rscratch1, ldarw);
3119 %}
3120
3121 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3122 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3123 rscratch1, ldarw);
3124 %}
3125
3126 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3127 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3128 rscratch1, ldar);
3129 %}
3130
3131 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3132 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, ldarw);
3134 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3135 %}
3136
3137 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3138 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3139 rscratch1, ldar);
3140 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3141 %}
3142
3143 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3144 Register src_reg = as_Register($src$$reg);
3145 // we sometimes get asked to store the stack pointer into the
3146 // current thread -- we cannot do that directly on AArch64
3147 if (src_reg == r31_sp) {
3148 C2_MacroAssembler _masm(&cbuf);
3149 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3150 __ mov(rscratch2, sp);
3151 src_reg = rscratch2;
3152 }
3153 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, stlr);
3155 %}
3156
3157 enc_class aarch64_enc_stlr0(memory mem) %{
3158 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlr);
3160 %}
3161
3162 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3163 {
3164 C2_MacroAssembler _masm(&cbuf);
3165 FloatRegister src_reg = as_FloatRegister($src$$reg);
3166 __ fmovs(rscratch2, src_reg);
3167 }
3168 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrw);
3170 %}
3171
3172 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3173 {
3174 C2_MacroAssembler _masm(&cbuf);
3175 FloatRegister src_reg = as_FloatRegister($src$$reg);
3176 __ fmovd(rscratch2, src_reg);
3177 }
3178 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, stlr);
3180 %}
3181
3182 // synchronized read/update encodings
3183
3184 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3185 C2_MacroAssembler _masm(&cbuf);
3186 Register dst_reg = as_Register($dst$$reg);
3187 Register base = as_Register($mem$$base);
3188 int index = $mem$$index;
3189 int scale = $mem$$scale;
3190 int disp = $mem$$disp;
3191 if (index == -1) {
3192 if (disp != 0) {
3193 __ lea(rscratch1, Address(base, disp));
3194 __ ldaxr(dst_reg, rscratch1);
3195 } else {
3196 // TODO
3197 // should we ever get anything other than this case?
3198 __ ldaxr(dst_reg, base);
3199 }
3200 } else {
3201 Register index_reg = as_Register(index);
3202 if (disp == 0) {
3203 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3204 __ ldaxr(dst_reg, rscratch1);
3205 } else {
3206 __ lea(rscratch1, Address(base, disp));
3207 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3208 __ ldaxr(dst_reg, rscratch1);
3209 }
3210 }
3211 %}
3212
3213 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3214 C2_MacroAssembler _masm(&cbuf);
3215 Register src_reg = as_Register($src$$reg);
3216 Register base = as_Register($mem$$base);
3217 int index = $mem$$index;
3218 int scale = $mem$$scale;
3219 int disp = $mem$$disp;
3220 if (index == -1) {
3221 if (disp != 0) {
3222 __ lea(rscratch2, Address(base, disp));
3223 __ stlxr(rscratch1, src_reg, rscratch2);
3224 } else {
3225 // TODO
3226 // should we ever get anything other than this case?
3227 __ stlxr(rscratch1, src_reg, base);
3228 }
3229 } else {
3230 Register index_reg = as_Register(index);
3231 if (disp == 0) {
3232 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3233 __ stlxr(rscratch1, src_reg, rscratch2);
3234 } else {
3235 __ lea(rscratch2, Address(base, disp));
3236 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3237 __ stlxr(rscratch1, src_reg, rscratch2);
3238 }
3239 }
3240 __ cmpw(rscratch1, zr);
3241 %}
3242
3243 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3244 C2_MacroAssembler _masm(&cbuf);
3245 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3246 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3247 Assembler::xword, /*acquire*/ false, /*release*/ true,
3248 /*weak*/ false, noreg);
3249 %}
3250
3251 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3252 C2_MacroAssembler _masm(&cbuf);
3253 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3254 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3255 Assembler::word, /*acquire*/ false, /*release*/ true,
3256 /*weak*/ false, noreg);
3257 %}
3258
3259 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3260 C2_MacroAssembler _masm(&cbuf);
3261 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3262 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3263 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3264 /*weak*/ false, noreg);
3265 %}
3266
3267 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3268 C2_MacroAssembler _masm(&cbuf);
3269 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3270 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3271 Assembler::byte, /*acquire*/ false, /*release*/ true,
3272 /*weak*/ false, noreg);
3273 %}
3274
3275
3276 // The only difference between aarch64_enc_cmpxchg and
3277 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3278 // CompareAndSwap sequence to serve as a barrier on acquiring a
3279 // lock.
3280 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3281 C2_MacroAssembler _masm(&cbuf);
3282 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3283 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3284 Assembler::xword, /*acquire*/ true, /*release*/ true,
3285 /*weak*/ false, noreg);
3286 %}
3287
3288 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3289 C2_MacroAssembler _masm(&cbuf);
3290 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3291 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3292 Assembler::word, /*acquire*/ true, /*release*/ true,
3293 /*weak*/ false, noreg);
3294 %}
3295
3296 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3297 C2_MacroAssembler _masm(&cbuf);
3298 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3299 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3300 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3301 /*weak*/ false, noreg);
3302 %}
3303
3304 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3305 C2_MacroAssembler _masm(&cbuf);
3306 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3307 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3308 Assembler::byte, /*acquire*/ true, /*release*/ true,
3309 /*weak*/ false, noreg);
3310 %}
3311
3312 // auxiliary used for CompareAndSwapX to set result register
3313 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3314 C2_MacroAssembler _masm(&cbuf);
3315 Register res_reg = as_Register($res$$reg);
3316 __ cset(res_reg, Assembler::EQ);
3317 %}
3318
3319 // prefetch encodings
3320
3321 enc_class aarch64_enc_prefetchw(memory mem) %{
3322 C2_MacroAssembler _masm(&cbuf);
3323 Register base = as_Register($mem$$base);
3324 int index = $mem$$index;
3325 int scale = $mem$$scale;
3326 int disp = $mem$$disp;
3327 if (index == -1) {
3328 __ prfm(Address(base, disp), PSTL1KEEP);
3329 } else {
3330 Register index_reg = as_Register(index);
3331 if (disp == 0) {
3332 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3333 } else {
3334 __ lea(rscratch1, Address(base, disp));
3335 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3336 }
3337 }
3338 %}
3339
3340 /// mov envcodings
3341
3342 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3343 C2_MacroAssembler _masm(&cbuf);
3344 uint32_t con = (uint32_t)$src$$constant;
3345 Register dst_reg = as_Register($dst$$reg);
3346 if (con == 0) {
3347 __ movw(dst_reg, zr);
3348 } else {
3349 __ movw(dst_reg, con);
3350 }
3351 %}
3352
3353 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3354 C2_MacroAssembler _masm(&cbuf);
3355 Register dst_reg = as_Register($dst$$reg);
3356 uint64_t con = (uint64_t)$src$$constant;
3357 if (con == 0) {
3358 __ mov(dst_reg, zr);
3359 } else {
3360 __ mov(dst_reg, con);
3361 }
3362 %}
3363
3364 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3365 C2_MacroAssembler _masm(&cbuf);
3366 Register dst_reg = as_Register($dst$$reg);
3367 address con = (address)$src$$constant;
3368 if (con == NULL || con == (address)1) {
3369 ShouldNotReachHere();
3370 } else {
3371 relocInfo::relocType rtype = $src->constant_reloc();
3372 if (rtype == relocInfo::oop_type) {
3373 __ movoop(dst_reg, (jobject)con);
3374 } else if (rtype == relocInfo::metadata_type) {
3375 __ mov_metadata(dst_reg, (Metadata*)con);
3376 } else {
3377 assert(rtype == relocInfo::none, "unexpected reloc type");
3378 if (con < (address)(uintptr_t)os::vm_page_size()) {
3379 __ mov(dst_reg, con);
3380 } else {
3381 uint64_t offset;
3382 __ adrp(dst_reg, con, offset);
3383 __ add(dst_reg, dst_reg, offset);
3384 }
3385 }
3386 }
3387 %}
3388
3389 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3390 C2_MacroAssembler _masm(&cbuf);
3391 Register dst_reg = as_Register($dst$$reg);
3392 __ mov(dst_reg, zr);
3393 %}
3394
3395 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3396 C2_MacroAssembler _masm(&cbuf);
3397 Register dst_reg = as_Register($dst$$reg);
3398 __ mov(dst_reg, (uint64_t)1);
3399 %}
3400
3401 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3402 C2_MacroAssembler _masm(&cbuf);
3403 __ load_byte_map_base($dst$$Register);
3404 %}
3405
3406 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3407 C2_MacroAssembler _masm(&cbuf);
3408 Register dst_reg = as_Register($dst$$reg);
3409 address con = (address)$src$$constant;
3410 if (con == NULL) {
3411 ShouldNotReachHere();
3412 } else {
3413 relocInfo::relocType rtype = $src->constant_reloc();
3414 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3415 __ set_narrow_oop(dst_reg, (jobject)con);
3416 }
3417 %}
3418
3419 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3420 C2_MacroAssembler _masm(&cbuf);
3421 Register dst_reg = as_Register($dst$$reg);
3422 __ mov(dst_reg, zr);
3423 %}
3424
3425 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3426 C2_MacroAssembler _masm(&cbuf);
3427 Register dst_reg = as_Register($dst$$reg);
3428 address con = (address)$src$$constant;
3429 if (con == NULL) {
3430 ShouldNotReachHere();
3431 } else {
3432 relocInfo::relocType rtype = $src->constant_reloc();
3433 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3434 __ set_narrow_klass(dst_reg, (Klass *)con);
3435 }
3436 %}
3437
3438 // arithmetic encodings
3439
3440 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3441 C2_MacroAssembler _masm(&cbuf);
3442 Register dst_reg = as_Register($dst$$reg);
3443 Register src_reg = as_Register($src1$$reg);
3444 int32_t con = (int32_t)$src2$$constant;
3445 // add has primary == 0, subtract has primary == 1
3446 if ($primary) { con = -con; }
3447 if (con < 0) {
3448 __ subw(dst_reg, src_reg, -con);
3449 } else {
3450 __ addw(dst_reg, src_reg, con);
3451 }
3452 %}
3453
3454 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3455 C2_MacroAssembler _masm(&cbuf);
3456 Register dst_reg = as_Register($dst$$reg);
3457 Register src_reg = as_Register($src1$$reg);
3458 int32_t con = (int32_t)$src2$$constant;
3459 // add has primary == 0, subtract has primary == 1
3460 if ($primary) { con = -con; }
3461 if (con < 0) {
3462 __ sub(dst_reg, src_reg, -con);
3463 } else {
3464 __ add(dst_reg, src_reg, con);
3465 }
3466 %}
3467
3468 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3469 C2_MacroAssembler _masm(&cbuf);
3470 Register dst_reg = as_Register($dst$$reg);
3471 Register src1_reg = as_Register($src1$$reg);
3472 Register src2_reg = as_Register($src2$$reg);
3473 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3474 %}
3475
3476 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3477 C2_MacroAssembler _masm(&cbuf);
3478 Register dst_reg = as_Register($dst$$reg);
3479 Register src1_reg = as_Register($src1$$reg);
3480 Register src2_reg = as_Register($src2$$reg);
3481 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3482 %}
3483
3484 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3485 C2_MacroAssembler _masm(&cbuf);
3486 Register dst_reg = as_Register($dst$$reg);
3487 Register src1_reg = as_Register($src1$$reg);
3488 Register src2_reg = as_Register($src2$$reg);
3489 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3490 %}
3491
3492 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3493 C2_MacroAssembler _masm(&cbuf);
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3498 %}
3499
3500 // compare instruction encodings
3501
3502 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3503 C2_MacroAssembler _masm(&cbuf);
3504 Register reg1 = as_Register($src1$$reg);
3505 Register reg2 = as_Register($src2$$reg);
3506 __ cmpw(reg1, reg2);
3507 %}
3508
3509 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3510 C2_MacroAssembler _masm(&cbuf);
3511 Register reg = as_Register($src1$$reg);
3512 int32_t val = $src2$$constant;
3513 if (val >= 0) {
3514 __ subsw(zr, reg, val);
3515 } else {
3516 __ addsw(zr, reg, -val);
3517 }
3518 %}
3519
3520 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3521 C2_MacroAssembler _masm(&cbuf);
3522 Register reg1 = as_Register($src1$$reg);
3523 uint32_t val = (uint32_t)$src2$$constant;
3524 __ movw(rscratch1, val);
3525 __ cmpw(reg1, rscratch1);
3526 %}
3527
3528 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3529 C2_MacroAssembler _masm(&cbuf);
3530 Register reg1 = as_Register($src1$$reg);
3531 Register reg2 = as_Register($src2$$reg);
3532 __ cmp(reg1, reg2);
3533 %}
3534
3535 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3536 C2_MacroAssembler _masm(&cbuf);
3537 Register reg = as_Register($src1$$reg);
3538 int64_t val = $src2$$constant;
3539 if (val >= 0) {
3540 __ subs(zr, reg, val);
3541 } else if (val != -val) {
3542 __ adds(zr, reg, -val);
3543 } else {
3544 // aargh, Long.MIN_VALUE is a special case
3545 __ orr(rscratch1, zr, (uint64_t)val);
3546 __ subs(zr, reg, rscratch1);
3547 }
3548 %}
3549
3550 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3551 C2_MacroAssembler _masm(&cbuf);
3552 Register reg1 = as_Register($src1$$reg);
3553 uint64_t val = (uint64_t)$src2$$constant;
3554 __ mov(rscratch1, val);
3555 __ cmp(reg1, rscratch1);
3556 %}
3557
3558 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3559 C2_MacroAssembler _masm(&cbuf);
3560 Register reg1 = as_Register($src1$$reg);
3561 Register reg2 = as_Register($src2$$reg);
3562 __ cmp(reg1, reg2);
3563 %}
3564
3565 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3566 C2_MacroAssembler _masm(&cbuf);
3567 Register reg1 = as_Register($src1$$reg);
3568 Register reg2 = as_Register($src2$$reg);
3569 __ cmpw(reg1, reg2);
3570 %}
3571
3572 enc_class aarch64_enc_testp(iRegP src) %{
3573 C2_MacroAssembler _masm(&cbuf);
3574 Register reg = as_Register($src$$reg);
3575 __ cmp(reg, zr);
3576 %}
3577
3578 enc_class aarch64_enc_testn(iRegN src) %{
3579 C2_MacroAssembler _masm(&cbuf);
3580 Register reg = as_Register($src$$reg);
3581 __ cmpw(reg, zr);
3582 %}
3583
3584 enc_class aarch64_enc_b(label lbl) %{
3585 C2_MacroAssembler _masm(&cbuf);
3586 Label *L = $lbl$$label;
3587 __ b(*L);
3588 %}
3589
3590 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3591 C2_MacroAssembler _masm(&cbuf);
3592 Label *L = $lbl$$label;
3593 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3594 %}
3595
3596 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3597 C2_MacroAssembler _masm(&cbuf);
3598 Label *L = $lbl$$label;
3599 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3600 %}
3601
3602 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3603 %{
3604 Register sub_reg = as_Register($sub$$reg);
3605 Register super_reg = as_Register($super$$reg);
3606 Register temp_reg = as_Register($temp$$reg);
3607 Register result_reg = as_Register($result$$reg);
3608
3609 Label miss;
3610 C2_MacroAssembler _masm(&cbuf);
3611 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3612 NULL, &miss,
3613 /*set_cond_codes:*/ true);
3614 if ($primary) {
3615 __ mov(result_reg, zr);
3616 }
3617 __ bind(miss);
3618 %}
3619
3620 enc_class aarch64_enc_java_static_call(method meth) %{
3621 C2_MacroAssembler _masm(&cbuf);
3622
3623 address addr = (address)$meth$$method;
3624 address call;
3625 if (!_method) {
3626 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3627 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3628 if (call == NULL) {
3629 ciEnv::current()->record_failure("CodeCache is full");
3630 return;
3631 }
3632 } else {
3633 int method_index = resolved_method_index(cbuf);
3634 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3635 : static_call_Relocation::spec(method_index);
3636 call = __ trampoline_call(Address(addr, rspec));
3637 if (call == NULL) {
3638 ciEnv::current()->record_failure("CodeCache is full");
3639 return;
3640 }
3641 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3642 // Calls of the same statically bound method can share
3643 // a stub to the interpreter.
3644 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3645 } else {
3646 // Emit stub for static call
3647 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3648 if (stub == NULL) {
3649 ciEnv::current()->record_failure("CodeCache is full");
3650 return;
3651 }
3652 }
3653 }
3654
3655 __ post_call_nop();
3656
3657 // Only non uncommon_trap calls need to reinitialize ptrue.
3658 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3659 __ reinitialize_ptrue();
3660 }
3661 %}
3662
3663 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3664 C2_MacroAssembler _masm(&cbuf);
3665 int method_index = resolved_method_index(cbuf);
3666 address call = __ ic_call((address)$meth$$method, method_index);
3667 if (call == NULL) {
3668 ciEnv::current()->record_failure("CodeCache is full");
3669 return;
3670 }
3671 _masm.clear_inst_mark();
3672 __ post_call_nop();
3673 if (Compile::current()->max_vector_size() > 0) {
3674 __ reinitialize_ptrue();
3675 }
3676 %}
3677
3678 enc_class aarch64_enc_call_epilog() %{
3679 C2_MacroAssembler _masm(&cbuf);
3680 if (VerifyStackAtCalls) {
3681 // Check that stack depth is unchanged: find majik cookie on stack
3682 __ call_Unimplemented();
3683 }
3684 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3685 if (!_method->signature()->returns_null_free_inline_type()) {
3686 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3687 // Search for the corresponding projection, get the register and emit code that initialized it.
3688 uint con = (tf()->range_cc()->cnt() - 1);
3689 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3690 ProjNode* proj = fast_out(i)->as_Proj();
3691 if (proj->_con == con) {
3692 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3693 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3694 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3695 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3696 __ cmp(r0, zr);
3697 __ cset(toReg, Assembler::NE);
3698 if (reg->is_stack()) {
3699 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3700 __ str(toReg, Address(sp, st_off));
3701 }
3702 break;
3703 }
3704 }
3705 }
3706 if (return_value_is_used()) {
3707 // An inline type is returned as fields in multiple registers.
3708 // R0 either contains an oop if the inline type is buffered or a pointer
3709 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3710 // if the lowest bit is set to allow C2 to use the oop after null checking.
3711 // r0 &= (r0 & 1) - 1
3712 __ andr(rscratch1, r0, 0x1);
3713 __ sub(rscratch1, rscratch1, 0x1);
3714 __ andr(r0, r0, rscratch1);
3715 }
3716 }
3717 %}
3718
3719 enc_class aarch64_enc_java_to_runtime(method meth) %{
3720 C2_MacroAssembler _masm(&cbuf);
3721
3722 // some calls to generated routines (arraycopy code) are scheduled
3723 // by C2 as runtime calls. if so we can call them using a br (they
3724 // will be in a reachable segment) otherwise we have to use a blr
3725 // which loads the absolute address into a register.
3726 address entry = (address)$meth$$method;
3727 CodeBlob *cb = CodeCache::find_blob(entry);
3728 if (cb) {
3729 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3730 if (call == NULL) {
3731 ciEnv::current()->record_failure("CodeCache is full");
3732 return;
3733 }
3734 __ post_call_nop();
3735 } else {
3736 Label retaddr;
3737 __ adr(rscratch2, retaddr);
3738 __ lea(rscratch1, RuntimeAddress(entry));
3739 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3740 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3741 __ blr(rscratch1);
3742 __ bind(retaddr);
3743 __ post_call_nop();
3744 __ add(sp, sp, 2 * wordSize);
3745 }
3746 if (Compile::current()->max_vector_size() > 0) {
3747 __ reinitialize_ptrue();
3748 }
3749 %}
3750
3751 enc_class aarch64_enc_rethrow() %{
3752 C2_MacroAssembler _masm(&cbuf);
3753 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3754 %}
3755
3756 enc_class aarch64_enc_ret() %{
3757 C2_MacroAssembler _masm(&cbuf);
3758 #ifdef ASSERT
3759 if (Compile::current()->max_vector_size() > 0) {
3760 __ verify_ptrue();
3761 }
3762 #endif
3763 __ ret(lr);
3764 %}
3765
3766 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3767 C2_MacroAssembler _masm(&cbuf);
3768 Register target_reg = as_Register($jump_target$$reg);
3769 __ br(target_reg);
3770 %}
3771
3772 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3773 C2_MacroAssembler _masm(&cbuf);
3774 Register target_reg = as_Register($jump_target$$reg);
3775 // exception oop should be in r0
3776 // ret addr has been popped into lr
3777 // callee expects it in r3
3778 __ mov(r3, lr);
3779 __ br(target_reg);
3780 %}
3781
3782 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3783 C2_MacroAssembler _masm(&cbuf);
3784 Register oop = as_Register($object$$reg);
3785 Register box = as_Register($box$$reg);
3786 Register disp_hdr = as_Register($tmp$$reg);
3787 Register tmp = as_Register($tmp2$$reg);
3788 Label cont;
3789 Label object_has_monitor;
3790 Label no_count;
3791
3792 assert_different_registers(oop, box, tmp, disp_hdr);
3793
3794 // Load markWord from object into displaced_header.
3795 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3796
3797 if (DiagnoseSyncOnValueBasedClasses != 0) {
3798 __ load_klass(tmp, oop);
3799 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3800 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3801 __ br(Assembler::NE, cont);
3802 }
3803
3804 // Check for existing monitor
3805 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3806
3807 if (!UseHeavyMonitors) {
3808 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3809 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3810
3811 if (EnableValhalla) {
3812 // Mask inline_type bit such that we go to the slow path if object is an inline type
3813 __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
3814 }
3815
3816 // Initialize the box. (Must happen before we update the object mark!)
3817 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3818
3819 // Compare object markWord with an unlocked value (tmp) and if
3820 // equal exchange the stack address of our box with object markWord.
3821 // On failure disp_hdr contains the possibly locked markWord.
3822 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3823 /*release*/ true, /*weak*/ false, disp_hdr);
3824 __ br(Assembler::EQ, cont);
3825
3826 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3827
3828 // If the compare-and-exchange succeeded, then we found an unlocked
3829 // object, will have now locked it will continue at label cont
3830
3831 // Check if the owner is self by comparing the value in the
3832 // markWord of object (disp_hdr) with the stack pointer.
3833 __ mov(rscratch1, sp);
3834 __ sub(disp_hdr, disp_hdr, rscratch1);
3835 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3836 // If condition is true we are cont and hence we can store 0 as the
3837 // displaced header in the box, which indicates that it is a recursive lock.
3838 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3839 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3840 } else {
3841 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3842 }
3843 __ b(cont);
3844
3845 // Handle existing monitor.
3846 __ bind(object_has_monitor);
3847
3848 // The object's monitor m is unlocked iff m->owner == NULL,
3849 // otherwise m->owner may contain a thread or a stack address.
3850 //
3851 // Try to CAS m->owner from NULL to current thread.
3852 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3853 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3854 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3855
3856 // Store a non-null value into the box to avoid looking like a re-entrant
3857 // lock. The fast-path monitor unlock code checks for
3858 // markWord::monitor_value so use markWord::unused_mark which has the
3859 // relevant bit set, and also matches ObjectSynchronizer::enter.
3860 __ mov(tmp, (address)markWord::unused_mark().value());
3861 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3862
3863 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3864
3865 __ cmp(rscratch1, rthread);
3866 __ br(Assembler::NE, cont); // Check for recursive locking
3867
3868 // Recursive lock case
3869 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3870 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3871
3872 __ bind(cont);
3873 // flag == EQ indicates success
3874 // flag == NE indicates failure
3875 __ br(Assembler::NE, no_count);
3876
3877 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3878
3879 __ bind(no_count);
3880 %}
3881
3882 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3883 C2_MacroAssembler _masm(&cbuf);
3884 Register oop = as_Register($object$$reg);
3885 Register box = as_Register($box$$reg);
3886 Register disp_hdr = as_Register($tmp$$reg);
3887 Register tmp = as_Register($tmp2$$reg);
3888 Label cont;
3889 Label object_has_monitor;
3890 Label no_count;
3891
3892 assert_different_registers(oop, box, tmp, disp_hdr);
3893
3894 if (!UseHeavyMonitors) {
3895 // Find the lock address and load the displaced header from the stack.
3896 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3897
3898 // If the displaced header is 0, we have a recursive unlock.
3899 __ cmp(disp_hdr, zr);
3900 __ br(Assembler::EQ, cont);
3901 }
3902
3903 // Handle existing monitor.
3904 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3905 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3906
3907 if (!UseHeavyMonitors) {
3908 // Check if it is still a light weight lock, this is is true if we
3909 // see the stack address of the basicLock in the markWord of the
3910 // object.
3911
3912 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3913 /*release*/ true, /*weak*/ false, tmp);
3914 } else {
3915 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3916 }
3917 __ b(cont);
3918
3919 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3920
3921 // Handle existing monitor.
3922 __ bind(object_has_monitor);
3923 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3924 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3925 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3926
3927 Label notRecursive;
3928 __ cbz(disp_hdr, notRecursive);
3929
3930 // Recursive lock
3931 __ sub(disp_hdr, disp_hdr, 1u);
3932 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3933 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3934 __ b(cont);
3935
3936 __ bind(notRecursive);
3937 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3938 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3939 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3940 __ cmp(rscratch1, zr); // Sets flags for result
3941 __ cbnz(rscratch1, cont);
3942 // need a release store here
3943 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3944 __ stlr(zr, tmp); // set unowned
3945
3946 __ bind(cont);
3947 // flag == EQ indicates success
3948 // flag == NE indicates failure
3949 __ br(Assembler::NE, no_count);
3950
3951 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
3952
3953 __ bind(no_count);
3954 %}
3955
3956 %}
3957
3958 //----------FRAME--------------------------------------------------------------
3959 // Definition of frame structure and management information.
3960 //
3961 // S T A C K L A Y O U T Allocators stack-slot number
3962 // | (to get allocators register number
3963 // G Owned by | | v add OptoReg::stack0())
3964 // r CALLER | |
3965 // o | +--------+ pad to even-align allocators stack-slot
3966 // w V | pad0 | numbers; owned by CALLER
3967 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3968 // h ^ | in | 5
3969 // | | args | 4 Holes in incoming args owned by SELF
3970 // | | | | 3
3971 // | | +--------+
3972 // V | | old out| Empty on Intel, window on Sparc
3973 // | old |preserve| Must be even aligned.
3974 // | SP-+--------+----> Matcher::_old_SP, even aligned
3975 // | | in | 3 area for Intel ret address
3976 // Owned by |preserve| Empty on Sparc.
3977 // SELF +--------+
3978 // | | pad2 | 2 pad to align old SP
3979 // | +--------+ 1
3980 // | | locks | 0
3981 // | +--------+----> OptoReg::stack0(), even aligned
3982 // | | pad1 | 11 pad to align new SP
3983 // | +--------+
3984 // | | | 10
3985 // | | spills | 9 spills
3986 // V | | 8 (pad0 slot for callee)
3987 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3988 // ^ | out | 7
3989 // | | args | 6 Holes in outgoing args owned by CALLEE
3990 // Owned by +--------+
3991 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3992 // | new |preserve| Must be even-aligned.
3993 // | SP-+--------+----> Matcher::_new_SP, even aligned
3994 // | | |
3995 //
3996 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3997 // known from SELF's arguments and the Java calling convention.
3998 // Region 6-7 is determined per call site.
3999 // Note 2: If the calling convention leaves holes in the incoming argument
4000 // area, those holes are owned by SELF. Holes in the outgoing area
4001 // are owned by the CALLEE. Holes should not be necessary in the
4002 // incoming area, as the Java calling convention is completely under
4003 // the control of the AD file. Doubles can be sorted and packed to
4004 // avoid holes. Holes in the outgoing arguments may be necessary for
4005 // varargs C calling conventions.
4006 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4007 // even aligned with pad0 as needed.
4008 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4009 // (the latter is true on Intel but is it false on AArch64?)
4010 // region 6-11 is even aligned; it may be padded out more so that
4011 // the region from SP to FP meets the minimum stack alignment.
4012 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4013 // alignment. Region 11, pad1, may be dynamically extended so that
4014 // SP meets the minimum alignment.
4015
4016 frame %{
4017 // These three registers define part of the calling convention
4018 // between compiled code and the interpreter.
4019
4020 // Inline Cache Register or Method for I2C.
4021 inline_cache_reg(R12);
4022
4023 // Number of stack slots consumed by locking an object
4024 sync_stack_slots(2);
4025
4026 // Compiled code's Frame Pointer
4027 frame_pointer(R31);
4028
4029 // Interpreter stores its frame pointer in a register which is
4030 // stored to the stack by I2CAdaptors.
4031 // I2CAdaptors convert from interpreted java to compiled java.
4032 interpreter_frame_pointer(R29);
4033
4034 // Stack alignment requirement
4035 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4036
4037 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4038 // for calls to C. Supports the var-args backing area for register parms.
4039 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4040
4041 // The after-PROLOG location of the return address. Location of
4042 // return address specifies a type (REG or STACK) and a number
4043 // representing the register number (i.e. - use a register name) or
4044 // stack slot.
4045 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4046 // Otherwise, it is above the locks and verification slot and alignment word
4047 // TODO this may well be correct but need to check why that - 2 is there
4048 // ppc port uses 0 but we definitely need to allow for fixed_slots
4049 // which folds in the space used for monitors
4050 return_addr(STACK - 2 +
4051 align_up((Compile::current()->in_preserve_stack_slots() +
4052 Compile::current()->fixed_slots()),
4053 stack_alignment_in_slots()));
4054
4055 // Location of compiled Java return values. Same as C for now.
4056 return_value
4057 %{
4058 // TODO do we allow ideal_reg == Op_RegN???
4059 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4060 "only return normal values");
4061
4062 static const int lo[Op_RegL + 1] = { // enum name
4063 0, // Op_Node
4064 0, // Op_Set
4065 R0_num, // Op_RegN
4066 R0_num, // Op_RegI
4067 R0_num, // Op_RegP
4068 V0_num, // Op_RegF
4069 V0_num, // Op_RegD
4070 R0_num // Op_RegL
4071 };
4072
4073 static const int hi[Op_RegL + 1] = { // enum name
4074 0, // Op_Node
4075 0, // Op_Set
4076 OptoReg::Bad, // Op_RegN
4077 OptoReg::Bad, // Op_RegI
4078 R0_H_num, // Op_RegP
4079 OptoReg::Bad, // Op_RegF
4080 V0_H_num, // Op_RegD
4081 R0_H_num // Op_RegL
4082 };
4083
4084 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4085 %}
4086 %}
4087
4088 //----------ATTRIBUTES---------------------------------------------------------
4089 //----------Operand Attributes-------------------------------------------------
4090 op_attrib op_cost(1); // Required cost attribute
4091
4092 //----------Instruction Attributes---------------------------------------------
4093 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4094 ins_attrib ins_size(32); // Required size attribute (in bits)
4095 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4096 // a non-matching short branch variant
4097 // of some long branch?
4098 ins_attrib ins_alignment(4); // Required alignment attribute (must
4099 // be a power of 2) specifies the
4100 // alignment that some part of the
4101 // instruction (not necessarily the
4102 // start) requires. If > 1, a
4103 // compute_padding() function must be
4104 // provided for the instruction
4105
4106 //----------OPERANDS-----------------------------------------------------------
4107 // Operand definitions must precede instruction definitions for correct parsing
4108 // in the ADLC because operands constitute user defined types which are used in
4109 // instruction definitions.
4110
4111 //----------Simple Operands----------------------------------------------------
4112
4113 // Integer operands 32 bit
4114 // 32 bit immediate
4115 operand immI()
4116 %{
4117 match(ConI);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 // 32 bit zero
4125 operand immI0()
4126 %{
4127 predicate(n->get_int() == 0);
4128 match(ConI);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 // 32 bit unit increment
4136 operand immI_1()
4137 %{
4138 predicate(n->get_int() == 1);
4139 match(ConI);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 // 32 bit unit decrement
4147 operand immI_M1()
4148 %{
4149 predicate(n->get_int() == -1);
4150 match(ConI);
4151
4152 op_cost(0);
4153 format %{ %}
4154 interface(CONST_INTER);
4155 %}
4156
4157 // Shift values for add/sub extension shift
4158 operand immIExt()
4159 %{
4160 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 operand immI_gt_1()
4169 %{
4170 predicate(n->get_int() > 1);
4171 match(ConI);
4172
4173 op_cost(0);
4174 format %{ %}
4175 interface(CONST_INTER);
4176 %}
4177
4178 operand immI_le_4()
4179 %{
4180 predicate(n->get_int() <= 4);
4181 match(ConI);
4182
4183 op_cost(0);
4184 format %{ %}
4185 interface(CONST_INTER);
4186 %}
4187
4188 operand immI_16()
4189 %{
4190 predicate(n->get_int() == 16);
4191 match(ConI);
4192
4193 op_cost(0);
4194 format %{ %}
4195 interface(CONST_INTER);
4196 %}
4197
4198 operand immI_24()
4199 %{
4200 predicate(n->get_int() == 24);
4201 match(ConI);
4202
4203 op_cost(0);
4204 format %{ %}
4205 interface(CONST_INTER);
4206 %}
4207
4208 operand immI_32()
4209 %{
4210 predicate(n->get_int() == 32);
4211 match(ConI);
4212
4213 op_cost(0);
4214 format %{ %}
4215 interface(CONST_INTER);
4216 %}
4217
4218 operand immI_48()
4219 %{
4220 predicate(n->get_int() == 48);
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 operand immI_56()
4229 %{
4230 predicate(n->get_int() == 56);
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 operand immI_63()
4239 %{
4240 predicate(n->get_int() == 63);
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 operand immI_64()
4249 %{
4250 predicate(n->get_int() == 64);
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 operand immI_255()
4259 %{
4260 predicate(n->get_int() == 255);
4261 match(ConI);
4262
4263 op_cost(0);
4264 format %{ %}
4265 interface(CONST_INTER);
4266 %}
4267
4268 operand immI_65535()
4269 %{
4270 predicate(n->get_int() == 65535);
4271 match(ConI);
4272
4273 op_cost(0);
4274 format %{ %}
4275 interface(CONST_INTER);
4276 %}
4277
4278 operand immI_positive()
4279 %{
4280 predicate(n->get_int() > 0);
4281 match(ConI);
4282
4283 op_cost(0);
4284 format %{ %}
4285 interface(CONST_INTER);
4286 %}
4287
4288 operand immL_255()
4289 %{
4290 predicate(n->get_long() == 255L);
4291 match(ConL);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immL_65535()
4299 %{
4300 predicate(n->get_long() == 65535L);
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 operand immL_4294967295()
4309 %{
4310 predicate(n->get_long() == 4294967295L);
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 operand immL_bitmask()
4319 %{
4320 predicate((n->get_long() != 0)
4321 && ((n->get_long() & 0xc000000000000000l) == 0)
4322 && is_power_of_2(n->get_long() + 1));
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immI_bitmask()
4331 %{
4332 predicate((n->get_int() != 0)
4333 && ((n->get_int() & 0xc0000000) == 0)
4334 && is_power_of_2(n->get_int() + 1));
4335 match(ConI);
4336
4337 op_cost(0);
4338 format %{ %}
4339 interface(CONST_INTER);
4340 %}
4341
4342 operand immL_positive_bitmaskI()
4343 %{
4344 predicate((n->get_long() != 0)
4345 && ((julong)n->get_long() < 0x80000000ULL)
4346 && is_power_of_2(n->get_long() + 1));
4347 match(ConL);
4348
4349 op_cost(0);
4350 format %{ %}
4351 interface(CONST_INTER);
4352 %}
4353
4354 // Scale values for scaled offset addressing modes (up to long but not quad)
4355 operand immIScale()
4356 %{
4357 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4358 match(ConI);
4359
4360 op_cost(0);
4361 format %{ %}
4362 interface(CONST_INTER);
4363 %}
4364
4365 // 26 bit signed offset -- for pc-relative branches
4366 operand immI26()
4367 %{
4368 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4369 match(ConI);
4370
4371 op_cost(0);
4372 format %{ %}
4373 interface(CONST_INTER);
4374 %}
4375
4376 // 19 bit signed offset -- for pc-relative loads
4377 operand immI19()
4378 %{
4379 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4380 match(ConI);
4381
4382 op_cost(0);
4383 format %{ %}
4384 interface(CONST_INTER);
4385 %}
4386
4387 // 12 bit unsigned offset -- for base plus immediate loads
4388 operand immIU12()
4389 %{
4390 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4391 match(ConI);
4392
4393 op_cost(0);
4394 format %{ %}
4395 interface(CONST_INTER);
4396 %}
4397
4398 operand immLU12()
4399 %{
4400 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4401 match(ConL);
4402
4403 op_cost(0);
4404 format %{ %}
4405 interface(CONST_INTER);
4406 %}
4407
4408 // Offset for scaled or unscaled immediate loads and stores
4409 operand immIOffset()
4410 %{
4411 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4412 match(ConI);
4413
4414 op_cost(0);
4415 format %{ %}
4416 interface(CONST_INTER);
4417 %}
4418
4419 operand immIOffset1()
4420 %{
4421 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4422 match(ConI);
4423
4424 op_cost(0);
4425 format %{ %}
4426 interface(CONST_INTER);
4427 %}
4428
4429 operand immIOffset2()
4430 %{
4431 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 operand immIOffset4()
4440 %{
4441 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4442 match(ConI);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 operand immIOffset8()
4450 %{
4451 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4452 match(ConI);
4453
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 operand immIOffset16()
4460 %{
4461 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4462 match(ConI);
4463
4464 op_cost(0);
4465 format %{ %}
4466 interface(CONST_INTER);
4467 %}
4468
4469 operand immLoffset()
4470 %{
4471 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4472 match(ConL);
4473
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 operand immLoffset1()
4480 %{
4481 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4482 match(ConL);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 operand immLoffset2()
4490 %{
4491 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4492 match(ConL);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 operand immLoffset4()
4500 %{
4501 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4502 match(ConL);
4503
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 operand immLoffset8()
4510 %{
4511 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4512 match(ConL);
4513
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 operand immLoffset16()
4520 %{
4521 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4522 match(ConL);
4523
4524 op_cost(0);
4525 format %{ %}
4526 interface(CONST_INTER);
4527 %}
4528
4529 // 8 bit signed value.
4530 operand immI8()
4531 %{
4532 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4533 match(ConI);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // 8 bit signed value (simm8), or #simm8 LSL 8.
4541 operand immI8_shift8()
4542 %{
4543 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4544 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4545 match(ConI);
4546
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 // 8 bit signed value (simm8), or #simm8 LSL 8.
4553 operand immL8_shift8()
4554 %{
4555 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4556 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4557 match(ConL);
4558
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // 8 bit integer valid for vector add sub immediate
4565 operand immBAddSubV()
4566 %{
4567 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4568 match(ConI);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // 32 bit integer valid for add sub immediate
4576 operand immIAddSub()
4577 %{
4578 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4579 match(ConI);
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 // 32 bit integer valid for vector add sub immediate
4586 operand immIAddSubV()
4587 %{
4588 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4589 match(ConI);
4590
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // 32 bit unsigned integer valid for logical immediate
4597
4598 operand immBLog()
4599 %{
4600 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4601 match(ConI);
4602
4603 op_cost(0);
4604 format %{ %}
4605 interface(CONST_INTER);
4606 %}
4607
4608 operand immSLog()
4609 %{
4610 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4611 match(ConI);
4612
4613 op_cost(0);
4614 format %{ %}
4615 interface(CONST_INTER);
4616 %}
4617
4618 operand immILog()
4619 %{
4620 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4621 match(ConI);
4622
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 // Integer operands 64 bit
4629 // 64 bit immediate
4630 operand immL()
4631 %{
4632 match(ConL);
4633
4634 op_cost(0);
4635 format %{ %}
4636 interface(CONST_INTER);
4637 %}
4638
4639 // 64 bit zero
4640 operand immL0()
4641 %{
4642 predicate(n->get_long() == 0);
4643 match(ConL);
4644
4645 op_cost(0);
4646 format %{ %}
4647 interface(CONST_INTER);
4648 %}
4649
4650 // 64 bit unit increment
4651 operand immL_1()
4652 %{
4653 predicate(n->get_long() == 1);
4654 match(ConL);
4655
4656 op_cost(0);
4657 format %{ %}
4658 interface(CONST_INTER);
4659 %}
4660
4661 // 64 bit unit decrement
4662 operand immL_M1()
4663 %{
4664 predicate(n->get_long() == -1);
4665 match(ConL);
4666
4667 op_cost(0);
4668 format %{ %}
4669 interface(CONST_INTER);
4670 %}
4671
4672 // 32 bit offset of pc in thread anchor
4673
4674 operand immL_pc_off()
4675 %{
4676 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4677 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4678 match(ConL);
4679
4680 op_cost(0);
4681 format %{ %}
4682 interface(CONST_INTER);
4683 %}
4684
4685 // 64 bit integer valid for add sub immediate
4686 operand immLAddSub()
4687 %{
4688 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4689 match(ConL);
4690 op_cost(0);
4691 format %{ %}
4692 interface(CONST_INTER);
4693 %}
4694
4695 // 64 bit integer valid for addv subv immediate
4696 operand immLAddSubV()
4697 %{
4698 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4699 match(ConL);
4700
4701 op_cost(0);
4702 format %{ %}
4703 interface(CONST_INTER);
4704 %}
4705
4706 // 64 bit integer valid for logical immediate
4707 operand immLLog()
4708 %{
4709 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4710 match(ConL);
4711 op_cost(0);
4712 format %{ %}
4713 interface(CONST_INTER);
4714 %}
4715
4716 // Long Immediate: low 32-bit mask
4717 operand immL_32bits()
4718 %{
4719 predicate(n->get_long() == 0xFFFFFFFFL);
4720 match(ConL);
4721 op_cost(0);
4722 format %{ %}
4723 interface(CONST_INTER);
4724 %}
4725
4726 // Pointer operands
4727 // Pointer Immediate
4728 operand immP()
4729 %{
4730 match(ConP);
4731
4732 op_cost(0);
4733 format %{ %}
4734 interface(CONST_INTER);
4735 %}
4736
4737 // NULL Pointer Immediate
4738 operand immP0()
4739 %{
4740 predicate(n->get_ptr() == 0);
4741 match(ConP);
4742
4743 op_cost(0);
4744 format %{ %}
4745 interface(CONST_INTER);
4746 %}
4747
4748 // Pointer Immediate One
4749 // this is used in object initialization (initial object header)
4750 operand immP_1()
4751 %{
4752 predicate(n->get_ptr() == 1);
4753 match(ConP);
4754
4755 op_cost(0);
4756 format %{ %}
4757 interface(CONST_INTER);
4758 %}
4759
4760 // Card Table Byte Map Base
4761 operand immByteMapBase()
4762 %{
4763 // Get base of card map
4764 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4765 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4766 match(ConP);
4767
4768 op_cost(0);
4769 format %{ %}
4770 interface(CONST_INTER);
4771 %}
4772
4773 // Pointer Immediate Minus One
4774 // this is used when we want to write the current PC to the thread anchor
4775 operand immP_M1()
4776 %{
4777 predicate(n->get_ptr() == -1);
4778 match(ConP);
4779
4780 op_cost(0);
4781 format %{ %}
4782 interface(CONST_INTER);
4783 %}
4784
4785 // Pointer Immediate Minus Two
4786 // this is used when we want to write the current PC to the thread anchor
4787 operand immP_M2()
4788 %{
4789 predicate(n->get_ptr() == -2);
4790 match(ConP);
4791
4792 op_cost(0);
4793 format %{ %}
4794 interface(CONST_INTER);
4795 %}
4796
4797 // Float and Double operands
4798 // Double Immediate
4799 operand immD()
4800 %{
4801 match(ConD);
4802 op_cost(0);
4803 format %{ %}
4804 interface(CONST_INTER);
4805 %}
4806
4807 // Double Immediate: +0.0d
4808 operand immD0()
4809 %{
4810 predicate(jlong_cast(n->getd()) == 0);
4811 match(ConD);
4812
4813 op_cost(0);
4814 format %{ %}
4815 interface(CONST_INTER);
4816 %}
4817
4818 // constant 'double +0.0'.
4819 operand immDPacked()
4820 %{
4821 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4822 match(ConD);
4823 op_cost(0);
4824 format %{ %}
4825 interface(CONST_INTER);
4826 %}
4827
4828 // Float Immediate
4829 operand immF()
4830 %{
4831 match(ConF);
4832 op_cost(0);
4833 format %{ %}
4834 interface(CONST_INTER);
4835 %}
4836
4837 // Float Immediate: +0.0f.
4838 operand immF0()
4839 %{
4840 predicate(jint_cast(n->getf()) == 0);
4841 match(ConF);
4842
4843 op_cost(0);
4844 format %{ %}
4845 interface(CONST_INTER);
4846 %}
4847
4848 //
4849 operand immFPacked()
4850 %{
4851 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4852 match(ConF);
4853 op_cost(0);
4854 format %{ %}
4855 interface(CONST_INTER);
4856 %}
4857
4858 // Narrow pointer operands
4859 // Narrow Pointer Immediate
4860 operand immN()
4861 %{
4862 match(ConN);
4863
4864 op_cost(0);
4865 format %{ %}
4866 interface(CONST_INTER);
4867 %}
4868
4869 // Narrow NULL Pointer Immediate
4870 operand immN0()
4871 %{
4872 predicate(n->get_narrowcon() == 0);
4873 match(ConN);
4874
4875 op_cost(0);
4876 format %{ %}
4877 interface(CONST_INTER);
4878 %}
4879
4880 operand immNKlass()
4881 %{
4882 match(ConNKlass);
4883
4884 op_cost(0);
4885 format %{ %}
4886 interface(CONST_INTER);
4887 %}
4888
4889 // Integer 32 bit Register Operands
4890 // Integer 32 bitRegister (excludes SP)
4891 operand iRegI()
4892 %{
4893 constraint(ALLOC_IN_RC(any_reg32));
4894 match(RegI);
4895 match(iRegINoSp);
4896 op_cost(0);
4897 format %{ %}
4898 interface(REG_INTER);
4899 %}
4900
4901 // Integer 32 bit Register not Special
4902 operand iRegINoSp()
4903 %{
4904 constraint(ALLOC_IN_RC(no_special_reg32));
4905 match(RegI);
4906 op_cost(0);
4907 format %{ %}
4908 interface(REG_INTER);
4909 %}
4910
4911 // Integer 64 bit Register Operands
4912 // Integer 64 bit Register (includes SP)
4913 operand iRegL()
4914 %{
4915 constraint(ALLOC_IN_RC(any_reg));
4916 match(RegL);
4917 match(iRegLNoSp);
4918 op_cost(0);
4919 format %{ %}
4920 interface(REG_INTER);
4921 %}
4922
4923 // Integer 64 bit Register not Special
4924 operand iRegLNoSp()
4925 %{
4926 constraint(ALLOC_IN_RC(no_special_reg));
4927 match(RegL);
4928 match(iRegL_R0);
4929 format %{ %}
4930 interface(REG_INTER);
4931 %}
4932
4933 // Pointer Register Operands
4934 // Pointer Register
4935 operand iRegP()
4936 %{
4937 constraint(ALLOC_IN_RC(ptr_reg));
4938 match(RegP);
4939 match(iRegPNoSp);
4940 match(iRegP_R0);
4941 //match(iRegP_R2);
4942 //match(iRegP_R4);
4943 //match(iRegP_R5);
4944 match(thread_RegP);
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950 // Pointer 64 bit Register not Special
4951 operand iRegPNoSp()
4952 %{
4953 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4954 match(RegP);
4955 // match(iRegP);
4956 // match(iRegP_R0);
4957 // match(iRegP_R2);
4958 // match(iRegP_R4);
4959 // match(iRegP_R5);
4960 // match(thread_RegP);
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 // Pointer 64 bit Register R0 only
4967 operand iRegP_R0()
4968 %{
4969 constraint(ALLOC_IN_RC(r0_reg));
4970 match(RegP);
4971 // match(iRegP);
4972 match(iRegPNoSp);
4973 op_cost(0);
4974 format %{ %}
4975 interface(REG_INTER);
4976 %}
4977
4978 // Pointer 64 bit Register R1 only
4979 operand iRegP_R1()
4980 %{
4981 constraint(ALLOC_IN_RC(r1_reg));
4982 match(RegP);
4983 // match(iRegP);
4984 match(iRegPNoSp);
4985 op_cost(0);
4986 format %{ %}
4987 interface(REG_INTER);
4988 %}
4989
4990 // Pointer 64 bit Register R2 only
4991 operand iRegP_R2()
4992 %{
4993 constraint(ALLOC_IN_RC(r2_reg));
4994 match(RegP);
4995 // match(iRegP);
4996 match(iRegPNoSp);
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 // Pointer 64 bit Register R3 only
5003 operand iRegP_R3()
5004 %{
5005 constraint(ALLOC_IN_RC(r3_reg));
5006 match(RegP);
5007 // match(iRegP);
5008 match(iRegPNoSp);
5009 op_cost(0);
5010 format %{ %}
5011 interface(REG_INTER);
5012 %}
5013
5014 // Pointer 64 bit Register R4 only
5015 operand iRegP_R4()
5016 %{
5017 constraint(ALLOC_IN_RC(r4_reg));
5018 match(RegP);
5019 // match(iRegP);
5020 match(iRegPNoSp);
5021 op_cost(0);
5022 format %{ %}
5023 interface(REG_INTER);
5024 %}
5025
5026 // Pointer 64 bit Register R5 only
5027 operand iRegP_R5()
5028 %{
5029 constraint(ALLOC_IN_RC(r5_reg));
5030 match(RegP);
5031 // match(iRegP);
5032 match(iRegPNoSp);
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 // Pointer 64 bit Register R10 only
5039 operand iRegP_R10()
5040 %{
5041 constraint(ALLOC_IN_RC(r10_reg));
5042 match(RegP);
5043 // match(iRegP);
5044 match(iRegPNoSp);
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 // Long 64 bit Register R0 only
5051 operand iRegL_R0()
5052 %{
5053 constraint(ALLOC_IN_RC(r0_reg));
5054 match(RegL);
5055 match(iRegLNoSp);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 // Long 64 bit Register R2 only
5062 operand iRegL_R2()
5063 %{
5064 constraint(ALLOC_IN_RC(r2_reg));
5065 match(RegL);
5066 match(iRegLNoSp);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 // Long 64 bit Register R3 only
5073 operand iRegL_R3()
5074 %{
5075 constraint(ALLOC_IN_RC(r3_reg));
5076 match(RegL);
5077 match(iRegLNoSp);
5078 op_cost(0);
5079 format %{ %}
5080 interface(REG_INTER);
5081 %}
5082
5083 // Long 64 bit Register R11 only
5084 operand iRegL_R11()
5085 %{
5086 constraint(ALLOC_IN_RC(r11_reg));
5087 match(RegL);
5088 match(iRegLNoSp);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 // Pointer 64 bit Register FP only
5095 operand iRegP_FP()
5096 %{
5097 constraint(ALLOC_IN_RC(fp_reg));
5098 match(RegP);
5099 // match(iRegP);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105 // Register R0 only
5106 operand iRegI_R0()
5107 %{
5108 constraint(ALLOC_IN_RC(int_r0_reg));
5109 match(RegI);
5110 match(iRegINoSp);
5111 op_cost(0);
5112 format %{ %}
5113 interface(REG_INTER);
5114 %}
5115
5116 // Register R2 only
5117 operand iRegI_R2()
5118 %{
5119 constraint(ALLOC_IN_RC(int_r2_reg));
5120 match(RegI);
5121 match(iRegINoSp);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 // Register R3 only
5128 operand iRegI_R3()
5129 %{
5130 constraint(ALLOC_IN_RC(int_r3_reg));
5131 match(RegI);
5132 match(iRegINoSp);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138
5139 // Register R4 only
5140 operand iRegI_R4()
5141 %{
5142 constraint(ALLOC_IN_RC(int_r4_reg));
5143 match(RegI);
5144 match(iRegINoSp);
5145 op_cost(0);
5146 format %{ %}
5147 interface(REG_INTER);
5148 %}
5149
5150
5151 // Pointer Register Operands
5152 // Narrow Pointer Register
5153 operand iRegN()
5154 %{
5155 constraint(ALLOC_IN_RC(any_reg32));
5156 match(RegN);
5157 match(iRegNNoSp);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 operand iRegN_R0()
5164 %{
5165 constraint(ALLOC_IN_RC(r0_reg));
5166 match(iRegN);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 operand iRegN_R2()
5173 %{
5174 constraint(ALLOC_IN_RC(r2_reg));
5175 match(iRegN);
5176 op_cost(0);
5177 format %{ %}
5178 interface(REG_INTER);
5179 %}
5180
5181 operand iRegN_R3()
5182 %{
5183 constraint(ALLOC_IN_RC(r3_reg));
5184 match(iRegN);
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 // Integer 64 bit Register not Special
5191 operand iRegNNoSp()
5192 %{
5193 constraint(ALLOC_IN_RC(no_special_reg32));
5194 match(RegN);
5195 op_cost(0);
5196 format %{ %}
5197 interface(REG_INTER);
5198 %}
5199
5200 // heap base register -- used for encoding immN0
5201
5202 operand iRegIHeapbase()
5203 %{
5204 constraint(ALLOC_IN_RC(heapbase_reg));
5205 match(RegI);
5206 op_cost(0);
5207 format %{ %}
5208 interface(REG_INTER);
5209 %}
5210
5211 // Float Register
5212 // Float register operands
5213 operand vRegF()
5214 %{
5215 constraint(ALLOC_IN_RC(float_reg));
5216 match(RegF);
5217
5218 op_cost(0);
5219 format %{ %}
5220 interface(REG_INTER);
5221 %}
5222
5223 // Double Register
5224 // Double register operands
5225 operand vRegD()
5226 %{
5227 constraint(ALLOC_IN_RC(double_reg));
5228 match(RegD);
5229
5230 op_cost(0);
5231 format %{ %}
5232 interface(REG_INTER);
5233 %}
5234
5235 // Generic vector class. This will be used for
5236 // all vector operands, including NEON and SVE.
5237 operand vReg()
5238 %{
5239 constraint(ALLOC_IN_RC(dynamic));
5240 match(VecA);
5241 match(VecD);
5242 match(VecX);
5243
5244 op_cost(0);
5245 format %{ %}
5246 interface(REG_INTER);
5247 %}
5248
5249 operand vecA()
5250 %{
5251 constraint(ALLOC_IN_RC(vectora_reg));
5252 match(VecA);
5253
5254 op_cost(0);
5255 format %{ %}
5256 interface(REG_INTER);
5257 %}
5258
5259 operand vecD()
5260 %{
5261 constraint(ALLOC_IN_RC(vectord_reg));
5262 match(VecD);
5263
5264 op_cost(0);
5265 format %{ %}
5266 interface(REG_INTER);
5267 %}
5268
5269 operand vecX()
5270 %{
5271 constraint(ALLOC_IN_RC(vectorx_reg));
5272 match(VecX);
5273
5274 op_cost(0);
5275 format %{ %}
5276 interface(REG_INTER);
5277 %}
5278
5279 operand vRegD_V0()
5280 %{
5281 constraint(ALLOC_IN_RC(v0_reg));
5282 match(RegD);
5283 op_cost(0);
5284 format %{ %}
5285 interface(REG_INTER);
5286 %}
5287
5288 operand vRegD_V1()
5289 %{
5290 constraint(ALLOC_IN_RC(v1_reg));
5291 match(RegD);
5292 op_cost(0);
5293 format %{ %}
5294 interface(REG_INTER);
5295 %}
5296
5297 operand vRegD_V2()
5298 %{
5299 constraint(ALLOC_IN_RC(v2_reg));
5300 match(RegD);
5301 op_cost(0);
5302 format %{ %}
5303 interface(REG_INTER);
5304 %}
5305
5306 operand vRegD_V3()
5307 %{
5308 constraint(ALLOC_IN_RC(v3_reg));
5309 match(RegD);
5310 op_cost(0);
5311 format %{ %}
5312 interface(REG_INTER);
5313 %}
5314
5315 operand vRegD_V4()
5316 %{
5317 constraint(ALLOC_IN_RC(v4_reg));
5318 match(RegD);
5319 op_cost(0);
5320 format %{ %}
5321 interface(REG_INTER);
5322 %}
5323
5324 operand vRegD_V5()
5325 %{
5326 constraint(ALLOC_IN_RC(v5_reg));
5327 match(RegD);
5328 op_cost(0);
5329 format %{ %}
5330 interface(REG_INTER);
5331 %}
5332
5333 operand vRegD_V6()
5334 %{
5335 constraint(ALLOC_IN_RC(v6_reg));
5336 match(RegD);
5337 op_cost(0);
5338 format %{ %}
5339 interface(REG_INTER);
5340 %}
5341
5342 operand vRegD_V7()
5343 %{
5344 constraint(ALLOC_IN_RC(v7_reg));
5345 match(RegD);
5346 op_cost(0);
5347 format %{ %}
5348 interface(REG_INTER);
5349 %}
5350
5351 operand vRegD_V8()
5352 %{
5353 constraint(ALLOC_IN_RC(v8_reg));
5354 match(RegD);
5355 op_cost(0);
5356 format %{ %}
5357 interface(REG_INTER);
5358 %}
5359
5360 operand vRegD_V9()
5361 %{
5362 constraint(ALLOC_IN_RC(v9_reg));
5363 match(RegD);
5364 op_cost(0);
5365 format %{ %}
5366 interface(REG_INTER);
5367 %}
5368
5369 operand vRegD_V10()
5370 %{
5371 constraint(ALLOC_IN_RC(v10_reg));
5372 match(RegD);
5373 op_cost(0);
5374 format %{ %}
5375 interface(REG_INTER);
5376 %}
5377
5378 operand vRegD_V11()
5379 %{
5380 constraint(ALLOC_IN_RC(v11_reg));
5381 match(RegD);
5382 op_cost(0);
5383 format %{ %}
5384 interface(REG_INTER);
5385 %}
5386
5387 operand vRegD_V12()
5388 %{
5389 constraint(ALLOC_IN_RC(v12_reg));
5390 match(RegD);
5391 op_cost(0);
5392 format %{ %}
5393 interface(REG_INTER);
5394 %}
5395
5396 operand vRegD_V13()
5397 %{
5398 constraint(ALLOC_IN_RC(v13_reg));
5399 match(RegD);
5400 op_cost(0);
5401 format %{ %}
5402 interface(REG_INTER);
5403 %}
5404
5405 operand vRegD_V14()
5406 %{
5407 constraint(ALLOC_IN_RC(v14_reg));
5408 match(RegD);
5409 op_cost(0);
5410 format %{ %}
5411 interface(REG_INTER);
5412 %}
5413
5414 operand vRegD_V15()
5415 %{
5416 constraint(ALLOC_IN_RC(v15_reg));
5417 match(RegD);
5418 op_cost(0);
5419 format %{ %}
5420 interface(REG_INTER);
5421 %}
5422
5423 operand vRegD_V16()
5424 %{
5425 constraint(ALLOC_IN_RC(v16_reg));
5426 match(RegD);
5427 op_cost(0);
5428 format %{ %}
5429 interface(REG_INTER);
5430 %}
5431
5432 operand vRegD_V17()
5433 %{
5434 constraint(ALLOC_IN_RC(v17_reg));
5435 match(RegD);
5436 op_cost(0);
5437 format %{ %}
5438 interface(REG_INTER);
5439 %}
5440
5441 operand vRegD_V18()
5442 %{
5443 constraint(ALLOC_IN_RC(v18_reg));
5444 match(RegD);
5445 op_cost(0);
5446 format %{ %}
5447 interface(REG_INTER);
5448 %}
5449
5450 operand vRegD_V19()
5451 %{
5452 constraint(ALLOC_IN_RC(v19_reg));
5453 match(RegD);
5454 op_cost(0);
5455 format %{ %}
5456 interface(REG_INTER);
5457 %}
5458
5459 operand vRegD_V20()
5460 %{
5461 constraint(ALLOC_IN_RC(v20_reg));
5462 match(RegD);
5463 op_cost(0);
5464 format %{ %}
5465 interface(REG_INTER);
5466 %}
5467
5468 operand vRegD_V21()
5469 %{
5470 constraint(ALLOC_IN_RC(v21_reg));
5471 match(RegD);
5472 op_cost(0);
5473 format %{ %}
5474 interface(REG_INTER);
5475 %}
5476
5477 operand vRegD_V22()
5478 %{
5479 constraint(ALLOC_IN_RC(v22_reg));
5480 match(RegD);
5481 op_cost(0);
5482 format %{ %}
5483 interface(REG_INTER);
5484 %}
5485
5486 operand vRegD_V23()
5487 %{
5488 constraint(ALLOC_IN_RC(v23_reg));
5489 match(RegD);
5490 op_cost(0);
5491 format %{ %}
5492 interface(REG_INTER);
5493 %}
5494
5495 operand vRegD_V24()
5496 %{
5497 constraint(ALLOC_IN_RC(v24_reg));
5498 match(RegD);
5499 op_cost(0);
5500 format %{ %}
5501 interface(REG_INTER);
5502 %}
5503
5504 operand vRegD_V25()
5505 %{
5506 constraint(ALLOC_IN_RC(v25_reg));
5507 match(RegD);
5508 op_cost(0);
5509 format %{ %}
5510 interface(REG_INTER);
5511 %}
5512
5513 operand vRegD_V26()
5514 %{
5515 constraint(ALLOC_IN_RC(v26_reg));
5516 match(RegD);
5517 op_cost(0);
5518 format %{ %}
5519 interface(REG_INTER);
5520 %}
5521
5522 operand vRegD_V27()
5523 %{
5524 constraint(ALLOC_IN_RC(v27_reg));
5525 match(RegD);
5526 op_cost(0);
5527 format %{ %}
5528 interface(REG_INTER);
5529 %}
5530
5531 operand vRegD_V28()
5532 %{
5533 constraint(ALLOC_IN_RC(v28_reg));
5534 match(RegD);
5535 op_cost(0);
5536 format %{ %}
5537 interface(REG_INTER);
5538 %}
5539
5540 operand vRegD_V29()
5541 %{
5542 constraint(ALLOC_IN_RC(v29_reg));
5543 match(RegD);
5544 op_cost(0);
5545 format %{ %}
5546 interface(REG_INTER);
5547 %}
5548
5549 operand vRegD_V30()
5550 %{
5551 constraint(ALLOC_IN_RC(v30_reg));
5552 match(RegD);
5553 op_cost(0);
5554 format %{ %}
5555 interface(REG_INTER);
5556 %}
5557
5558 operand vRegD_V31()
5559 %{
5560 constraint(ALLOC_IN_RC(v31_reg));
5561 match(RegD);
5562 op_cost(0);
5563 format %{ %}
5564 interface(REG_INTER);
5565 %}
5566
5567 operand pReg()
5568 %{
5569 constraint(ALLOC_IN_RC(pr_reg));
5570 match(RegVectMask);
5571 match(pRegGov);
5572 op_cost(0);
5573 format %{ %}
5574 interface(REG_INTER);
5575 %}
5576
5577 operand pRegGov()
5578 %{
5579 constraint(ALLOC_IN_RC(gov_pr));
5580 match(RegVectMask);
5581 op_cost(0);
5582 format %{ %}
5583 interface(REG_INTER);
5584 %}
5585
5586 operand pRegGov_P0()
5587 %{
5588 constraint(ALLOC_IN_RC(p0_reg));
5589 match(RegVectMask);
5590 op_cost(0);
5591 format %{ %}
5592 interface(REG_INTER);
5593 %}
5594
5595 operand pRegGov_P1()
5596 %{
5597 constraint(ALLOC_IN_RC(p1_reg));
5598 match(RegVectMask);
5599 op_cost(0);
5600 format %{ %}
5601 interface(REG_INTER);
5602 %}
5603
5604 // Flags register, used as output of signed compare instructions
5605
5606 // note that on AArch64 we also use this register as the output for
5607 // for floating point compare instructions (CmpF CmpD). this ensures
5608 // that ordered inequality tests use GT, GE, LT or LE none of which
5609 // pass through cases where the result is unordered i.e. one or both
5610 // inputs to the compare is a NaN. this means that the ideal code can
5611 // replace e.g. a GT with an LE and not end up capturing the NaN case
5612 // (where the comparison should always fail). EQ and NE tests are
5613 // always generated in ideal code so that unordered folds into the NE
5614 // case, matching the behaviour of AArch64 NE.
5615 //
5616 // This differs from x86 where the outputs of FP compares use a
5617 // special FP flags registers and where compares based on this
5618 // register are distinguished into ordered inequalities (cmpOpUCF) and
5619 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5620 // to explicitly handle the unordered case in branches. x86 also has
5621 // to include extra CMoveX rules to accept a cmpOpUCF input.
5622
5623 operand rFlagsReg()
5624 %{
5625 constraint(ALLOC_IN_RC(int_flags));
5626 match(RegFlags);
5627
5628 op_cost(0);
5629 format %{ "RFLAGS" %}
5630 interface(REG_INTER);
5631 %}
5632
5633 // Flags register, used as output of unsigned compare instructions
5634 operand rFlagsRegU()
5635 %{
5636 constraint(ALLOC_IN_RC(int_flags));
5637 match(RegFlags);
5638
5639 op_cost(0);
5640 format %{ "RFLAGSU" %}
5641 interface(REG_INTER);
5642 %}
5643
5644 // Special Registers
5645
5646 // Method Register
5647 operand inline_cache_RegP(iRegP reg)
5648 %{
5649 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5650 match(reg);
5651 match(iRegPNoSp);
5652 op_cost(0);
5653 format %{ %}
5654 interface(REG_INTER);
5655 %}
5656
5657 // Thread Register
5658 operand thread_RegP(iRegP reg)
5659 %{
5660 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5661 match(reg);
5662 op_cost(0);
5663 format %{ %}
5664 interface(REG_INTER);
5665 %}
5666
5667 operand lr_RegP(iRegP reg)
5668 %{
5669 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5670 match(reg);
5671 op_cost(0);
5672 format %{ %}
5673 interface(REG_INTER);
5674 %}
5675
5676 //----------Memory Operands----------------------------------------------------
5677
5678 operand indirect(iRegP reg)
5679 %{
5680 constraint(ALLOC_IN_RC(ptr_reg));
5681 match(reg);
5682 op_cost(0);
5683 format %{ "[$reg]" %}
5684 interface(MEMORY_INTER) %{
5685 base($reg);
5686 index(0xffffffff);
5687 scale(0x0);
5688 disp(0x0);
5689 %}
5690 %}
5691
5692 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5693 %{
5694 constraint(ALLOC_IN_RC(ptr_reg));
5695 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5696 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5697 op_cost(0);
5698 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5699 interface(MEMORY_INTER) %{
5700 base($reg);
5701 index($ireg);
5702 scale($scale);
5703 disp(0x0);
5704 %}
5705 %}
5706
5707 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5708 %{
5709 constraint(ALLOC_IN_RC(ptr_reg));
5710 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5711 match(AddP reg (LShiftL lreg scale));
5712 op_cost(0);
5713 format %{ "$reg, $lreg lsl($scale)" %}
5714 interface(MEMORY_INTER) %{
5715 base($reg);
5716 index($lreg);
5717 scale($scale);
5718 disp(0x0);
5719 %}
5720 %}
5721
5722 operand indIndexI2L(iRegP reg, iRegI ireg)
5723 %{
5724 constraint(ALLOC_IN_RC(ptr_reg));
5725 match(AddP reg (ConvI2L ireg));
5726 op_cost(0);
5727 format %{ "$reg, $ireg, 0, I2L" %}
5728 interface(MEMORY_INTER) %{
5729 base($reg);
5730 index($ireg);
5731 scale(0x0);
5732 disp(0x0);
5733 %}
5734 %}
5735
5736 operand indIndex(iRegP reg, iRegL lreg)
5737 %{
5738 constraint(ALLOC_IN_RC(ptr_reg));
5739 match(AddP reg lreg);
5740 op_cost(0);
5741 format %{ "$reg, $lreg" %}
5742 interface(MEMORY_INTER) %{
5743 base($reg);
5744 index($lreg);
5745 scale(0x0);
5746 disp(0x0);
5747 %}
5748 %}
5749
5750 operand indOffI(iRegP reg, immIOffset off)
5751 %{
5752 constraint(ALLOC_IN_RC(ptr_reg));
5753 match(AddP reg off);
5754 op_cost(0);
5755 format %{ "[$reg, $off]" %}
5756 interface(MEMORY_INTER) %{
5757 base($reg);
5758 index(0xffffffff);
5759 scale(0x0);
5760 disp($off);
5761 %}
5762 %}
5763
5764 operand indOffI1(iRegP reg, immIOffset1 off)
5765 %{
5766 constraint(ALLOC_IN_RC(ptr_reg));
5767 match(AddP reg off);
5768 op_cost(0);
5769 format %{ "[$reg, $off]" %}
5770 interface(MEMORY_INTER) %{
5771 base($reg);
5772 index(0xffffffff);
5773 scale(0x0);
5774 disp($off);
5775 %}
5776 %}
5777
5778 operand indOffI2(iRegP reg, immIOffset2 off)
5779 %{
5780 constraint(ALLOC_IN_RC(ptr_reg));
5781 match(AddP reg off);
5782 op_cost(0);
5783 format %{ "[$reg, $off]" %}
5784 interface(MEMORY_INTER) %{
5785 base($reg);
5786 index(0xffffffff);
5787 scale(0x0);
5788 disp($off);
5789 %}
5790 %}
5791
5792 operand indOffI4(iRegP reg, immIOffset4 off)
5793 %{
5794 constraint(ALLOC_IN_RC(ptr_reg));
5795 match(AddP reg off);
5796 op_cost(0);
5797 format %{ "[$reg, $off]" %}
5798 interface(MEMORY_INTER) %{
5799 base($reg);
5800 index(0xffffffff);
5801 scale(0x0);
5802 disp($off);
5803 %}
5804 %}
5805
5806 operand indOffI8(iRegP reg, immIOffset8 off)
5807 %{
5808 constraint(ALLOC_IN_RC(ptr_reg));
5809 match(AddP reg off);
5810 op_cost(0);
5811 format %{ "[$reg, $off]" %}
5812 interface(MEMORY_INTER) %{
5813 base($reg);
5814 index(0xffffffff);
5815 scale(0x0);
5816 disp($off);
5817 %}
5818 %}
5819
5820 operand indOffI16(iRegP reg, immIOffset16 off)
5821 %{
5822 constraint(ALLOC_IN_RC(ptr_reg));
5823 match(AddP reg off);
5824 op_cost(0);
5825 format %{ "[$reg, $off]" %}
5826 interface(MEMORY_INTER) %{
5827 base($reg);
5828 index(0xffffffff);
5829 scale(0x0);
5830 disp($off);
5831 %}
5832 %}
5833
5834 operand indOffL(iRegP reg, immLoffset off)
5835 %{
5836 constraint(ALLOC_IN_RC(ptr_reg));
5837 match(AddP reg off);
5838 op_cost(0);
5839 format %{ "[$reg, $off]" %}
5840 interface(MEMORY_INTER) %{
5841 base($reg);
5842 index(0xffffffff);
5843 scale(0x0);
5844 disp($off);
5845 %}
5846 %}
5847
5848 operand indOffL1(iRegP reg, immLoffset1 off)
5849 %{
5850 constraint(ALLOC_IN_RC(ptr_reg));
5851 match(AddP reg off);
5852 op_cost(0);
5853 format %{ "[$reg, $off]" %}
5854 interface(MEMORY_INTER) %{
5855 base($reg);
5856 index(0xffffffff);
5857 scale(0x0);
5858 disp($off);
5859 %}
5860 %}
5861
5862 operand indOffL2(iRegP reg, immLoffset2 off)
5863 %{
5864 constraint(ALLOC_IN_RC(ptr_reg));
5865 match(AddP reg off);
5866 op_cost(0);
5867 format %{ "[$reg, $off]" %}
5868 interface(MEMORY_INTER) %{
5869 base($reg);
5870 index(0xffffffff);
5871 scale(0x0);
5872 disp($off);
5873 %}
5874 %}
5875
5876 operand indOffL4(iRegP reg, immLoffset4 off)
5877 %{
5878 constraint(ALLOC_IN_RC(ptr_reg));
5879 match(AddP reg off);
5880 op_cost(0);
5881 format %{ "[$reg, $off]" %}
5882 interface(MEMORY_INTER) %{
5883 base($reg);
5884 index(0xffffffff);
5885 scale(0x0);
5886 disp($off);
5887 %}
5888 %}
5889
5890 operand indOffL8(iRegP reg, immLoffset8 off)
5891 %{
5892 constraint(ALLOC_IN_RC(ptr_reg));
5893 match(AddP reg off);
5894 op_cost(0);
5895 format %{ "[$reg, $off]" %}
5896 interface(MEMORY_INTER) %{
5897 base($reg);
5898 index(0xffffffff);
5899 scale(0x0);
5900 disp($off);
5901 %}
5902 %}
5903
5904 operand indOffL16(iRegP reg, immLoffset16 off)
5905 %{
5906 constraint(ALLOC_IN_RC(ptr_reg));
5907 match(AddP reg off);
5908 op_cost(0);
5909 format %{ "[$reg, $off]" %}
5910 interface(MEMORY_INTER) %{
5911 base($reg);
5912 index(0xffffffff);
5913 scale(0x0);
5914 disp($off);
5915 %}
5916 %}
5917
5918 operand indirectN(iRegN reg)
5919 %{
5920 predicate(CompressedOops::shift() == 0);
5921 constraint(ALLOC_IN_RC(ptr_reg));
5922 match(DecodeN reg);
5923 op_cost(0);
5924 format %{ "[$reg]\t# narrow" %}
5925 interface(MEMORY_INTER) %{
5926 base($reg);
5927 index(0xffffffff);
5928 scale(0x0);
5929 disp(0x0);
5930 %}
5931 %}
5932
5933 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5934 %{
5935 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5936 constraint(ALLOC_IN_RC(ptr_reg));
5937 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5938 op_cost(0);
5939 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5940 interface(MEMORY_INTER) %{
5941 base($reg);
5942 index($ireg);
5943 scale($scale);
5944 disp(0x0);
5945 %}
5946 %}
5947
5948 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5949 %{
5950 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5951 constraint(ALLOC_IN_RC(ptr_reg));
5952 match(AddP (DecodeN reg) (LShiftL lreg scale));
5953 op_cost(0);
5954 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5955 interface(MEMORY_INTER) %{
5956 base($reg);
5957 index($lreg);
5958 scale($scale);
5959 disp(0x0);
5960 %}
5961 %}
5962
5963 operand indIndexI2LN(iRegN reg, iRegI ireg)
5964 %{
5965 predicate(CompressedOops::shift() == 0);
5966 constraint(ALLOC_IN_RC(ptr_reg));
5967 match(AddP (DecodeN reg) (ConvI2L ireg));
5968 op_cost(0);
5969 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5970 interface(MEMORY_INTER) %{
5971 base($reg);
5972 index($ireg);
5973 scale(0x0);
5974 disp(0x0);
5975 %}
5976 %}
5977
5978 operand indIndexN(iRegN reg, iRegL lreg)
5979 %{
5980 predicate(CompressedOops::shift() == 0);
5981 constraint(ALLOC_IN_RC(ptr_reg));
5982 match(AddP (DecodeN reg) lreg);
5983 op_cost(0);
5984 format %{ "$reg, $lreg\t# narrow" %}
5985 interface(MEMORY_INTER) %{
5986 base($reg);
5987 index($lreg);
5988 scale(0x0);
5989 disp(0x0);
5990 %}
5991 %}
5992
5993 operand indOffIN(iRegN reg, immIOffset off)
5994 %{
5995 predicate(CompressedOops::shift() == 0);
5996 constraint(ALLOC_IN_RC(ptr_reg));
5997 match(AddP (DecodeN reg) off);
5998 op_cost(0);
5999 format %{ "[$reg, $off]\t# narrow" %}
6000 interface(MEMORY_INTER) %{
6001 base($reg);
6002 index(0xffffffff);
6003 scale(0x0);
6004 disp($off);
6005 %}
6006 %}
6007
6008 operand indOffLN(iRegN reg, immLoffset off)
6009 %{
6010 predicate(CompressedOops::shift() == 0);
6011 constraint(ALLOC_IN_RC(ptr_reg));
6012 match(AddP (DecodeN reg) off);
6013 op_cost(0);
6014 format %{ "[$reg, $off]\t# narrow" %}
6015 interface(MEMORY_INTER) %{
6016 base($reg);
6017 index(0xffffffff);
6018 scale(0x0);
6019 disp($off);
6020 %}
6021 %}
6022
6023
6024
6025 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6026 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6027 %{
6028 constraint(ALLOC_IN_RC(ptr_reg));
6029 match(AddP reg off);
6030 op_cost(0);
6031 format %{ "[$reg, $off]" %}
6032 interface(MEMORY_INTER) %{
6033 base($reg);
6034 index(0xffffffff);
6035 scale(0x0);
6036 disp($off);
6037 %}
6038 %}
6039
6040 //----------Special Memory Operands--------------------------------------------
6041 // Stack Slot Operand - This operand is used for loading and storing temporary
6042 // values on the stack where a match requires a value to
6043 // flow through memory.
6044 operand stackSlotP(sRegP reg)
6045 %{
6046 constraint(ALLOC_IN_RC(stack_slots));
6047 op_cost(100);
6048 // No match rule because this operand is only generated in matching
6049 // match(RegP);
6050 format %{ "[$reg]" %}
6051 interface(MEMORY_INTER) %{
6052 base(0x1e); // RSP
6053 index(0x0); // No Index
6054 scale(0x0); // No Scale
6055 disp($reg); // Stack Offset
6056 %}
6057 %}
6058
6059 operand stackSlotI(sRegI reg)
6060 %{
6061 constraint(ALLOC_IN_RC(stack_slots));
6062 // No match rule because this operand is only generated in matching
6063 // match(RegI);
6064 format %{ "[$reg]" %}
6065 interface(MEMORY_INTER) %{
6066 base(0x1e); // RSP
6067 index(0x0); // No Index
6068 scale(0x0); // No Scale
6069 disp($reg); // Stack Offset
6070 %}
6071 %}
6072
6073 operand stackSlotF(sRegF reg)
6074 %{
6075 constraint(ALLOC_IN_RC(stack_slots));
6076 // No match rule because this operand is only generated in matching
6077 // match(RegF);
6078 format %{ "[$reg]" %}
6079 interface(MEMORY_INTER) %{
6080 base(0x1e); // RSP
6081 index(0x0); // No Index
6082 scale(0x0); // No Scale
6083 disp($reg); // Stack Offset
6084 %}
6085 %}
6086
6087 operand stackSlotD(sRegD reg)
6088 %{
6089 constraint(ALLOC_IN_RC(stack_slots));
6090 // No match rule because this operand is only generated in matching
6091 // match(RegD);
6092 format %{ "[$reg]" %}
6093 interface(MEMORY_INTER) %{
6094 base(0x1e); // RSP
6095 index(0x0); // No Index
6096 scale(0x0); // No Scale
6097 disp($reg); // Stack Offset
6098 %}
6099 %}
6100
6101 operand stackSlotL(sRegL reg)
6102 %{
6103 constraint(ALLOC_IN_RC(stack_slots));
6104 // No match rule because this operand is only generated in matching
6105 // match(RegL);
6106 format %{ "[$reg]" %}
6107 interface(MEMORY_INTER) %{
6108 base(0x1e); // RSP
6109 index(0x0); // No Index
6110 scale(0x0); // No Scale
6111 disp($reg); // Stack Offset
6112 %}
6113 %}
6114
6115 // Operands for expressing Control Flow
6116 // NOTE: Label is a predefined operand which should not be redefined in
6117 // the AD file. It is generically handled within the ADLC.
6118
6119 //----------Conditional Branch Operands----------------------------------------
6120 // Comparison Op - This is the operation of the comparison, and is limited to
6121 // the following set of codes:
6122 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6123 //
6124 // Other attributes of the comparison, such as unsignedness, are specified
6125 // by the comparison instruction that sets a condition code flags register.
6126 // That result is represented by a flags operand whose subtype is appropriate
6127 // to the unsignedness (etc.) of the comparison.
6128 //
6129 // Later, the instruction which matches both the Comparison Op (a Bool) and
6130 // the flags (produced by the Cmp) specifies the coding of the comparison op
6131 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6132
6133 // used for signed integral comparisons and fp comparisons
6134
6135 operand cmpOp()
6136 %{
6137 match(Bool);
6138
6139 format %{ "" %}
6140 interface(COND_INTER) %{
6141 equal(0x0, "eq");
6142 not_equal(0x1, "ne");
6143 less(0xb, "lt");
6144 greater_equal(0xa, "ge");
6145 less_equal(0xd, "le");
6146 greater(0xc, "gt");
6147 overflow(0x6, "vs");
6148 no_overflow(0x7, "vc");
6149 %}
6150 %}
6151
6152 // used for unsigned integral comparisons
6153
6154 operand cmpOpU()
6155 %{
6156 match(Bool);
6157
6158 format %{ "" %}
6159 interface(COND_INTER) %{
6160 equal(0x0, "eq");
6161 not_equal(0x1, "ne");
6162 less(0x3, "lo");
6163 greater_equal(0x2, "hs");
6164 less_equal(0x9, "ls");
6165 greater(0x8, "hi");
6166 overflow(0x6, "vs");
6167 no_overflow(0x7, "vc");
6168 %}
6169 %}
6170
6171 // used for certain integral comparisons which can be
6172 // converted to cbxx or tbxx instructions
6173
6174 operand cmpOpEqNe()
6175 %{
6176 match(Bool);
6177 op_cost(0);
6178 predicate(n->as_Bool()->_test._test == BoolTest::ne
6179 || n->as_Bool()->_test._test == BoolTest::eq);
6180
6181 format %{ "" %}
6182 interface(COND_INTER) %{
6183 equal(0x0, "eq");
6184 not_equal(0x1, "ne");
6185 less(0xb, "lt");
6186 greater_equal(0xa, "ge");
6187 less_equal(0xd, "le");
6188 greater(0xc, "gt");
6189 overflow(0x6, "vs");
6190 no_overflow(0x7, "vc");
6191 %}
6192 %}
6193
6194 // used for certain integral comparisons which can be
6195 // converted to cbxx or tbxx instructions
6196
6197 operand cmpOpLtGe()
6198 %{
6199 match(Bool);
6200 op_cost(0);
6201
6202 predicate(n->as_Bool()->_test._test == BoolTest::lt
6203 || n->as_Bool()->_test._test == BoolTest::ge);
6204
6205 format %{ "" %}
6206 interface(COND_INTER) %{
6207 equal(0x0, "eq");
6208 not_equal(0x1, "ne");
6209 less(0xb, "lt");
6210 greater_equal(0xa, "ge");
6211 less_equal(0xd, "le");
6212 greater(0xc, "gt");
6213 overflow(0x6, "vs");
6214 no_overflow(0x7, "vc");
6215 %}
6216 %}
6217
6218 // used for certain unsigned integral comparisons which can be
6219 // converted to cbxx or tbxx instructions
6220
6221 operand cmpOpUEqNeLtGe()
6222 %{
6223 match(Bool);
6224 op_cost(0);
6225
6226 predicate(n->as_Bool()->_test._test == BoolTest::eq
6227 || n->as_Bool()->_test._test == BoolTest::ne
6228 || n->as_Bool()->_test._test == BoolTest::lt
6229 || n->as_Bool()->_test._test == BoolTest::ge);
6230
6231 format %{ "" %}
6232 interface(COND_INTER) %{
6233 equal(0x0, "eq");
6234 not_equal(0x1, "ne");
6235 less(0xb, "lt");
6236 greater_equal(0xa, "ge");
6237 less_equal(0xd, "le");
6238 greater(0xc, "gt");
6239 overflow(0x6, "vs");
6240 no_overflow(0x7, "vc");
6241 %}
6242 %}
6243
6244 // Special operand allowing long args to int ops to be truncated for free
6245
6246 operand iRegL2I(iRegL reg) %{
6247
6248 op_cost(0);
6249
6250 match(ConvL2I reg);
6251
6252 format %{ "l2i($reg)" %}
6253
6254 interface(REG_INTER)
6255 %}
6256
6257 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6258 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6259 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6260 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6261
6262 //----------OPERAND CLASSES----------------------------------------------------
6263 // Operand Classes are groups of operands that are used as to simplify
6264 // instruction definitions by not requiring the AD writer to specify
6265 // separate instructions for every form of operand when the
6266 // instruction accepts multiple operand types with the same basic
6267 // encoding and format. The classic case of this is memory operands.
6268
6269 // memory is used to define read/write location for load/store
6270 // instruction defs. we can turn a memory op into an Address
6271
6272 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6273 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6274
6275 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6276 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6277
6278 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6279 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6280
6281 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6282 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6283
6284 // All of the memory operands. For the pipeline description.
6285 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6286 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6287 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6288
6289
6290 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6291 // operations. it allows the src to be either an iRegI or a (ConvL2I
6292 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6293 // can be elided because the 32-bit instruction will just employ the
6294 // lower 32 bits anyway.
6295 //
6296 // n.b. this does not elide all L2I conversions. if the truncated
6297 // value is consumed by more than one operation then the ConvL2I
6298 // cannot be bundled into the consuming nodes so an l2i gets planted
6299 // (actually a movw $dst $src) and the downstream instructions consume
6300 // the result of the l2i as an iRegI input. That's a shame since the
6301 // movw is actually redundant but its not too costly.
6302
6303 opclass iRegIorL2I(iRegI, iRegL2I);
6304
6305 //----------PIPELINE-----------------------------------------------------------
6306 // Rules which define the behavior of the target architectures pipeline.
6307
6308 // For specific pipelines, eg A53, define the stages of that pipeline
6309 //pipe_desc(ISS, EX1, EX2, WR);
6310 #define ISS S0
6311 #define EX1 S1
6312 #define EX2 S2
6313 #define WR S3
6314
6315 // Integer ALU reg operation
6316 pipeline %{
6317
6318 attributes %{
6319 // ARM instructions are of fixed length
6320 fixed_size_instructions; // Fixed size instructions TODO does
6321 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6322 // ARM instructions come in 32-bit word units
6323 instruction_unit_size = 4; // An instruction is 4 bytes long
6324 instruction_fetch_unit_size = 64; // The processor fetches one line
6325 instruction_fetch_units = 1; // of 64 bytes
6326
6327 // List of nop instructions
6328 nops( MachNop );
6329 %}
6330
6331 // We don't use an actual pipeline model so don't care about resources
6332 // or description. we do use pipeline classes to introduce fixed
6333 // latencies
6334
6335 //----------RESOURCES----------------------------------------------------------
6336 // Resources are the functional units available to the machine
6337
6338 resources( INS0, INS1, INS01 = INS0 | INS1,
6339 ALU0, ALU1, ALU = ALU0 | ALU1,
6340 MAC,
6341 DIV,
6342 BRANCH,
6343 LDST,
6344 NEON_FP);
6345
6346 //----------PIPELINE DESCRIPTION-----------------------------------------------
6347 // Pipeline Description specifies the stages in the machine's pipeline
6348
6349 // Define the pipeline as a generic 6 stage pipeline
6350 pipe_desc(S0, S1, S2, S3, S4, S5);
6351
6352 //----------PIPELINE CLASSES---------------------------------------------------
6353 // Pipeline Classes describe the stages in which input and output are
6354 // referenced by the hardware pipeline.
6355
6356 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6357 %{
6358 single_instruction;
6359 src1 : S1(read);
6360 src2 : S2(read);
6361 dst : S5(write);
6362 INS01 : ISS;
6363 NEON_FP : S5;
6364 %}
6365
6366 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6367 %{
6368 single_instruction;
6369 src1 : S1(read);
6370 src2 : S2(read);
6371 dst : S5(write);
6372 INS01 : ISS;
6373 NEON_FP : S5;
6374 %}
6375
6376 pipe_class fp_uop_s(vRegF dst, vRegF src)
6377 %{
6378 single_instruction;
6379 src : S1(read);
6380 dst : S5(write);
6381 INS01 : ISS;
6382 NEON_FP : S5;
6383 %}
6384
6385 pipe_class fp_uop_d(vRegD dst, vRegD src)
6386 %{
6387 single_instruction;
6388 src : S1(read);
6389 dst : S5(write);
6390 INS01 : ISS;
6391 NEON_FP : S5;
6392 %}
6393
6394 pipe_class fp_d2f(vRegF dst, vRegD src)
6395 %{
6396 single_instruction;
6397 src : S1(read);
6398 dst : S5(write);
6399 INS01 : ISS;
6400 NEON_FP : S5;
6401 %}
6402
6403 pipe_class fp_f2d(vRegD dst, vRegF src)
6404 %{
6405 single_instruction;
6406 src : S1(read);
6407 dst : S5(write);
6408 INS01 : ISS;
6409 NEON_FP : S5;
6410 %}
6411
6412 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6413 %{
6414 single_instruction;
6415 src : S1(read);
6416 dst : S5(write);
6417 INS01 : ISS;
6418 NEON_FP : S5;
6419 %}
6420
6421 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6422 %{
6423 single_instruction;
6424 src : S1(read);
6425 dst : S5(write);
6426 INS01 : ISS;
6427 NEON_FP : S5;
6428 %}
6429
6430 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6431 %{
6432 single_instruction;
6433 src : S1(read);
6434 dst : S5(write);
6435 INS01 : ISS;
6436 NEON_FP : S5;
6437 %}
6438
6439 pipe_class fp_l2f(vRegF dst, iRegL src)
6440 %{
6441 single_instruction;
6442 src : S1(read);
6443 dst : S5(write);
6444 INS01 : ISS;
6445 NEON_FP : S5;
6446 %}
6447
6448 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6449 %{
6450 single_instruction;
6451 src : S1(read);
6452 dst : S5(write);
6453 INS01 : ISS;
6454 NEON_FP : S5;
6455 %}
6456
6457 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6458 %{
6459 single_instruction;
6460 src : S1(read);
6461 dst : S5(write);
6462 INS01 : ISS;
6463 NEON_FP : S5;
6464 %}
6465
6466 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6467 %{
6468 single_instruction;
6469 src : S1(read);
6470 dst : S5(write);
6471 INS01 : ISS;
6472 NEON_FP : S5;
6473 %}
6474
6475 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6476 %{
6477 single_instruction;
6478 src : S1(read);
6479 dst : S5(write);
6480 INS01 : ISS;
6481 NEON_FP : S5;
6482 %}
6483
6484 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6485 %{
6486 single_instruction;
6487 src1 : S1(read);
6488 src2 : S2(read);
6489 dst : S5(write);
6490 INS0 : ISS;
6491 NEON_FP : S5;
6492 %}
6493
6494 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6495 %{
6496 single_instruction;
6497 src1 : S1(read);
6498 src2 : S2(read);
6499 dst : S5(write);
6500 INS0 : ISS;
6501 NEON_FP : S5;
6502 %}
6503
6504 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6505 %{
6506 single_instruction;
6507 cr : S1(read);
6508 src1 : S1(read);
6509 src2 : S1(read);
6510 dst : S3(write);
6511 INS01 : ISS;
6512 NEON_FP : S3;
6513 %}
6514
6515 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6516 %{
6517 single_instruction;
6518 cr : S1(read);
6519 src1 : S1(read);
6520 src2 : S1(read);
6521 dst : S3(write);
6522 INS01 : ISS;
6523 NEON_FP : S3;
6524 %}
6525
6526 pipe_class fp_imm_s(vRegF dst)
6527 %{
6528 single_instruction;
6529 dst : S3(write);
6530 INS01 : ISS;
6531 NEON_FP : S3;
6532 %}
6533
6534 pipe_class fp_imm_d(vRegD dst)
6535 %{
6536 single_instruction;
6537 dst : S3(write);
6538 INS01 : ISS;
6539 NEON_FP : S3;
6540 %}
6541
6542 pipe_class fp_load_constant_s(vRegF dst)
6543 %{
6544 single_instruction;
6545 dst : S4(write);
6546 INS01 : ISS;
6547 NEON_FP : S4;
6548 %}
6549
6550 pipe_class fp_load_constant_d(vRegD dst)
6551 %{
6552 single_instruction;
6553 dst : S4(write);
6554 INS01 : ISS;
6555 NEON_FP : S4;
6556 %}
6557
6558 //------- Integer ALU operations --------------------------
6559
6560 // Integer ALU reg-reg operation
6561 // Operands needed in EX1, result generated in EX2
6562 // Eg. ADD x0, x1, x2
6563 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6564 %{
6565 single_instruction;
6566 dst : EX2(write);
6567 src1 : EX1(read);
6568 src2 : EX1(read);
6569 INS01 : ISS; // Dual issue as instruction 0 or 1
6570 ALU : EX2;
6571 %}
6572
6573 // Integer ALU reg-reg operation with constant shift
6574 // Shifted register must be available in LATE_ISS instead of EX1
6575 // Eg. ADD x0, x1, x2, LSL #2
6576 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6577 %{
6578 single_instruction;
6579 dst : EX2(write);
6580 src1 : EX1(read);
6581 src2 : ISS(read);
6582 INS01 : ISS;
6583 ALU : EX2;
6584 %}
6585
6586 // Integer ALU reg operation with constant shift
6587 // Eg. LSL x0, x1, #shift
6588 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6589 %{
6590 single_instruction;
6591 dst : EX2(write);
6592 src1 : ISS(read);
6593 INS01 : ISS;
6594 ALU : EX2;
6595 %}
6596
6597 // Integer ALU reg-reg operation with variable shift
6598 // Both operands must be available in LATE_ISS instead of EX1
6599 // Result is available in EX1 instead of EX2
6600 // Eg. LSLV x0, x1, x2
6601 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6602 %{
6603 single_instruction;
6604 dst : EX1(write);
6605 src1 : ISS(read);
6606 src2 : ISS(read);
6607 INS01 : ISS;
6608 ALU : EX1;
6609 %}
6610
6611 // Integer ALU reg-reg operation with extract
6612 // As for _vshift above, but result generated in EX2
6613 // Eg. EXTR x0, x1, x2, #N
6614 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6615 %{
6616 single_instruction;
6617 dst : EX2(write);
6618 src1 : ISS(read);
6619 src2 : ISS(read);
6620 INS1 : ISS; // Can only dual issue as Instruction 1
6621 ALU : EX1;
6622 %}
6623
6624 // Integer ALU reg operation
6625 // Eg. NEG x0, x1
6626 pipe_class ialu_reg(iRegI dst, iRegI src)
6627 %{
6628 single_instruction;
6629 dst : EX2(write);
6630 src : EX1(read);
6631 INS01 : ISS;
6632 ALU : EX2;
6633 %}
6634
6635 // Integer ALU reg mmediate operation
6636 // Eg. ADD x0, x1, #N
6637 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6638 %{
6639 single_instruction;
6640 dst : EX2(write);
6641 src1 : EX1(read);
6642 INS01 : ISS;
6643 ALU : EX2;
6644 %}
6645
6646 // Integer ALU immediate operation (no source operands)
6647 // Eg. MOV x0, #N
6648 pipe_class ialu_imm(iRegI dst)
6649 %{
6650 single_instruction;
6651 dst : EX1(write);
6652 INS01 : ISS;
6653 ALU : EX1;
6654 %}
6655
6656 //------- Compare operation -------------------------------
6657
6658 // Compare reg-reg
6659 // Eg. CMP x0, x1
6660 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6661 %{
6662 single_instruction;
6663 // fixed_latency(16);
6664 cr : EX2(write);
6665 op1 : EX1(read);
6666 op2 : EX1(read);
6667 INS01 : ISS;
6668 ALU : EX2;
6669 %}
6670
6671 // Compare reg-reg
6672 // Eg. CMP x0, #N
6673 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6674 %{
6675 single_instruction;
6676 // fixed_latency(16);
6677 cr : EX2(write);
6678 op1 : EX1(read);
6679 INS01 : ISS;
6680 ALU : EX2;
6681 %}
6682
6683 //------- Conditional instructions ------------------------
6684
6685 // Conditional no operands
6686 // Eg. CSINC x0, zr, zr, <cond>
6687 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6688 %{
6689 single_instruction;
6690 cr : EX1(read);
6691 dst : EX2(write);
6692 INS01 : ISS;
6693 ALU : EX2;
6694 %}
6695
6696 // Conditional 2 operand
6697 // EG. CSEL X0, X1, X2, <cond>
6698 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6699 %{
6700 single_instruction;
6701 cr : EX1(read);
6702 src1 : EX1(read);
6703 src2 : EX1(read);
6704 dst : EX2(write);
6705 INS01 : ISS;
6706 ALU : EX2;
6707 %}
6708
6709 // Conditional 2 operand
6710 // EG. CSEL X0, X1, X2, <cond>
6711 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6712 %{
6713 single_instruction;
6714 cr : EX1(read);
6715 src : EX1(read);
6716 dst : EX2(write);
6717 INS01 : ISS;
6718 ALU : EX2;
6719 %}
6720
6721 //------- Multiply pipeline operations --------------------
6722
6723 // Multiply reg-reg
6724 // Eg. MUL w0, w1, w2
6725 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6726 %{
6727 single_instruction;
6728 dst : WR(write);
6729 src1 : ISS(read);
6730 src2 : ISS(read);
6731 INS01 : ISS;
6732 MAC : WR;
6733 %}
6734
6735 // Multiply accumulate
6736 // Eg. MADD w0, w1, w2, w3
6737 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6738 %{
6739 single_instruction;
6740 dst : WR(write);
6741 src1 : ISS(read);
6742 src2 : ISS(read);
6743 src3 : ISS(read);
6744 INS01 : ISS;
6745 MAC : WR;
6746 %}
6747
6748 // Eg. MUL w0, w1, w2
6749 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6750 %{
6751 single_instruction;
6752 fixed_latency(3); // Maximum latency for 64 bit mul
6753 dst : WR(write);
6754 src1 : ISS(read);
6755 src2 : ISS(read);
6756 INS01 : ISS;
6757 MAC : WR;
6758 %}
6759
6760 // Multiply accumulate
6761 // Eg. MADD w0, w1, w2, w3
6762 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6763 %{
6764 single_instruction;
6765 fixed_latency(3); // Maximum latency for 64 bit mul
6766 dst : WR(write);
6767 src1 : ISS(read);
6768 src2 : ISS(read);
6769 src3 : ISS(read);
6770 INS01 : ISS;
6771 MAC : WR;
6772 %}
6773
6774 //------- Divide pipeline operations --------------------
6775
6776 // Eg. SDIV w0, w1, w2
6777 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6778 %{
6779 single_instruction;
6780 fixed_latency(8); // Maximum latency for 32 bit divide
6781 dst : WR(write);
6782 src1 : ISS(read);
6783 src2 : ISS(read);
6784 INS0 : ISS; // Can only dual issue as instruction 0
6785 DIV : WR;
6786 %}
6787
6788 // Eg. SDIV x0, x1, x2
6789 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6790 %{
6791 single_instruction;
6792 fixed_latency(16); // Maximum latency for 64 bit divide
6793 dst : WR(write);
6794 src1 : ISS(read);
6795 src2 : ISS(read);
6796 INS0 : ISS; // Can only dual issue as instruction 0
6797 DIV : WR;
6798 %}
6799
6800 //------- Load pipeline operations ------------------------
6801
6802 // Load - prefetch
6803 // Eg. PFRM <mem>
6804 pipe_class iload_prefetch(memory mem)
6805 %{
6806 single_instruction;
6807 mem : ISS(read);
6808 INS01 : ISS;
6809 LDST : WR;
6810 %}
6811
6812 // Load - reg, mem
6813 // Eg. LDR x0, <mem>
6814 pipe_class iload_reg_mem(iRegI dst, memory mem)
6815 %{
6816 single_instruction;
6817 dst : WR(write);
6818 mem : ISS(read);
6819 INS01 : ISS;
6820 LDST : WR;
6821 %}
6822
6823 // Load - reg, reg
6824 // Eg. LDR x0, [sp, x1]
6825 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6826 %{
6827 single_instruction;
6828 dst : WR(write);
6829 src : ISS(read);
6830 INS01 : ISS;
6831 LDST : WR;
6832 %}
6833
6834 //------- Store pipeline operations -----------------------
6835
6836 // Store - zr, mem
6837 // Eg. STR zr, <mem>
6838 pipe_class istore_mem(memory mem)
6839 %{
6840 single_instruction;
6841 mem : ISS(read);
6842 INS01 : ISS;
6843 LDST : WR;
6844 %}
6845
6846 // Store - reg, mem
6847 // Eg. STR x0, <mem>
6848 pipe_class istore_reg_mem(iRegI src, memory mem)
6849 %{
6850 single_instruction;
6851 mem : ISS(read);
6852 src : EX2(read);
6853 INS01 : ISS;
6854 LDST : WR;
6855 %}
6856
6857 // Store - reg, reg
6858 // Eg. STR x0, [sp, x1]
6859 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6860 %{
6861 single_instruction;
6862 dst : ISS(read);
6863 src : EX2(read);
6864 INS01 : ISS;
6865 LDST : WR;
6866 %}
6867
6868 //------- Store pipeline operations -----------------------
6869
6870 // Branch
6871 pipe_class pipe_branch()
6872 %{
6873 single_instruction;
6874 INS01 : ISS;
6875 BRANCH : EX1;
6876 %}
6877
6878 // Conditional branch
6879 pipe_class pipe_branch_cond(rFlagsReg cr)
6880 %{
6881 single_instruction;
6882 cr : EX1(read);
6883 INS01 : ISS;
6884 BRANCH : EX1;
6885 %}
6886
6887 // Compare & Branch
6888 // EG. CBZ/CBNZ
6889 pipe_class pipe_cmp_branch(iRegI op1)
6890 %{
6891 single_instruction;
6892 op1 : EX1(read);
6893 INS01 : ISS;
6894 BRANCH : EX1;
6895 %}
6896
6897 //------- Synchronisation operations ----------------------
6898
6899 // Any operation requiring serialization.
6900 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6901 pipe_class pipe_serial()
6902 %{
6903 single_instruction;
6904 force_serialization;
6905 fixed_latency(16);
6906 INS01 : ISS(2); // Cannot dual issue with any other instruction
6907 LDST : WR;
6908 %}
6909
6910 // Generic big/slow expanded idiom - also serialized
6911 pipe_class pipe_slow()
6912 %{
6913 instruction_count(10);
6914 multiple_bundles;
6915 force_serialization;
6916 fixed_latency(16);
6917 INS01 : ISS(2); // Cannot dual issue with any other instruction
6918 LDST : WR;
6919 %}
6920
6921 // Empty pipeline class
6922 pipe_class pipe_class_empty()
6923 %{
6924 single_instruction;
6925 fixed_latency(0);
6926 %}
6927
6928 // Default pipeline class.
6929 pipe_class pipe_class_default()
6930 %{
6931 single_instruction;
6932 fixed_latency(2);
6933 %}
6934
6935 // Pipeline class for compares.
6936 pipe_class pipe_class_compare()
6937 %{
6938 single_instruction;
6939 fixed_latency(16);
6940 %}
6941
6942 // Pipeline class for memory operations.
6943 pipe_class pipe_class_memory()
6944 %{
6945 single_instruction;
6946 fixed_latency(16);
6947 %}
6948
6949 // Pipeline class for call.
6950 pipe_class pipe_class_call()
6951 %{
6952 single_instruction;
6953 fixed_latency(100);
6954 %}
6955
6956 // Define the class for the Nop node.
6957 define %{
6958 MachNop = pipe_class_empty;
6959 %}
6960
6961 %}
6962 //----------INSTRUCTIONS-------------------------------------------------------
6963 //
6964 // match -- States which machine-independent subtree may be replaced
6965 // by this instruction.
6966 // ins_cost -- The estimated cost of this instruction is used by instruction
6967 // selection to identify a minimum cost tree of machine
6968 // instructions that matches a tree of machine-independent
6969 // instructions.
6970 // format -- A string providing the disassembly for this instruction.
6971 // The value of an instruction's operand may be inserted
6972 // by referring to it with a '$' prefix.
6973 // opcode -- Three instruction opcodes may be provided. These are referred
6974 // to within an encode class as $primary, $secondary, and $tertiary
6975 // rrspectively. The primary opcode is commonly used to
6976 // indicate the type of machine instruction, while secondary
6977 // and tertiary are often used for prefix options or addressing
6978 // modes.
6979 // ins_encode -- A list of encode classes with parameters. The encode class
6980 // name must have been defined in an 'enc_class' specification
6981 // in the encode section of the architecture description.
6982
6983 // ============================================================================
6984 // Memory (Load/Store) Instructions
6985
6986 // Load Instructions
6987
6988 // Load Byte (8 bit signed)
6989 instruct loadB(iRegINoSp dst, memory1 mem)
6990 %{
6991 match(Set dst (LoadB mem));
6992 predicate(!needs_acquiring_load(n));
6993
6994 ins_cost(4 * INSN_COST);
6995 format %{ "ldrsbw $dst, $mem\t# byte" %}
6996
6997 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6998
6999 ins_pipe(iload_reg_mem);
7000 %}
7001
7002 // Load Byte (8 bit signed) into long
7003 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7004 %{
7005 match(Set dst (ConvI2L (LoadB mem)));
7006 predicate(!needs_acquiring_load(n->in(1)));
7007
7008 ins_cost(4 * INSN_COST);
7009 format %{ "ldrsb $dst, $mem\t# byte" %}
7010
7011 ins_encode(aarch64_enc_ldrsb(dst, mem));
7012
7013 ins_pipe(iload_reg_mem);
7014 %}
7015
7016 // Load Byte (8 bit unsigned)
7017 instruct loadUB(iRegINoSp dst, memory1 mem)
7018 %{
7019 match(Set dst (LoadUB mem));
7020 predicate(!needs_acquiring_load(n));
7021
7022 ins_cost(4 * INSN_COST);
7023 format %{ "ldrbw $dst, $mem\t# byte" %}
7024
7025 ins_encode(aarch64_enc_ldrb(dst, mem));
7026
7027 ins_pipe(iload_reg_mem);
7028 %}
7029
7030 // Load Byte (8 bit unsigned) into long
7031 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7032 %{
7033 match(Set dst (ConvI2L (LoadUB mem)));
7034 predicate(!needs_acquiring_load(n->in(1)));
7035
7036 ins_cost(4 * INSN_COST);
7037 format %{ "ldrb $dst, $mem\t# byte" %}
7038
7039 ins_encode(aarch64_enc_ldrb(dst, mem));
7040
7041 ins_pipe(iload_reg_mem);
7042 %}
7043
7044 // Load Short (16 bit signed)
7045 instruct loadS(iRegINoSp dst, memory2 mem)
7046 %{
7047 match(Set dst (LoadS mem));
7048 predicate(!needs_acquiring_load(n));
7049
7050 ins_cost(4 * INSN_COST);
7051 format %{ "ldrshw $dst, $mem\t# short" %}
7052
7053 ins_encode(aarch64_enc_ldrshw(dst, mem));
7054
7055 ins_pipe(iload_reg_mem);
7056 %}
7057
7058 // Load Short (16 bit signed) into long
7059 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7060 %{
7061 match(Set dst (ConvI2L (LoadS mem)));
7062 predicate(!needs_acquiring_load(n->in(1)));
7063
7064 ins_cost(4 * INSN_COST);
7065 format %{ "ldrsh $dst, $mem\t# short" %}
7066
7067 ins_encode(aarch64_enc_ldrsh(dst, mem));
7068
7069 ins_pipe(iload_reg_mem);
7070 %}
7071
7072 // Load Char (16 bit unsigned)
7073 instruct loadUS(iRegINoSp dst, memory2 mem)
7074 %{
7075 match(Set dst (LoadUS mem));
7076 predicate(!needs_acquiring_load(n));
7077
7078 ins_cost(4 * INSN_COST);
7079 format %{ "ldrh $dst, $mem\t# short" %}
7080
7081 ins_encode(aarch64_enc_ldrh(dst, mem));
7082
7083 ins_pipe(iload_reg_mem);
7084 %}
7085
7086 // Load Short/Char (16 bit unsigned) into long
7087 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7088 %{
7089 match(Set dst (ConvI2L (LoadUS mem)));
7090 predicate(!needs_acquiring_load(n->in(1)));
7091
7092 ins_cost(4 * INSN_COST);
7093 format %{ "ldrh $dst, $mem\t# short" %}
7094
7095 ins_encode(aarch64_enc_ldrh(dst, mem));
7096
7097 ins_pipe(iload_reg_mem);
7098 %}
7099
7100 // Load Integer (32 bit signed)
7101 instruct loadI(iRegINoSp dst, memory4 mem)
7102 %{
7103 match(Set dst (LoadI mem));
7104 predicate(!needs_acquiring_load(n));
7105
7106 ins_cost(4 * INSN_COST);
7107 format %{ "ldrw $dst, $mem\t# int" %}
7108
7109 ins_encode(aarch64_enc_ldrw(dst, mem));
7110
7111 ins_pipe(iload_reg_mem);
7112 %}
7113
7114 // Load Integer (32 bit signed) into long
7115 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7116 %{
7117 match(Set dst (ConvI2L (LoadI mem)));
7118 predicate(!needs_acquiring_load(n->in(1)));
7119
7120 ins_cost(4 * INSN_COST);
7121 format %{ "ldrsw $dst, $mem\t# int" %}
7122
7123 ins_encode(aarch64_enc_ldrsw(dst, mem));
7124
7125 ins_pipe(iload_reg_mem);
7126 %}
7127
7128 // Load Integer (32 bit unsigned) into long
7129 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7130 %{
7131 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7132 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7133
7134 ins_cost(4 * INSN_COST);
7135 format %{ "ldrw $dst, $mem\t# int" %}
7136
7137 ins_encode(aarch64_enc_ldrw(dst, mem));
7138
7139 ins_pipe(iload_reg_mem);
7140 %}
7141
7142 // Load Long (64 bit signed)
7143 instruct loadL(iRegLNoSp dst, memory8 mem)
7144 %{
7145 match(Set dst (LoadL mem));
7146 predicate(!needs_acquiring_load(n));
7147
7148 ins_cost(4 * INSN_COST);
7149 format %{ "ldr $dst, $mem\t# int" %}
7150
7151 ins_encode(aarch64_enc_ldr(dst, mem));
7152
7153 ins_pipe(iload_reg_mem);
7154 %}
7155
7156 // Load Range
7157 instruct loadRange(iRegINoSp dst, memory4 mem)
7158 %{
7159 match(Set dst (LoadRange mem));
7160
7161 ins_cost(4 * INSN_COST);
7162 format %{ "ldrw $dst, $mem\t# range" %}
7163
7164 ins_encode(aarch64_enc_ldrw(dst, mem));
7165
7166 ins_pipe(iload_reg_mem);
7167 %}
7168
7169 // Load Pointer
7170 instruct loadP(iRegPNoSp dst, memory8 mem)
7171 %{
7172 match(Set dst (LoadP mem));
7173 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7174
7175 ins_cost(4 * INSN_COST);
7176 format %{ "ldr $dst, $mem\t# ptr" %}
7177
7178 ins_encode(aarch64_enc_ldr(dst, mem));
7179
7180 ins_pipe(iload_reg_mem);
7181 %}
7182
7183 // Load Compressed Pointer
7184 instruct loadN(iRegNNoSp dst, memory4 mem)
7185 %{
7186 match(Set dst (LoadN mem));
7187 predicate(!needs_acquiring_load(n));
7188
7189 ins_cost(4 * INSN_COST);
7190 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7191
7192 ins_encode(aarch64_enc_ldrw(dst, mem));
7193
7194 ins_pipe(iload_reg_mem);
7195 %}
7196
7197 // Load Klass Pointer
7198 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7199 %{
7200 match(Set dst (LoadKlass mem));
7201 predicate(!needs_acquiring_load(n));
7202
7203 ins_cost(4 * INSN_COST);
7204 format %{ "ldr $dst, $mem\t# class" %}
7205
7206 ins_encode(aarch64_enc_ldr(dst, mem));
7207
7208 ins_pipe(iload_reg_mem);
7209 %}
7210
7211 // Load Narrow Klass Pointer
7212 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7213 %{
7214 match(Set dst (LoadNKlass mem));
7215 predicate(!needs_acquiring_load(n));
7216
7217 ins_cost(4 * INSN_COST);
7218 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7219
7220 ins_encode(aarch64_enc_ldrw(dst, mem));
7221
7222 ins_pipe(iload_reg_mem);
7223 %}
7224
7225 // Load Float
7226 instruct loadF(vRegF dst, memory4 mem)
7227 %{
7228 match(Set dst (LoadF mem));
7229 predicate(!needs_acquiring_load(n));
7230
7231 ins_cost(4 * INSN_COST);
7232 format %{ "ldrs $dst, $mem\t# float" %}
7233
7234 ins_encode( aarch64_enc_ldrs(dst, mem) );
7235
7236 ins_pipe(pipe_class_memory);
7237 %}
7238
7239 // Load Double
7240 instruct loadD(vRegD dst, memory8 mem)
7241 %{
7242 match(Set dst (LoadD mem));
7243 predicate(!needs_acquiring_load(n));
7244
7245 ins_cost(4 * INSN_COST);
7246 format %{ "ldrd $dst, $mem\t# double" %}
7247
7248 ins_encode( aarch64_enc_ldrd(dst, mem) );
7249
7250 ins_pipe(pipe_class_memory);
7251 %}
7252
7253
7254 // Load Int Constant
7255 instruct loadConI(iRegINoSp dst, immI src)
7256 %{
7257 match(Set dst src);
7258
7259 ins_cost(INSN_COST);
7260 format %{ "mov $dst, $src\t# int" %}
7261
7262 ins_encode( aarch64_enc_movw_imm(dst, src) );
7263
7264 ins_pipe(ialu_imm);
7265 %}
7266
7267 // Load Long Constant
7268 instruct loadConL(iRegLNoSp dst, immL src)
7269 %{
7270 match(Set dst src);
7271
7272 ins_cost(INSN_COST);
7273 format %{ "mov $dst, $src\t# long" %}
7274
7275 ins_encode( aarch64_enc_mov_imm(dst, src) );
7276
7277 ins_pipe(ialu_imm);
7278 %}
7279
7280 // Load Pointer Constant
7281
7282 instruct loadConP(iRegPNoSp dst, immP con)
7283 %{
7284 match(Set dst con);
7285
7286 ins_cost(INSN_COST * 4);
7287 format %{
7288 "mov $dst, $con\t# ptr"
7289 %}
7290
7291 ins_encode(aarch64_enc_mov_p(dst, con));
7292
7293 ins_pipe(ialu_imm);
7294 %}
7295
7296 // Load Null Pointer Constant
7297
7298 instruct loadConP0(iRegPNoSp dst, immP0 con)
7299 %{
7300 match(Set dst con);
7301
7302 ins_cost(INSN_COST);
7303 format %{ "mov $dst, $con\t# NULL ptr" %}
7304
7305 ins_encode(aarch64_enc_mov_p0(dst, con));
7306
7307 ins_pipe(ialu_imm);
7308 %}
7309
7310 // Load Pointer Constant One
7311
7312 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7313 %{
7314 match(Set dst con);
7315
7316 ins_cost(INSN_COST);
7317 format %{ "mov $dst, $con\t# NULL ptr" %}
7318
7319 ins_encode(aarch64_enc_mov_p1(dst, con));
7320
7321 ins_pipe(ialu_imm);
7322 %}
7323
7324 // Load Byte Map Base Constant
7325
7326 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7327 %{
7328 match(Set dst con);
7329
7330 ins_cost(INSN_COST);
7331 format %{ "adr $dst, $con\t# Byte Map Base" %}
7332
7333 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7334
7335 ins_pipe(ialu_imm);
7336 %}
7337
7338 // Load Narrow Pointer Constant
7339
7340 instruct loadConN(iRegNNoSp dst, immN con)
7341 %{
7342 match(Set dst con);
7343
7344 ins_cost(INSN_COST * 4);
7345 format %{ "mov $dst, $con\t# compressed ptr" %}
7346
7347 ins_encode(aarch64_enc_mov_n(dst, con));
7348
7349 ins_pipe(ialu_imm);
7350 %}
7351
7352 // Load Narrow Null Pointer Constant
7353
7354 instruct loadConN0(iRegNNoSp dst, immN0 con)
7355 %{
7356 match(Set dst con);
7357
7358 ins_cost(INSN_COST);
7359 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7360
7361 ins_encode(aarch64_enc_mov_n0(dst, con));
7362
7363 ins_pipe(ialu_imm);
7364 %}
7365
7366 // Load Narrow Klass Constant
7367
7368 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7369 %{
7370 match(Set dst con);
7371
7372 ins_cost(INSN_COST);
7373 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7374
7375 ins_encode(aarch64_enc_mov_nk(dst, con));
7376
7377 ins_pipe(ialu_imm);
7378 %}
7379
7380 // Load Packed Float Constant
7381
7382 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7383 match(Set dst con);
7384 ins_cost(INSN_COST * 4);
7385 format %{ "fmovs $dst, $con"%}
7386 ins_encode %{
7387 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7388 %}
7389
7390 ins_pipe(fp_imm_s);
7391 %}
7392
7393 // Load Float Constant
7394
7395 instruct loadConF(vRegF dst, immF con) %{
7396 match(Set dst con);
7397
7398 ins_cost(INSN_COST * 4);
7399
7400 format %{
7401 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7402 %}
7403
7404 ins_encode %{
7405 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7406 %}
7407
7408 ins_pipe(fp_load_constant_s);
7409 %}
7410
7411 // Load Packed Double Constant
7412
7413 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7414 match(Set dst con);
7415 ins_cost(INSN_COST);
7416 format %{ "fmovd $dst, $con"%}
7417 ins_encode %{
7418 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7419 %}
7420
7421 ins_pipe(fp_imm_d);
7422 %}
7423
7424 // Load Double Constant
7425
7426 instruct loadConD(vRegD dst, immD con) %{
7427 match(Set dst con);
7428
7429 ins_cost(INSN_COST * 5);
7430 format %{
7431 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7432 %}
7433
7434 ins_encode %{
7435 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7436 %}
7437
7438 ins_pipe(fp_load_constant_d);
7439 %}
7440
7441 // Store Instructions
7442
7443 // Store CMS card-mark Immediate
7444 instruct storeimmCM0(immI0 zero, memory1 mem)
7445 %{
7446 match(Set mem (StoreCM mem zero));
7447
7448 ins_cost(INSN_COST);
7449 format %{ "storestore (elided)\n\t"
7450 "strb zr, $mem\t# byte" %}
7451
7452 ins_encode(aarch64_enc_strb0(mem));
7453
7454 ins_pipe(istore_mem);
7455 %}
7456
7457 // Store CMS card-mark Immediate with intervening StoreStore
7458 // needed when using CMS with no conditional card marking
7459 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7460 %{
7461 match(Set mem (StoreCM mem zero));
7462
7463 ins_cost(INSN_COST * 2);
7464 format %{ "storestore\n\t"
7465 "dmb ishst"
7466 "\n\tstrb zr, $mem\t# byte" %}
7467
7468 ins_encode(aarch64_enc_strb0_ordered(mem));
7469
7470 ins_pipe(istore_mem);
7471 %}
7472
7473 // Store Byte
7474 instruct storeB(iRegIorL2I src, memory1 mem)
7475 %{
7476 match(Set mem (StoreB mem src));
7477 predicate(!needs_releasing_store(n));
7478
7479 ins_cost(INSN_COST);
7480 format %{ "strb $src, $mem\t# byte" %}
7481
7482 ins_encode(aarch64_enc_strb(src, mem));
7483
7484 ins_pipe(istore_reg_mem);
7485 %}
7486
7487
7488 instruct storeimmB0(immI0 zero, memory1 mem)
7489 %{
7490 match(Set mem (StoreB mem zero));
7491 predicate(!needs_releasing_store(n));
7492
7493 ins_cost(INSN_COST);
7494 format %{ "strb rscractch2, $mem\t# byte" %}
7495
7496 ins_encode(aarch64_enc_strb0(mem));
7497
7498 ins_pipe(istore_mem);
7499 %}
7500
7501 // Store Char/Short
7502 instruct storeC(iRegIorL2I src, memory2 mem)
7503 %{
7504 match(Set mem (StoreC mem src));
7505 predicate(!needs_releasing_store(n));
7506
7507 ins_cost(INSN_COST);
7508 format %{ "strh $src, $mem\t# short" %}
7509
7510 ins_encode(aarch64_enc_strh(src, mem));
7511
7512 ins_pipe(istore_reg_mem);
7513 %}
7514
7515 instruct storeimmC0(immI0 zero, memory2 mem)
7516 %{
7517 match(Set mem (StoreC mem zero));
7518 predicate(!needs_releasing_store(n));
7519
7520 ins_cost(INSN_COST);
7521 format %{ "strh zr, $mem\t# short" %}
7522
7523 ins_encode(aarch64_enc_strh0(mem));
7524
7525 ins_pipe(istore_mem);
7526 %}
7527
7528 // Store Integer
7529
7530 instruct storeI(iRegIorL2I src, memory4 mem)
7531 %{
7532 match(Set mem(StoreI mem src));
7533 predicate(!needs_releasing_store(n));
7534
7535 ins_cost(INSN_COST);
7536 format %{ "strw $src, $mem\t# int" %}
7537
7538 ins_encode(aarch64_enc_strw(src, mem));
7539
7540 ins_pipe(istore_reg_mem);
7541 %}
7542
7543 instruct storeimmI0(immI0 zero, memory4 mem)
7544 %{
7545 match(Set mem(StoreI mem zero));
7546 predicate(!needs_releasing_store(n));
7547
7548 ins_cost(INSN_COST);
7549 format %{ "strw zr, $mem\t# int" %}
7550
7551 ins_encode(aarch64_enc_strw0(mem));
7552
7553 ins_pipe(istore_mem);
7554 %}
7555
7556 // Store Long (64 bit signed)
7557 instruct storeL(iRegL src, memory8 mem)
7558 %{
7559 match(Set mem (StoreL mem src));
7560 predicate(!needs_releasing_store(n));
7561
7562 ins_cost(INSN_COST);
7563 format %{ "str $src, $mem\t# int" %}
7564
7565 ins_encode(aarch64_enc_str(src, mem));
7566
7567 ins_pipe(istore_reg_mem);
7568 %}
7569
7570 // Store Long (64 bit signed)
7571 instruct storeimmL0(immL0 zero, memory8 mem)
7572 %{
7573 match(Set mem (StoreL mem zero));
7574 predicate(!needs_releasing_store(n));
7575
7576 ins_cost(INSN_COST);
7577 format %{ "str zr, $mem\t# int" %}
7578
7579 ins_encode(aarch64_enc_str0(mem));
7580
7581 ins_pipe(istore_mem);
7582 %}
7583
7584 // Store Pointer
7585 instruct storeP(iRegP src, memory8 mem)
7586 %{
7587 match(Set mem (StoreP mem src));
7588 predicate(!needs_releasing_store(n));
7589
7590 ins_cost(INSN_COST);
7591 format %{ "str $src, $mem\t# ptr" %}
7592
7593 ins_encode(aarch64_enc_str(src, mem));
7594
7595 ins_pipe(istore_reg_mem);
7596 %}
7597
7598 // Store Pointer
7599 instruct storeimmP0(immP0 zero, memory8 mem)
7600 %{
7601 match(Set mem (StoreP mem zero));
7602 predicate(!needs_releasing_store(n));
7603
7604 ins_cost(INSN_COST);
7605 format %{ "str zr, $mem\t# ptr" %}
7606
7607 ins_encode(aarch64_enc_str0(mem));
7608
7609 ins_pipe(istore_mem);
7610 %}
7611
7612 // Store Compressed Pointer
7613 instruct storeN(iRegN src, memory4 mem)
7614 %{
7615 match(Set mem (StoreN mem src));
7616 predicate(!needs_releasing_store(n));
7617
7618 ins_cost(INSN_COST);
7619 format %{ "strw $src, $mem\t# compressed ptr" %}
7620
7621 ins_encode(aarch64_enc_strw(src, mem));
7622
7623 ins_pipe(istore_reg_mem);
7624 %}
7625
7626 instruct storeImmN0(immN0 zero, memory4 mem)
7627 %{
7628 match(Set mem (StoreN mem zero));
7629 predicate(!needs_releasing_store(n));
7630
7631 ins_cost(INSN_COST);
7632 format %{ "strw zr, $mem\t# compressed ptr" %}
7633
7634 ins_encode(aarch64_enc_strw0(mem));
7635
7636 ins_pipe(istore_mem);
7637 %}
7638
7639 // Store Float
7640 instruct storeF(vRegF src, memory4 mem)
7641 %{
7642 match(Set mem (StoreF mem src));
7643 predicate(!needs_releasing_store(n));
7644
7645 ins_cost(INSN_COST);
7646 format %{ "strs $src, $mem\t# float" %}
7647
7648 ins_encode( aarch64_enc_strs(src, mem) );
7649
7650 ins_pipe(pipe_class_memory);
7651 %}
7652
7653 // TODO
7654 // implement storeImmF0 and storeFImmPacked
7655
7656 // Store Double
7657 instruct storeD(vRegD src, memory8 mem)
7658 %{
7659 match(Set mem (StoreD mem src));
7660 predicate(!needs_releasing_store(n));
7661
7662 ins_cost(INSN_COST);
7663 format %{ "strd $src, $mem\t# double" %}
7664
7665 ins_encode( aarch64_enc_strd(src, mem) );
7666
7667 ins_pipe(pipe_class_memory);
7668 %}
7669
7670 // Store Compressed Klass Pointer
7671 instruct storeNKlass(iRegN src, memory4 mem)
7672 %{
7673 predicate(!needs_releasing_store(n));
7674 match(Set mem (StoreNKlass mem src));
7675
7676 ins_cost(INSN_COST);
7677 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7678
7679 ins_encode(aarch64_enc_strw(src, mem));
7680
7681 ins_pipe(istore_reg_mem);
7682 %}
7683
7684 // TODO
7685 // implement storeImmD0 and storeDImmPacked
7686
7687 // prefetch instructions
7688 // Must be safe to execute with invalid address (cannot fault).
7689
7690 instruct prefetchalloc( memory8 mem ) %{
7691 match(PrefetchAllocation mem);
7692
7693 ins_cost(INSN_COST);
7694 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7695
7696 ins_encode( aarch64_enc_prefetchw(mem) );
7697
7698 ins_pipe(iload_prefetch);
7699 %}
7700
7701 // ---------------- volatile loads and stores ----------------
7702
7703 // Load Byte (8 bit signed)
7704 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7705 %{
7706 match(Set dst (LoadB mem));
7707
7708 ins_cost(VOLATILE_REF_COST);
7709 format %{ "ldarsb $dst, $mem\t# byte" %}
7710
7711 ins_encode(aarch64_enc_ldarsb(dst, mem));
7712
7713 ins_pipe(pipe_serial);
7714 %}
7715
7716 // Load Byte (8 bit signed) into long
7717 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7718 %{
7719 match(Set dst (ConvI2L (LoadB mem)));
7720
7721 ins_cost(VOLATILE_REF_COST);
7722 format %{ "ldarsb $dst, $mem\t# byte" %}
7723
7724 ins_encode(aarch64_enc_ldarsb(dst, mem));
7725
7726 ins_pipe(pipe_serial);
7727 %}
7728
7729 // Load Byte (8 bit unsigned)
7730 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7731 %{
7732 match(Set dst (LoadUB mem));
7733
7734 ins_cost(VOLATILE_REF_COST);
7735 format %{ "ldarb $dst, $mem\t# byte" %}
7736
7737 ins_encode(aarch64_enc_ldarb(dst, mem));
7738
7739 ins_pipe(pipe_serial);
7740 %}
7741
7742 // Load Byte (8 bit unsigned) into long
7743 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7744 %{
7745 match(Set dst (ConvI2L (LoadUB mem)));
7746
7747 ins_cost(VOLATILE_REF_COST);
7748 format %{ "ldarb $dst, $mem\t# byte" %}
7749
7750 ins_encode(aarch64_enc_ldarb(dst, mem));
7751
7752 ins_pipe(pipe_serial);
7753 %}
7754
7755 // Load Short (16 bit signed)
7756 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7757 %{
7758 match(Set dst (LoadS mem));
7759
7760 ins_cost(VOLATILE_REF_COST);
7761 format %{ "ldarshw $dst, $mem\t# short" %}
7762
7763 ins_encode(aarch64_enc_ldarshw(dst, mem));
7764
7765 ins_pipe(pipe_serial);
7766 %}
7767
7768 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7769 %{
7770 match(Set dst (LoadUS mem));
7771
7772 ins_cost(VOLATILE_REF_COST);
7773 format %{ "ldarhw $dst, $mem\t# short" %}
7774
7775 ins_encode(aarch64_enc_ldarhw(dst, mem));
7776
7777 ins_pipe(pipe_serial);
7778 %}
7779
7780 // Load Short/Char (16 bit unsigned) into long
7781 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7782 %{
7783 match(Set dst (ConvI2L (LoadUS mem)));
7784
7785 ins_cost(VOLATILE_REF_COST);
7786 format %{ "ldarh $dst, $mem\t# short" %}
7787
7788 ins_encode(aarch64_enc_ldarh(dst, mem));
7789
7790 ins_pipe(pipe_serial);
7791 %}
7792
7793 // Load Short/Char (16 bit signed) into long
7794 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7795 %{
7796 match(Set dst (ConvI2L (LoadS mem)));
7797
7798 ins_cost(VOLATILE_REF_COST);
7799 format %{ "ldarh $dst, $mem\t# short" %}
7800
7801 ins_encode(aarch64_enc_ldarsh(dst, mem));
7802
7803 ins_pipe(pipe_serial);
7804 %}
7805
7806 // Load Integer (32 bit signed)
7807 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7808 %{
7809 match(Set dst (LoadI mem));
7810
7811 ins_cost(VOLATILE_REF_COST);
7812 format %{ "ldarw $dst, $mem\t# int" %}
7813
7814 ins_encode(aarch64_enc_ldarw(dst, mem));
7815
7816 ins_pipe(pipe_serial);
7817 %}
7818
7819 // Load Integer (32 bit unsigned) into long
7820 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7821 %{
7822 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7823
7824 ins_cost(VOLATILE_REF_COST);
7825 format %{ "ldarw $dst, $mem\t# int" %}
7826
7827 ins_encode(aarch64_enc_ldarw(dst, mem));
7828
7829 ins_pipe(pipe_serial);
7830 %}
7831
7832 // Load Long (64 bit signed)
7833 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7834 %{
7835 match(Set dst (LoadL mem));
7836
7837 ins_cost(VOLATILE_REF_COST);
7838 format %{ "ldar $dst, $mem\t# int" %}
7839
7840 ins_encode(aarch64_enc_ldar(dst, mem));
7841
7842 ins_pipe(pipe_serial);
7843 %}
7844
7845 // Load Pointer
7846 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7847 %{
7848 match(Set dst (LoadP mem));
7849 predicate(n->as_Load()->barrier_data() == 0);
7850
7851 ins_cost(VOLATILE_REF_COST);
7852 format %{ "ldar $dst, $mem\t# ptr" %}
7853
7854 ins_encode(aarch64_enc_ldar(dst, mem));
7855
7856 ins_pipe(pipe_serial);
7857 %}
7858
7859 // Load Compressed Pointer
7860 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7861 %{
7862 match(Set dst (LoadN mem));
7863
7864 ins_cost(VOLATILE_REF_COST);
7865 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7866
7867 ins_encode(aarch64_enc_ldarw(dst, mem));
7868
7869 ins_pipe(pipe_serial);
7870 %}
7871
7872 // Load Float
7873 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7874 %{
7875 match(Set dst (LoadF mem));
7876
7877 ins_cost(VOLATILE_REF_COST);
7878 format %{ "ldars $dst, $mem\t# float" %}
7879
7880 ins_encode( aarch64_enc_fldars(dst, mem) );
7881
7882 ins_pipe(pipe_serial);
7883 %}
7884
7885 // Load Double
7886 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7887 %{
7888 match(Set dst (LoadD mem));
7889
7890 ins_cost(VOLATILE_REF_COST);
7891 format %{ "ldard $dst, $mem\t# double" %}
7892
7893 ins_encode( aarch64_enc_fldard(dst, mem) );
7894
7895 ins_pipe(pipe_serial);
7896 %}
7897
7898 // Store Byte
7899 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7900 %{
7901 match(Set mem (StoreB mem src));
7902
7903 ins_cost(VOLATILE_REF_COST);
7904 format %{ "stlrb $src, $mem\t# byte" %}
7905
7906 ins_encode(aarch64_enc_stlrb(src, mem));
7907
7908 ins_pipe(pipe_class_memory);
7909 %}
7910
7911 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7912 %{
7913 match(Set mem (StoreB mem zero));
7914
7915 ins_cost(VOLATILE_REF_COST);
7916 format %{ "stlrb zr, $mem\t# byte" %}
7917
7918 ins_encode(aarch64_enc_stlrb0(mem));
7919
7920 ins_pipe(pipe_class_memory);
7921 %}
7922
7923 // Store Char/Short
7924 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7925 %{
7926 match(Set mem (StoreC mem src));
7927
7928 ins_cost(VOLATILE_REF_COST);
7929 format %{ "stlrh $src, $mem\t# short" %}
7930
7931 ins_encode(aarch64_enc_stlrh(src, mem));
7932
7933 ins_pipe(pipe_class_memory);
7934 %}
7935
7936 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7937 %{
7938 match(Set mem (StoreC mem zero));
7939
7940 ins_cost(VOLATILE_REF_COST);
7941 format %{ "stlrh zr, $mem\t# short" %}
7942
7943 ins_encode(aarch64_enc_stlrh0(mem));
7944
7945 ins_pipe(pipe_class_memory);
7946 %}
7947
7948 // Store Integer
7949
7950 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7951 %{
7952 match(Set mem(StoreI mem src));
7953
7954 ins_cost(VOLATILE_REF_COST);
7955 format %{ "stlrw $src, $mem\t# int" %}
7956
7957 ins_encode(aarch64_enc_stlrw(src, mem));
7958
7959 ins_pipe(pipe_class_memory);
7960 %}
7961
7962 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7963 %{
7964 match(Set mem(StoreI mem zero));
7965
7966 ins_cost(VOLATILE_REF_COST);
7967 format %{ "stlrw zr, $mem\t# int" %}
7968
7969 ins_encode(aarch64_enc_stlrw0(mem));
7970
7971 ins_pipe(pipe_class_memory);
7972 %}
7973
7974 // Store Long (64 bit signed)
7975 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7976 %{
7977 match(Set mem (StoreL mem src));
7978
7979 ins_cost(VOLATILE_REF_COST);
7980 format %{ "stlr $src, $mem\t# int" %}
7981
7982 ins_encode(aarch64_enc_stlr(src, mem));
7983
7984 ins_pipe(pipe_class_memory);
7985 %}
7986
7987 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7988 %{
7989 match(Set mem (StoreL mem zero));
7990
7991 ins_cost(VOLATILE_REF_COST);
7992 format %{ "stlr zr, $mem\t# int" %}
7993
7994 ins_encode(aarch64_enc_stlr0(mem));
7995
7996 ins_pipe(pipe_class_memory);
7997 %}
7998
7999 // Store Pointer
8000 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8001 %{
8002 match(Set mem (StoreP mem src));
8003
8004 ins_cost(VOLATILE_REF_COST);
8005 format %{ "stlr $src, $mem\t# ptr" %}
8006
8007 ins_encode(aarch64_enc_stlr(src, mem));
8008
8009 ins_pipe(pipe_class_memory);
8010 %}
8011
8012 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8013 %{
8014 match(Set mem (StoreP mem zero));
8015
8016 ins_cost(VOLATILE_REF_COST);
8017 format %{ "stlr zr, $mem\t# ptr" %}
8018
8019 ins_encode(aarch64_enc_stlr0(mem));
8020
8021 ins_pipe(pipe_class_memory);
8022 %}
8023
8024 // Store Compressed Pointer
8025 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8026 %{
8027 match(Set mem (StoreN mem src));
8028
8029 ins_cost(VOLATILE_REF_COST);
8030 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8031
8032 ins_encode(aarch64_enc_stlrw(src, mem));
8033
8034 ins_pipe(pipe_class_memory);
8035 %}
8036
8037 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8038 %{
8039 match(Set mem (StoreN mem zero));
8040
8041 ins_cost(VOLATILE_REF_COST);
8042 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8043
8044 ins_encode(aarch64_enc_stlrw0(mem));
8045
8046 ins_pipe(pipe_class_memory);
8047 %}
8048
8049 // Store Float
8050 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8051 %{
8052 match(Set mem (StoreF mem src));
8053
8054 ins_cost(VOLATILE_REF_COST);
8055 format %{ "stlrs $src, $mem\t# float" %}
8056
8057 ins_encode( aarch64_enc_fstlrs(src, mem) );
8058
8059 ins_pipe(pipe_class_memory);
8060 %}
8061
8062 // TODO
8063 // implement storeImmF0 and storeFImmPacked
8064
8065 // Store Double
8066 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8067 %{
8068 match(Set mem (StoreD mem src));
8069
8070 ins_cost(VOLATILE_REF_COST);
8071 format %{ "stlrd $src, $mem\t# double" %}
8072
8073 ins_encode( aarch64_enc_fstlrd(src, mem) );
8074
8075 ins_pipe(pipe_class_memory);
8076 %}
8077
8078 // ---------------- end of volatile loads and stores ----------------
8079
8080 instruct cacheWB(indirect addr)
8081 %{
8082 predicate(VM_Version::supports_data_cache_line_flush());
8083 match(CacheWB addr);
8084
8085 ins_cost(100);
8086 format %{"cache wb $addr" %}
8087 ins_encode %{
8088 assert($addr->index_position() < 0, "should be");
8089 assert($addr$$disp == 0, "should be");
8090 __ cache_wb(Address($addr$$base$$Register, 0));
8091 %}
8092 ins_pipe(pipe_slow); // XXX
8093 %}
8094
8095 instruct cacheWBPreSync()
8096 %{
8097 predicate(VM_Version::supports_data_cache_line_flush());
8098 match(CacheWBPreSync);
8099
8100 ins_cost(100);
8101 format %{"cache wb presync" %}
8102 ins_encode %{
8103 __ cache_wbsync(true);
8104 %}
8105 ins_pipe(pipe_slow); // XXX
8106 %}
8107
8108 instruct cacheWBPostSync()
8109 %{
8110 predicate(VM_Version::supports_data_cache_line_flush());
8111 match(CacheWBPostSync);
8112
8113 ins_cost(100);
8114 format %{"cache wb postsync" %}
8115 ins_encode %{
8116 __ cache_wbsync(false);
8117 %}
8118 ins_pipe(pipe_slow); // XXX
8119 %}
8120
8121 // ============================================================================
8122 // BSWAP Instructions
8123
8124 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8125 match(Set dst (ReverseBytesI src));
8126
8127 ins_cost(INSN_COST);
8128 format %{ "revw $dst, $src" %}
8129
8130 ins_encode %{
8131 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8132 %}
8133
8134 ins_pipe(ialu_reg);
8135 %}
8136
8137 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8138 match(Set dst (ReverseBytesL src));
8139
8140 ins_cost(INSN_COST);
8141 format %{ "rev $dst, $src" %}
8142
8143 ins_encode %{
8144 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8145 %}
8146
8147 ins_pipe(ialu_reg);
8148 %}
8149
8150 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8151 match(Set dst (ReverseBytesUS src));
8152
8153 ins_cost(INSN_COST);
8154 format %{ "rev16w $dst, $src" %}
8155
8156 ins_encode %{
8157 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8158 %}
8159
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8164 match(Set dst (ReverseBytesS src));
8165
8166 ins_cost(INSN_COST);
8167 format %{ "rev16w $dst, $src\n\t"
8168 "sbfmw $dst, $dst, #0, #15" %}
8169
8170 ins_encode %{
8171 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8172 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8173 %}
8174
8175 ins_pipe(ialu_reg);
8176 %}
8177
8178 // ============================================================================
8179 // Zero Count Instructions
8180
8181 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8182 match(Set dst (CountLeadingZerosI src));
8183
8184 ins_cost(INSN_COST);
8185 format %{ "clzw $dst, $src" %}
8186 ins_encode %{
8187 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8188 %}
8189
8190 ins_pipe(ialu_reg);
8191 %}
8192
8193 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8194 match(Set dst (CountLeadingZerosL src));
8195
8196 ins_cost(INSN_COST);
8197 format %{ "clz $dst, $src" %}
8198 ins_encode %{
8199 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8206 match(Set dst (CountTrailingZerosI src));
8207
8208 ins_cost(INSN_COST * 2);
8209 format %{ "rbitw $dst, $src\n\t"
8210 "clzw $dst, $dst" %}
8211 ins_encode %{
8212 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8213 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8214 %}
8215
8216 ins_pipe(ialu_reg);
8217 %}
8218
8219 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8220 match(Set dst (CountTrailingZerosL src));
8221
8222 ins_cost(INSN_COST * 2);
8223 format %{ "rbit $dst, $src\n\t"
8224 "clz $dst, $dst" %}
8225 ins_encode %{
8226 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8227 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8228 %}
8229
8230 ins_pipe(ialu_reg);
8231 %}
8232
8233 //---------- Population Count Instructions -------------------------------------
8234 //
8235
8236 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8237 match(Set dst (PopCountI src));
8238 effect(TEMP tmp);
8239 ins_cost(INSN_COST * 13);
8240
8241 format %{ "movw $src, $src\n\t"
8242 "mov $tmp, $src\t# vector (1D)\n\t"
8243 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8244 "addv $tmp, $tmp\t# vector (8B)\n\t"
8245 "mov $dst, $tmp\t# vector (1D)" %}
8246 ins_encode %{
8247 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8248 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8249 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8250 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8251 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8252 %}
8253
8254 ins_pipe(pipe_class_default);
8255 %}
8256
8257 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8258 match(Set dst (PopCountI (LoadI mem)));
8259 effect(TEMP tmp);
8260 ins_cost(INSN_COST * 13);
8261
8262 format %{ "ldrs $tmp, $mem\n\t"
8263 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8264 "addv $tmp, $tmp\t# vector (8B)\n\t"
8265 "mov $dst, $tmp\t# vector (1D)" %}
8266 ins_encode %{
8267 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8268 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8269 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8270 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8271 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8272 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8273 %}
8274
8275 ins_pipe(pipe_class_default);
8276 %}
8277
8278 // Note: Long.bitCount(long) returns an int.
8279 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8280 match(Set dst (PopCountL src));
8281 effect(TEMP tmp);
8282 ins_cost(INSN_COST * 13);
8283
8284 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8285 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8286 "addv $tmp, $tmp\t# vector (8B)\n\t"
8287 "mov $dst, $tmp\t# vector (1D)" %}
8288 ins_encode %{
8289 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8290 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8291 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8292 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8293 %}
8294
8295 ins_pipe(pipe_class_default);
8296 %}
8297
8298 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8299 match(Set dst (PopCountL (LoadL mem)));
8300 effect(TEMP tmp);
8301 ins_cost(INSN_COST * 13);
8302
8303 format %{ "ldrd $tmp, $mem\n\t"
8304 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8305 "addv $tmp, $tmp\t# vector (8B)\n\t"
8306 "mov $dst, $tmp\t# vector (1D)" %}
8307 ins_encode %{
8308 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8309 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8310 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8311 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8312 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8313 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8314 %}
8315
8316 ins_pipe(pipe_class_default);
8317 %}
8318
8319 // ============================================================================
8320 // MemBar Instruction
8321
8322 instruct load_fence() %{
8323 match(LoadFence);
8324 ins_cost(VOLATILE_REF_COST);
8325
8326 format %{ "load_fence" %}
8327
8328 ins_encode %{
8329 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8330 %}
8331 ins_pipe(pipe_serial);
8332 %}
8333
8334 instruct unnecessary_membar_acquire() %{
8335 predicate(unnecessary_acquire(n));
8336 match(MemBarAcquire);
8337 ins_cost(0);
8338
8339 format %{ "membar_acquire (elided)" %}
8340
8341 ins_encode %{
8342 __ block_comment("membar_acquire (elided)");
8343 %}
8344
8345 ins_pipe(pipe_class_empty);
8346 %}
8347
8348 instruct membar_acquire() %{
8349 match(MemBarAcquire);
8350 ins_cost(VOLATILE_REF_COST);
8351
8352 format %{ "membar_acquire\n\t"
8353 "dmb ish" %}
8354
8355 ins_encode %{
8356 __ block_comment("membar_acquire");
8357 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8358 %}
8359
8360 ins_pipe(pipe_serial);
8361 %}
8362
8363
8364 instruct membar_acquire_lock() %{
8365 match(MemBarAcquireLock);
8366 ins_cost(VOLATILE_REF_COST);
8367
8368 format %{ "membar_acquire_lock (elided)" %}
8369
8370 ins_encode %{
8371 __ block_comment("membar_acquire_lock (elided)");
8372 %}
8373
8374 ins_pipe(pipe_serial);
8375 %}
8376
8377 instruct store_fence() %{
8378 match(StoreFence);
8379 ins_cost(VOLATILE_REF_COST);
8380
8381 format %{ "store_fence" %}
8382
8383 ins_encode %{
8384 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8385 %}
8386 ins_pipe(pipe_serial);
8387 %}
8388
8389 instruct unnecessary_membar_release() %{
8390 predicate(unnecessary_release(n));
8391 match(MemBarRelease);
8392 ins_cost(0);
8393
8394 format %{ "membar_release (elided)" %}
8395
8396 ins_encode %{
8397 __ block_comment("membar_release (elided)");
8398 %}
8399 ins_pipe(pipe_serial);
8400 %}
8401
8402 instruct membar_release() %{
8403 match(MemBarRelease);
8404 ins_cost(VOLATILE_REF_COST);
8405
8406 format %{ "membar_release\n\t"
8407 "dmb ish" %}
8408
8409 ins_encode %{
8410 __ block_comment("membar_release");
8411 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8412 %}
8413 ins_pipe(pipe_serial);
8414 %}
8415
8416 instruct membar_storestore() %{
8417 match(MemBarStoreStore);
8418 match(StoreStoreFence);
8419 ins_cost(VOLATILE_REF_COST);
8420
8421 format %{ "MEMBAR-store-store" %}
8422
8423 ins_encode %{
8424 __ membar(Assembler::StoreStore);
8425 %}
8426 ins_pipe(pipe_serial);
8427 %}
8428
8429 instruct membar_release_lock() %{
8430 match(MemBarReleaseLock);
8431 ins_cost(VOLATILE_REF_COST);
8432
8433 format %{ "membar_release_lock (elided)" %}
8434
8435 ins_encode %{
8436 __ block_comment("membar_release_lock (elided)");
8437 %}
8438
8439 ins_pipe(pipe_serial);
8440 %}
8441
8442 instruct unnecessary_membar_volatile() %{
8443 predicate(unnecessary_volatile(n));
8444 match(MemBarVolatile);
8445 ins_cost(0);
8446
8447 format %{ "membar_volatile (elided)" %}
8448
8449 ins_encode %{
8450 __ block_comment("membar_volatile (elided)");
8451 %}
8452
8453 ins_pipe(pipe_serial);
8454 %}
8455
8456 instruct membar_volatile() %{
8457 match(MemBarVolatile);
8458 ins_cost(VOLATILE_REF_COST*100);
8459
8460 format %{ "membar_volatile\n\t"
8461 "dmb ish"%}
8462
8463 ins_encode %{
8464 __ block_comment("membar_volatile");
8465 __ membar(Assembler::StoreLoad);
8466 %}
8467
8468 ins_pipe(pipe_serial);
8469 %}
8470
8471 // ============================================================================
8472 // Cast/Convert Instructions
8473
8474 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8475 match(Set dst (CastX2P src));
8476
8477 ins_cost(INSN_COST);
8478 format %{ "mov $dst, $src\t# long -> ptr" %}
8479
8480 ins_encode %{
8481 if ($dst$$reg != $src$$reg) {
8482 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8483 }
8484 %}
8485
8486 ins_pipe(ialu_reg);
8487 %}
8488
8489 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8490 match(Set dst (CastP2X src));
8491
8492 ins_cost(INSN_COST);
8493 format %{ "mov $dst, $src\t# ptr -> long" %}
8494
8495 ins_encode %{
8496 if ($dst$$reg != $src$$reg) {
8497 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8498 }
8499 %}
8500
8501 ins_pipe(ialu_reg);
8502 %}
8503
8504 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8505 match(Set dst (CastP2X src));
8506
8507 ins_cost(INSN_COST);
8508 format %{ "mov $dst, $src\t# ptr -> long" %}
8509
8510 ins_encode %{
8511 if ($dst$$reg != $src$$reg) {
8512 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8513 }
8514 %}
8515
8516 ins_pipe(ialu_reg);
8517 %}
8518
8519 // Convert oop into int for vectors alignment masking
8520 instruct convP2I(iRegINoSp dst, iRegP src) %{
8521 match(Set dst (ConvL2I (CastP2X src)));
8522
8523 ins_cost(INSN_COST);
8524 format %{ "movw $dst, $src\t# ptr -> int" %}
8525 ins_encode %{
8526 __ movw($dst$$Register, $src$$Register);
8527 %}
8528
8529 ins_pipe(ialu_reg);
8530 %}
8531
8532 // Convert compressed oop into int for vectors alignment masking
8533 // in case of 32bit oops (heap < 4Gb).
8534 instruct convN2I(iRegINoSp dst, iRegN src)
8535 %{
8536 predicate(CompressedOops::shift() == 0);
8537 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8538
8539 ins_cost(INSN_COST);
8540 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8541 ins_encode %{
8542 __ movw($dst$$Register, $src$$Register);
8543 %}
8544
8545 ins_pipe(ialu_reg);
8546 %}
8547
8548
8549 // Convert oop pointer into compressed form
8550 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8551 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8552 match(Set dst (EncodeP src));
8553 effect(KILL cr);
8554 ins_cost(INSN_COST * 3);
8555 format %{ "encode_heap_oop $dst, $src" %}
8556 ins_encode %{
8557 Register s = $src$$Register;
8558 Register d = $dst$$Register;
8559 __ encode_heap_oop(d, s);
8560 %}
8561 ins_pipe(ialu_reg);
8562 %}
8563
8564 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8565 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8566 match(Set dst (EncodeP src));
8567 ins_cost(INSN_COST * 3);
8568 format %{ "encode_heap_oop_not_null $dst, $src" %}
8569 ins_encode %{
8570 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8571 %}
8572 ins_pipe(ialu_reg);
8573 %}
8574
8575 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8576 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8577 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8578 match(Set dst (DecodeN src));
8579 ins_cost(INSN_COST * 3);
8580 format %{ "decode_heap_oop $dst, $src" %}
8581 ins_encode %{
8582 Register s = $src$$Register;
8583 Register d = $dst$$Register;
8584 __ decode_heap_oop(d, s);
8585 %}
8586 ins_pipe(ialu_reg);
8587 %}
8588
8589 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8590 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8591 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8592 match(Set dst (DecodeN src));
8593 ins_cost(INSN_COST * 3);
8594 format %{ "decode_heap_oop_not_null $dst, $src" %}
8595 ins_encode %{
8596 Register s = $src$$Register;
8597 Register d = $dst$$Register;
8598 __ decode_heap_oop_not_null(d, s);
8599 %}
8600 ins_pipe(ialu_reg);
8601 %}
8602
8603 // n.b. AArch64 implementations of encode_klass_not_null and
8604 // decode_klass_not_null do not modify the flags register so, unlike
8605 // Intel, we don't kill CR as a side effect here
8606
8607 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8608 match(Set dst (EncodePKlass src));
8609
8610 ins_cost(INSN_COST * 3);
8611 format %{ "encode_klass_not_null $dst,$src" %}
8612
8613 ins_encode %{
8614 Register src_reg = as_Register($src$$reg);
8615 Register dst_reg = as_Register($dst$$reg);
8616 __ encode_klass_not_null(dst_reg, src_reg);
8617 %}
8618
8619 ins_pipe(ialu_reg);
8620 %}
8621
8622 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8623 match(Set dst (DecodeNKlass src));
8624
8625 ins_cost(INSN_COST * 3);
8626 format %{ "decode_klass_not_null $dst,$src" %}
8627
8628 ins_encode %{
8629 Register src_reg = as_Register($src$$reg);
8630 Register dst_reg = as_Register($dst$$reg);
8631 if (dst_reg != src_reg) {
8632 __ decode_klass_not_null(dst_reg, src_reg);
8633 } else {
8634 __ decode_klass_not_null(dst_reg);
8635 }
8636 %}
8637
8638 ins_pipe(ialu_reg);
8639 %}
8640
8641 instruct checkCastPP(iRegPNoSp dst)
8642 %{
8643 match(Set dst (CheckCastPP dst));
8644
8645 size(0);
8646 format %{ "# checkcastPP of $dst" %}
8647 ins_encode(/* empty encoding */);
8648 ins_pipe(pipe_class_empty);
8649 %}
8650
8651 instruct castPP(iRegPNoSp dst)
8652 %{
8653 match(Set dst (CastPP dst));
8654
8655 size(0);
8656 format %{ "# castPP of $dst" %}
8657 ins_encode(/* empty encoding */);
8658 ins_pipe(pipe_class_empty);
8659 %}
8660
8661 instruct castII(iRegI dst)
8662 %{
8663 match(Set dst (CastII dst));
8664
8665 size(0);
8666 format %{ "# castII of $dst" %}
8667 ins_encode(/* empty encoding */);
8668 ins_cost(0);
8669 ins_pipe(pipe_class_empty);
8670 %}
8671
8672 instruct castLL(iRegL dst)
8673 %{
8674 match(Set dst (CastLL dst));
8675
8676 size(0);
8677 format %{ "# castLL of $dst" %}
8678 ins_encode(/* empty encoding */);
8679 ins_cost(0);
8680 ins_pipe(pipe_class_empty);
8681 %}
8682
8683 instruct castFF(vRegF dst)
8684 %{
8685 match(Set dst (CastFF dst));
8686
8687 size(0);
8688 format %{ "# castFF of $dst" %}
8689 ins_encode(/* empty encoding */);
8690 ins_cost(0);
8691 ins_pipe(pipe_class_empty);
8692 %}
8693
8694 instruct castDD(vRegD dst)
8695 %{
8696 match(Set dst (CastDD dst));
8697
8698 size(0);
8699 format %{ "# castDD of $dst" %}
8700 ins_encode(/* empty encoding */);
8701 ins_cost(0);
8702 ins_pipe(pipe_class_empty);
8703 %}
8704
8705 instruct castVV(vReg dst)
8706 %{
8707 match(Set dst (CastVV dst));
8708
8709 size(0);
8710 format %{ "# castVV of $dst" %}
8711 ins_encode(/* empty encoding */);
8712 ins_cost(0);
8713 ins_pipe(pipe_class_empty);
8714 %}
8715
8716 instruct castVVMask(pRegGov dst)
8717 %{
8718 match(Set dst (CastVV dst));
8719
8720 size(0);
8721 format %{ "# castVV of $dst" %}
8722 ins_encode(/* empty encoding */);
8723 ins_cost(0);
8724 ins_pipe(pipe_class_empty);
8725 %}
8726
8727 // ============================================================================
8728 // Atomic operation instructions
8729 //
8730
8731 // standard CompareAndSwapX when we are using barriers
8732 // these have higher priority than the rules selected by a predicate
8733
8734 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8735 // can't match them
8736
8737 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8738
8739 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8740 ins_cost(2 * VOLATILE_REF_COST);
8741
8742 effect(KILL cr);
8743
8744 format %{
8745 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8746 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8747 %}
8748
8749 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8750 aarch64_enc_cset_eq(res));
8751
8752 ins_pipe(pipe_slow);
8753 %}
8754
8755 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8756
8757 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8758 ins_cost(2 * VOLATILE_REF_COST);
8759
8760 effect(KILL cr);
8761
8762 format %{
8763 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8764 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8765 %}
8766
8767 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8768 aarch64_enc_cset_eq(res));
8769
8770 ins_pipe(pipe_slow);
8771 %}
8772
8773 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8774
8775 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8776 ins_cost(2 * VOLATILE_REF_COST);
8777
8778 effect(KILL cr);
8779
8780 format %{
8781 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8782 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8783 %}
8784
8785 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8786 aarch64_enc_cset_eq(res));
8787
8788 ins_pipe(pipe_slow);
8789 %}
8790
8791 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8792
8793 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8794 ins_cost(2 * VOLATILE_REF_COST);
8795
8796 effect(KILL cr);
8797
8798 format %{
8799 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8800 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8801 %}
8802
8803 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8804 aarch64_enc_cset_eq(res));
8805
8806 ins_pipe(pipe_slow);
8807 %}
8808
8809 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8810
8811 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8812 predicate(n->as_LoadStore()->barrier_data() == 0);
8813 ins_cost(2 * VOLATILE_REF_COST);
8814
8815 effect(KILL cr);
8816
8817 format %{
8818 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8819 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8820 %}
8821
8822 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8823 aarch64_enc_cset_eq(res));
8824
8825 ins_pipe(pipe_slow);
8826 %}
8827
8828 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8829
8830 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8831 ins_cost(2 * VOLATILE_REF_COST);
8832
8833 effect(KILL cr);
8834
8835 format %{
8836 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8837 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8838 %}
8839
8840 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8841 aarch64_enc_cset_eq(res));
8842
8843 ins_pipe(pipe_slow);
8844 %}
8845
8846 // alternative CompareAndSwapX when we are eliding barriers
8847
8848 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8849
8850 predicate(needs_acquiring_load_exclusive(n));
8851 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8852 ins_cost(VOLATILE_REF_COST);
8853
8854 effect(KILL cr);
8855
8856 format %{
8857 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8858 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8859 %}
8860
8861 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8862 aarch64_enc_cset_eq(res));
8863
8864 ins_pipe(pipe_slow);
8865 %}
8866
8867 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8868
8869 predicate(needs_acquiring_load_exclusive(n));
8870 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8871 ins_cost(VOLATILE_REF_COST);
8872
8873 effect(KILL cr);
8874
8875 format %{
8876 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8877 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8878 %}
8879
8880 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8881 aarch64_enc_cset_eq(res));
8882
8883 ins_pipe(pipe_slow);
8884 %}
8885
8886 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8887
8888 predicate(needs_acquiring_load_exclusive(n));
8889 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8890 ins_cost(VOLATILE_REF_COST);
8891
8892 effect(KILL cr);
8893
8894 format %{
8895 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8896 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8897 %}
8898
8899 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8900 aarch64_enc_cset_eq(res));
8901
8902 ins_pipe(pipe_slow);
8903 %}
8904
8905 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8906
8907 predicate(needs_acquiring_load_exclusive(n));
8908 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8909 ins_cost(VOLATILE_REF_COST);
8910
8911 effect(KILL cr);
8912
8913 format %{
8914 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8915 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8916 %}
8917
8918 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8919 aarch64_enc_cset_eq(res));
8920
8921 ins_pipe(pipe_slow);
8922 %}
8923
8924 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8925
8926 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8927 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8928 ins_cost(VOLATILE_REF_COST);
8929
8930 effect(KILL cr);
8931
8932 format %{
8933 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8934 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8935 %}
8936
8937 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8938 aarch64_enc_cset_eq(res));
8939
8940 ins_pipe(pipe_slow);
8941 %}
8942
8943 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8944
8945 predicate(needs_acquiring_load_exclusive(n));
8946 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8947 ins_cost(VOLATILE_REF_COST);
8948
8949 effect(KILL cr);
8950
8951 format %{
8952 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8953 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8954 %}
8955
8956 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8957 aarch64_enc_cset_eq(res));
8958
8959 ins_pipe(pipe_slow);
8960 %}
8961
8962
8963 // ---------------------------------------------------------------------
8964
8965 // BEGIN This section of the file is automatically generated. Do not edit --------------
8966
8967 // Sundry CAS operations. Note that release is always true,
8968 // regardless of the memory ordering of the CAS. This is because we
8969 // need the volatile case to be sequentially consistent but there is
8970 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8971 // can't check the type of memory ordering here, so we always emit a
8972 // STLXR.
8973
8974 // This section is generated from aarch64_ad_cas.m4
8975
8976
8977
8978 // This pattern is generated automatically from cas.m4.
8979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8980 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8981
8982 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8983 ins_cost(2 * VOLATILE_REF_COST);
8984 effect(TEMP_DEF res, KILL cr);
8985 format %{
8986 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8987 %}
8988 ins_encode %{
8989 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8990 Assembler::byte, /*acquire*/ false, /*release*/ true,
8991 /*weak*/ false, $res$$Register);
8992 __ sxtbw($res$$Register, $res$$Register);
8993 %}
8994 ins_pipe(pipe_slow);
8995 %}
8996
8997 // This pattern is generated automatically from cas.m4.
8998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8999 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9000
9001 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9002 ins_cost(2 * VOLATILE_REF_COST);
9003 effect(TEMP_DEF res, KILL cr);
9004 format %{
9005 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9006 %}
9007 ins_encode %{
9008 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9009 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9010 /*weak*/ false, $res$$Register);
9011 __ sxthw($res$$Register, $res$$Register);
9012 %}
9013 ins_pipe(pipe_slow);
9014 %}
9015
9016 // This pattern is generated automatically from cas.m4.
9017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9018 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9019
9020 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9021 ins_cost(2 * VOLATILE_REF_COST);
9022 effect(TEMP_DEF res, KILL cr);
9023 format %{
9024 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9025 %}
9026 ins_encode %{
9027 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9028 Assembler::word, /*acquire*/ false, /*release*/ true,
9029 /*weak*/ false, $res$$Register);
9030 %}
9031 ins_pipe(pipe_slow);
9032 %}
9033
9034 // This pattern is generated automatically from cas.m4.
9035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9036 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9037
9038 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9039 ins_cost(2 * VOLATILE_REF_COST);
9040 effect(TEMP_DEF res, KILL cr);
9041 format %{
9042 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9043 %}
9044 ins_encode %{
9045 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9046 Assembler::xword, /*acquire*/ false, /*release*/ true,
9047 /*weak*/ false, $res$$Register);
9048 %}
9049 ins_pipe(pipe_slow);
9050 %}
9051
9052 // This pattern is generated automatically from cas.m4.
9053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9054 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9055
9056 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9057 ins_cost(2 * VOLATILE_REF_COST);
9058 effect(TEMP_DEF res, KILL cr);
9059 format %{
9060 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9061 %}
9062 ins_encode %{
9063 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9064 Assembler::word, /*acquire*/ false, /*release*/ true,
9065 /*weak*/ false, $res$$Register);
9066 %}
9067 ins_pipe(pipe_slow);
9068 %}
9069
9070 // This pattern is generated automatically from cas.m4.
9071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9072 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9073 predicate(n->as_LoadStore()->barrier_data() == 0);
9074 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9075 ins_cost(2 * VOLATILE_REF_COST);
9076 effect(TEMP_DEF res, KILL cr);
9077 format %{
9078 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9079 %}
9080 ins_encode %{
9081 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9082 Assembler::xword, /*acquire*/ false, /*release*/ true,
9083 /*weak*/ false, $res$$Register);
9084 %}
9085 ins_pipe(pipe_slow);
9086 %}
9087
9088 // This pattern is generated automatically from cas.m4.
9089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9090 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9091 predicate(needs_acquiring_load_exclusive(n));
9092 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9093 ins_cost(VOLATILE_REF_COST);
9094 effect(TEMP_DEF res, KILL cr);
9095 format %{
9096 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9097 %}
9098 ins_encode %{
9099 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9100 Assembler::byte, /*acquire*/ true, /*release*/ true,
9101 /*weak*/ false, $res$$Register);
9102 __ sxtbw($res$$Register, $res$$Register);
9103 %}
9104 ins_pipe(pipe_slow);
9105 %}
9106
9107 // This pattern is generated automatically from cas.m4.
9108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9109 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9110 predicate(needs_acquiring_load_exclusive(n));
9111 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9112 ins_cost(VOLATILE_REF_COST);
9113 effect(TEMP_DEF res, KILL cr);
9114 format %{
9115 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9116 %}
9117 ins_encode %{
9118 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9119 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9120 /*weak*/ false, $res$$Register);
9121 __ sxthw($res$$Register, $res$$Register);
9122 %}
9123 ins_pipe(pipe_slow);
9124 %}
9125
9126 // This pattern is generated automatically from cas.m4.
9127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9128 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9129 predicate(needs_acquiring_load_exclusive(n));
9130 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9131 ins_cost(VOLATILE_REF_COST);
9132 effect(TEMP_DEF res, KILL cr);
9133 format %{
9134 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9135 %}
9136 ins_encode %{
9137 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9138 Assembler::word, /*acquire*/ true, /*release*/ true,
9139 /*weak*/ false, $res$$Register);
9140 %}
9141 ins_pipe(pipe_slow);
9142 %}
9143
9144 // This pattern is generated automatically from cas.m4.
9145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9146 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9147 predicate(needs_acquiring_load_exclusive(n));
9148 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9149 ins_cost(VOLATILE_REF_COST);
9150 effect(TEMP_DEF res, KILL cr);
9151 format %{
9152 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9153 %}
9154 ins_encode %{
9155 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9156 Assembler::xword, /*acquire*/ true, /*release*/ true,
9157 /*weak*/ false, $res$$Register);
9158 %}
9159 ins_pipe(pipe_slow);
9160 %}
9161
9162 // This pattern is generated automatically from cas.m4.
9163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9164 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9165 predicate(needs_acquiring_load_exclusive(n));
9166 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9167 ins_cost(VOLATILE_REF_COST);
9168 effect(TEMP_DEF res, KILL cr);
9169 format %{
9170 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9171 %}
9172 ins_encode %{
9173 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9174 Assembler::word, /*acquire*/ true, /*release*/ true,
9175 /*weak*/ false, $res$$Register);
9176 %}
9177 ins_pipe(pipe_slow);
9178 %}
9179
9180 // This pattern is generated automatically from cas.m4.
9181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9182 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9183 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9184 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9185 ins_cost(VOLATILE_REF_COST);
9186 effect(TEMP_DEF res, KILL cr);
9187 format %{
9188 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9189 %}
9190 ins_encode %{
9191 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9192 Assembler::xword, /*acquire*/ true, /*release*/ true,
9193 /*weak*/ false, $res$$Register);
9194 %}
9195 ins_pipe(pipe_slow);
9196 %}
9197
9198 // This pattern is generated automatically from cas.m4.
9199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9200 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9201
9202 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9203 ins_cost(2 * VOLATILE_REF_COST);
9204 effect(KILL cr);
9205 format %{
9206 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9207 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9208 %}
9209 ins_encode %{
9210 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9211 Assembler::byte, /*acquire*/ false, /*release*/ true,
9212 /*weak*/ true, noreg);
9213 __ csetw($res$$Register, Assembler::EQ);
9214 %}
9215 ins_pipe(pipe_slow);
9216 %}
9217
9218 // This pattern is generated automatically from cas.m4.
9219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9220 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9221
9222 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9223 ins_cost(2 * VOLATILE_REF_COST);
9224 effect(KILL cr);
9225 format %{
9226 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9227 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9228 %}
9229 ins_encode %{
9230 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9231 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9232 /*weak*/ true, noreg);
9233 __ csetw($res$$Register, Assembler::EQ);
9234 %}
9235 ins_pipe(pipe_slow);
9236 %}
9237
9238 // This pattern is generated automatically from cas.m4.
9239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9240 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9241
9242 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9243 ins_cost(2 * VOLATILE_REF_COST);
9244 effect(KILL cr);
9245 format %{
9246 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9247 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9248 %}
9249 ins_encode %{
9250 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9251 Assembler::word, /*acquire*/ false, /*release*/ true,
9252 /*weak*/ true, noreg);
9253 __ csetw($res$$Register, Assembler::EQ);
9254 %}
9255 ins_pipe(pipe_slow);
9256 %}
9257
9258 // This pattern is generated automatically from cas.m4.
9259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9260 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9261
9262 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9263 ins_cost(2 * VOLATILE_REF_COST);
9264 effect(KILL cr);
9265 format %{
9266 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9267 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9268 %}
9269 ins_encode %{
9270 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9271 Assembler::xword, /*acquire*/ false, /*release*/ true,
9272 /*weak*/ true, noreg);
9273 __ csetw($res$$Register, Assembler::EQ);
9274 %}
9275 ins_pipe(pipe_slow);
9276 %}
9277
9278 // This pattern is generated automatically from cas.m4.
9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9280 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9281
9282 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9283 ins_cost(2 * VOLATILE_REF_COST);
9284 effect(KILL cr);
9285 format %{
9286 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9287 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9288 %}
9289 ins_encode %{
9290 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9291 Assembler::word, /*acquire*/ false, /*release*/ true,
9292 /*weak*/ true, noreg);
9293 __ csetw($res$$Register, Assembler::EQ);
9294 %}
9295 ins_pipe(pipe_slow);
9296 %}
9297
9298 // This pattern is generated automatically from cas.m4.
9299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9300 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9301 predicate(n->as_LoadStore()->barrier_data() == 0);
9302 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9303 ins_cost(2 * VOLATILE_REF_COST);
9304 effect(KILL cr);
9305 format %{
9306 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9307 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9308 %}
9309 ins_encode %{
9310 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9311 Assembler::xword, /*acquire*/ false, /*release*/ true,
9312 /*weak*/ true, noreg);
9313 __ csetw($res$$Register, Assembler::EQ);
9314 %}
9315 ins_pipe(pipe_slow);
9316 %}
9317
9318 // This pattern is generated automatically from cas.m4.
9319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9320 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9321 predicate(needs_acquiring_load_exclusive(n));
9322 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9323 ins_cost(VOLATILE_REF_COST);
9324 effect(KILL cr);
9325 format %{
9326 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9327 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9328 %}
9329 ins_encode %{
9330 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9331 Assembler::byte, /*acquire*/ true, /*release*/ true,
9332 /*weak*/ true, noreg);
9333 __ csetw($res$$Register, Assembler::EQ);
9334 %}
9335 ins_pipe(pipe_slow);
9336 %}
9337
9338 // This pattern is generated automatically from cas.m4.
9339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9340 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9341 predicate(needs_acquiring_load_exclusive(n));
9342 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9343 ins_cost(VOLATILE_REF_COST);
9344 effect(KILL cr);
9345 format %{
9346 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9347 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9348 %}
9349 ins_encode %{
9350 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9351 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9352 /*weak*/ true, noreg);
9353 __ csetw($res$$Register, Assembler::EQ);
9354 %}
9355 ins_pipe(pipe_slow);
9356 %}
9357
9358 // This pattern is generated automatically from cas.m4.
9359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9360 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9361 predicate(needs_acquiring_load_exclusive(n));
9362 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9363 ins_cost(VOLATILE_REF_COST);
9364 effect(KILL cr);
9365 format %{
9366 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9367 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9368 %}
9369 ins_encode %{
9370 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9371 Assembler::word, /*acquire*/ true, /*release*/ true,
9372 /*weak*/ true, noreg);
9373 __ csetw($res$$Register, Assembler::EQ);
9374 %}
9375 ins_pipe(pipe_slow);
9376 %}
9377
9378 // This pattern is generated automatically from cas.m4.
9379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9380 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9381 predicate(needs_acquiring_load_exclusive(n));
9382 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9383 ins_cost(VOLATILE_REF_COST);
9384 effect(KILL cr);
9385 format %{
9386 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9387 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9388 %}
9389 ins_encode %{
9390 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9391 Assembler::xword, /*acquire*/ true, /*release*/ true,
9392 /*weak*/ true, noreg);
9393 __ csetw($res$$Register, Assembler::EQ);
9394 %}
9395 ins_pipe(pipe_slow);
9396 %}
9397
9398 // This pattern is generated automatically from cas.m4.
9399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9400 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9401 predicate(needs_acquiring_load_exclusive(n));
9402 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9403 ins_cost(VOLATILE_REF_COST);
9404 effect(KILL cr);
9405 format %{
9406 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9407 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9408 %}
9409 ins_encode %{
9410 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9411 Assembler::word, /*acquire*/ true, /*release*/ true,
9412 /*weak*/ true, noreg);
9413 __ csetw($res$$Register, Assembler::EQ);
9414 %}
9415 ins_pipe(pipe_slow);
9416 %}
9417
9418 // This pattern is generated automatically from cas.m4.
9419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9420 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9421 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9422 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9423 ins_cost(VOLATILE_REF_COST);
9424 effect(KILL cr);
9425 format %{
9426 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9427 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9428 %}
9429 ins_encode %{
9430 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9431 Assembler::xword, /*acquire*/ true, /*release*/ true,
9432 /*weak*/ true, noreg);
9433 __ csetw($res$$Register, Assembler::EQ);
9434 %}
9435 ins_pipe(pipe_slow);
9436 %}
9437
9438 // END This section of the file is automatically generated. Do not edit --------------
9439 // ---------------------------------------------------------------------
9440
9441 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9442 match(Set prev (GetAndSetI mem newv));
9443 ins_cost(2 * VOLATILE_REF_COST);
9444 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9445 ins_encode %{
9446 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9447 %}
9448 ins_pipe(pipe_serial);
9449 %}
9450
9451 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9452 match(Set prev (GetAndSetL mem newv));
9453 ins_cost(2 * VOLATILE_REF_COST);
9454 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9455 ins_encode %{
9456 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9457 %}
9458 ins_pipe(pipe_serial);
9459 %}
9460
9461 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9462 match(Set prev (GetAndSetN mem newv));
9463 ins_cost(2 * VOLATILE_REF_COST);
9464 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9465 ins_encode %{
9466 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9467 %}
9468 ins_pipe(pipe_serial);
9469 %}
9470
9471 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9472 predicate(n->as_LoadStore()->barrier_data() == 0);
9473 match(Set prev (GetAndSetP mem newv));
9474 ins_cost(2 * VOLATILE_REF_COST);
9475 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9476 ins_encode %{
9477 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9478 %}
9479 ins_pipe(pipe_serial);
9480 %}
9481
9482 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9483 predicate(needs_acquiring_load_exclusive(n));
9484 match(Set prev (GetAndSetI mem newv));
9485 ins_cost(VOLATILE_REF_COST);
9486 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9487 ins_encode %{
9488 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9489 %}
9490 ins_pipe(pipe_serial);
9491 %}
9492
9493 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9494 predicate(needs_acquiring_load_exclusive(n));
9495 match(Set prev (GetAndSetL mem newv));
9496 ins_cost(VOLATILE_REF_COST);
9497 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9498 ins_encode %{
9499 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9500 %}
9501 ins_pipe(pipe_serial);
9502 %}
9503
9504 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9505 predicate(needs_acquiring_load_exclusive(n));
9506 match(Set prev (GetAndSetN mem newv));
9507 ins_cost(VOLATILE_REF_COST);
9508 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9509 ins_encode %{
9510 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9511 %}
9512 ins_pipe(pipe_serial);
9513 %}
9514
9515 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9516 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9517 match(Set prev (GetAndSetP mem newv));
9518 ins_cost(VOLATILE_REF_COST);
9519 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9520 ins_encode %{
9521 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9522 %}
9523 ins_pipe(pipe_serial);
9524 %}
9525
9526
9527 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9528 match(Set newval (GetAndAddL mem incr));
9529 ins_cost(2 * VOLATILE_REF_COST + 1);
9530 format %{ "get_and_addL $newval, [$mem], $incr" %}
9531 ins_encode %{
9532 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9533 %}
9534 ins_pipe(pipe_serial);
9535 %}
9536
9537 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9538 predicate(n->as_LoadStore()->result_not_used());
9539 match(Set dummy (GetAndAddL mem incr));
9540 ins_cost(2 * VOLATILE_REF_COST);
9541 format %{ "get_and_addL [$mem], $incr" %}
9542 ins_encode %{
9543 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9544 %}
9545 ins_pipe(pipe_serial);
9546 %}
9547
9548 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9549 match(Set newval (GetAndAddL mem incr));
9550 ins_cost(2 * VOLATILE_REF_COST + 1);
9551 format %{ "get_and_addL $newval, [$mem], $incr" %}
9552 ins_encode %{
9553 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9554 %}
9555 ins_pipe(pipe_serial);
9556 %}
9557
9558 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9559 predicate(n->as_LoadStore()->result_not_used());
9560 match(Set dummy (GetAndAddL mem incr));
9561 ins_cost(2 * VOLATILE_REF_COST);
9562 format %{ "get_and_addL [$mem], $incr" %}
9563 ins_encode %{
9564 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9565 %}
9566 ins_pipe(pipe_serial);
9567 %}
9568
9569 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9570 match(Set newval (GetAndAddI mem incr));
9571 ins_cost(2 * VOLATILE_REF_COST + 1);
9572 format %{ "get_and_addI $newval, [$mem], $incr" %}
9573 ins_encode %{
9574 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9575 %}
9576 ins_pipe(pipe_serial);
9577 %}
9578
9579 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9580 predicate(n->as_LoadStore()->result_not_used());
9581 match(Set dummy (GetAndAddI mem incr));
9582 ins_cost(2 * VOLATILE_REF_COST);
9583 format %{ "get_and_addI [$mem], $incr" %}
9584 ins_encode %{
9585 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9586 %}
9587 ins_pipe(pipe_serial);
9588 %}
9589
9590 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9591 match(Set newval (GetAndAddI mem incr));
9592 ins_cost(2 * VOLATILE_REF_COST + 1);
9593 format %{ "get_and_addI $newval, [$mem], $incr" %}
9594 ins_encode %{
9595 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9596 %}
9597 ins_pipe(pipe_serial);
9598 %}
9599
9600 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9601 predicate(n->as_LoadStore()->result_not_used());
9602 match(Set dummy (GetAndAddI mem incr));
9603 ins_cost(2 * VOLATILE_REF_COST);
9604 format %{ "get_and_addI [$mem], $incr" %}
9605 ins_encode %{
9606 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9607 %}
9608 ins_pipe(pipe_serial);
9609 %}
9610
9611 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9612 predicate(needs_acquiring_load_exclusive(n));
9613 match(Set newval (GetAndAddL mem incr));
9614 ins_cost(VOLATILE_REF_COST + 1);
9615 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9616 ins_encode %{
9617 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9618 %}
9619 ins_pipe(pipe_serial);
9620 %}
9621
9622 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9623 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9624 match(Set dummy (GetAndAddL mem incr));
9625 ins_cost(VOLATILE_REF_COST);
9626 format %{ "get_and_addL_acq [$mem], $incr" %}
9627 ins_encode %{
9628 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9629 %}
9630 ins_pipe(pipe_serial);
9631 %}
9632
9633 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9634 predicate(needs_acquiring_load_exclusive(n));
9635 match(Set newval (GetAndAddL mem incr));
9636 ins_cost(VOLATILE_REF_COST + 1);
9637 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9638 ins_encode %{
9639 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9640 %}
9641 ins_pipe(pipe_serial);
9642 %}
9643
9644 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9645 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9646 match(Set dummy (GetAndAddL mem incr));
9647 ins_cost(VOLATILE_REF_COST);
9648 format %{ "get_and_addL_acq [$mem], $incr" %}
9649 ins_encode %{
9650 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9651 %}
9652 ins_pipe(pipe_serial);
9653 %}
9654
9655 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9656 predicate(needs_acquiring_load_exclusive(n));
9657 match(Set newval (GetAndAddI mem incr));
9658 ins_cost(VOLATILE_REF_COST + 1);
9659 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9660 ins_encode %{
9661 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9662 %}
9663 ins_pipe(pipe_serial);
9664 %}
9665
9666 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9667 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9668 match(Set dummy (GetAndAddI mem incr));
9669 ins_cost(VOLATILE_REF_COST);
9670 format %{ "get_and_addI_acq [$mem], $incr" %}
9671 ins_encode %{
9672 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9673 %}
9674 ins_pipe(pipe_serial);
9675 %}
9676
9677 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9678 predicate(needs_acquiring_load_exclusive(n));
9679 match(Set newval (GetAndAddI mem incr));
9680 ins_cost(VOLATILE_REF_COST + 1);
9681 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9682 ins_encode %{
9683 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9684 %}
9685 ins_pipe(pipe_serial);
9686 %}
9687
9688 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9689 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9690 match(Set dummy (GetAndAddI mem incr));
9691 ins_cost(VOLATILE_REF_COST);
9692 format %{ "get_and_addI_acq [$mem], $incr" %}
9693 ins_encode %{
9694 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9695 %}
9696 ins_pipe(pipe_serial);
9697 %}
9698
9699 // Manifest a CmpL result in an integer register.
9700 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9701 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9702 %{
9703 match(Set dst (CmpL3 src1 src2));
9704 effect(KILL flags);
9705
9706 ins_cost(INSN_COST * 6);
9707 format %{
9708 "cmp $src1, $src2"
9709 "csetw $dst, ne"
9710 "cnegw $dst, lt"
9711 %}
9712 // format %{ "CmpL3 $dst, $src1, $src2" %}
9713 ins_encode %{
9714 __ cmp($src1$$Register, $src2$$Register);
9715 __ csetw($dst$$Register, Assembler::NE);
9716 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9717 %}
9718
9719 ins_pipe(pipe_class_default);
9720 %}
9721
9722 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9723 %{
9724 match(Set dst (CmpL3 src1 src2));
9725 effect(KILL flags);
9726
9727 ins_cost(INSN_COST * 6);
9728 format %{
9729 "cmp $src1, $src2"
9730 "csetw $dst, ne"
9731 "cnegw $dst, lt"
9732 %}
9733 ins_encode %{
9734 int32_t con = (int32_t)$src2$$constant;
9735 if (con < 0) {
9736 __ adds(zr, $src1$$Register, -con);
9737 } else {
9738 __ subs(zr, $src1$$Register, con);
9739 }
9740 __ csetw($dst$$Register, Assembler::NE);
9741 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9742 %}
9743
9744 ins_pipe(pipe_class_default);
9745 %}
9746
9747 // ============================================================================
9748 // Conditional Move Instructions
9749
9750 // n.b. we have identical rules for both a signed compare op (cmpOp)
9751 // and an unsigned compare op (cmpOpU). it would be nice if we could
9752 // define an op class which merged both inputs and use it to type the
9753 // argument to a single rule. unfortunatelyt his fails because the
9754 // opclass does not live up to the COND_INTER interface of its
9755 // component operands. When the generic code tries to negate the
9756 // operand it ends up running the generci Machoper::negate method
9757 // which throws a ShouldNotHappen. So, we have to provide two flavours
9758 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9759
9760 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9761 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9762
9763 ins_cost(INSN_COST * 2);
9764 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9765
9766 ins_encode %{
9767 __ cselw(as_Register($dst$$reg),
9768 as_Register($src2$$reg),
9769 as_Register($src1$$reg),
9770 (Assembler::Condition)$cmp$$cmpcode);
9771 %}
9772
9773 ins_pipe(icond_reg_reg);
9774 %}
9775
9776 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9777 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9778
9779 ins_cost(INSN_COST * 2);
9780 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9781
9782 ins_encode %{
9783 __ cselw(as_Register($dst$$reg),
9784 as_Register($src2$$reg),
9785 as_Register($src1$$reg),
9786 (Assembler::Condition)$cmp$$cmpcode);
9787 %}
9788
9789 ins_pipe(icond_reg_reg);
9790 %}
9791
9792 // special cases where one arg is zero
9793
9794 // n.b. this is selected in preference to the rule above because it
9795 // avoids loading constant 0 into a source register
9796
9797 // TODO
9798 // we ought only to be able to cull one of these variants as the ideal
9799 // transforms ought always to order the zero consistently (to left/right?)
9800
9801 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9802 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9803
9804 ins_cost(INSN_COST * 2);
9805 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9806
9807 ins_encode %{
9808 __ cselw(as_Register($dst$$reg),
9809 as_Register($src$$reg),
9810 zr,
9811 (Assembler::Condition)$cmp$$cmpcode);
9812 %}
9813
9814 ins_pipe(icond_reg);
9815 %}
9816
9817 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9818 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9819
9820 ins_cost(INSN_COST * 2);
9821 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9822
9823 ins_encode %{
9824 __ cselw(as_Register($dst$$reg),
9825 as_Register($src$$reg),
9826 zr,
9827 (Assembler::Condition)$cmp$$cmpcode);
9828 %}
9829
9830 ins_pipe(icond_reg);
9831 %}
9832
9833 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9834 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9835
9836 ins_cost(INSN_COST * 2);
9837 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9838
9839 ins_encode %{
9840 __ cselw(as_Register($dst$$reg),
9841 zr,
9842 as_Register($src$$reg),
9843 (Assembler::Condition)$cmp$$cmpcode);
9844 %}
9845
9846 ins_pipe(icond_reg);
9847 %}
9848
9849 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9850 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9851
9852 ins_cost(INSN_COST * 2);
9853 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9854
9855 ins_encode %{
9856 __ cselw(as_Register($dst$$reg),
9857 zr,
9858 as_Register($src$$reg),
9859 (Assembler::Condition)$cmp$$cmpcode);
9860 %}
9861
9862 ins_pipe(icond_reg);
9863 %}
9864
9865 // special case for creating a boolean 0 or 1
9866
9867 // n.b. this is selected in preference to the rule above because it
9868 // avoids loading constants 0 and 1 into a source register
9869
9870 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9871 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9872
9873 ins_cost(INSN_COST * 2);
9874 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9875
9876 ins_encode %{
9877 // equivalently
9878 // cset(as_Register($dst$$reg),
9879 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9880 __ csincw(as_Register($dst$$reg),
9881 zr,
9882 zr,
9883 (Assembler::Condition)$cmp$$cmpcode);
9884 %}
9885
9886 ins_pipe(icond_none);
9887 %}
9888
9889 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9890 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9891
9892 ins_cost(INSN_COST * 2);
9893 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9894
9895 ins_encode %{
9896 // equivalently
9897 // cset(as_Register($dst$$reg),
9898 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9899 __ csincw(as_Register($dst$$reg),
9900 zr,
9901 zr,
9902 (Assembler::Condition)$cmp$$cmpcode);
9903 %}
9904
9905 ins_pipe(icond_none);
9906 %}
9907
9908 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9909 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9910
9911 ins_cost(INSN_COST * 2);
9912 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9913
9914 ins_encode %{
9915 __ csel(as_Register($dst$$reg),
9916 as_Register($src2$$reg),
9917 as_Register($src1$$reg),
9918 (Assembler::Condition)$cmp$$cmpcode);
9919 %}
9920
9921 ins_pipe(icond_reg_reg);
9922 %}
9923
9924 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9925 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9926
9927 ins_cost(INSN_COST * 2);
9928 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9929
9930 ins_encode %{
9931 __ csel(as_Register($dst$$reg),
9932 as_Register($src2$$reg),
9933 as_Register($src1$$reg),
9934 (Assembler::Condition)$cmp$$cmpcode);
9935 %}
9936
9937 ins_pipe(icond_reg_reg);
9938 %}
9939
9940 // special cases where one arg is zero
9941
9942 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9943 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9944
9945 ins_cost(INSN_COST * 2);
9946 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9947
9948 ins_encode %{
9949 __ csel(as_Register($dst$$reg),
9950 zr,
9951 as_Register($src$$reg),
9952 (Assembler::Condition)$cmp$$cmpcode);
9953 %}
9954
9955 ins_pipe(icond_reg);
9956 %}
9957
9958 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9959 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9960
9961 ins_cost(INSN_COST * 2);
9962 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9963
9964 ins_encode %{
9965 __ csel(as_Register($dst$$reg),
9966 zr,
9967 as_Register($src$$reg),
9968 (Assembler::Condition)$cmp$$cmpcode);
9969 %}
9970
9971 ins_pipe(icond_reg);
9972 %}
9973
9974 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9975 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9976
9977 ins_cost(INSN_COST * 2);
9978 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9979
9980 ins_encode %{
9981 __ csel(as_Register($dst$$reg),
9982 as_Register($src$$reg),
9983 zr,
9984 (Assembler::Condition)$cmp$$cmpcode);
9985 %}
9986
9987 ins_pipe(icond_reg);
9988 %}
9989
9990 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9991 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9992
9993 ins_cost(INSN_COST * 2);
9994 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9995
9996 ins_encode %{
9997 __ csel(as_Register($dst$$reg),
9998 as_Register($src$$reg),
9999 zr,
10000 (Assembler::Condition)$cmp$$cmpcode);
10001 %}
10002
10003 ins_pipe(icond_reg);
10004 %}
10005
10006 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10007 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10008
10009 ins_cost(INSN_COST * 2);
10010 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10011
10012 ins_encode %{
10013 __ csel(as_Register($dst$$reg),
10014 as_Register($src2$$reg),
10015 as_Register($src1$$reg),
10016 (Assembler::Condition)$cmp$$cmpcode);
10017 %}
10018
10019 ins_pipe(icond_reg_reg);
10020 %}
10021
10022 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10023 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10024
10025 ins_cost(INSN_COST * 2);
10026 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10027
10028 ins_encode %{
10029 __ csel(as_Register($dst$$reg),
10030 as_Register($src2$$reg),
10031 as_Register($src1$$reg),
10032 (Assembler::Condition)$cmp$$cmpcode);
10033 %}
10034
10035 ins_pipe(icond_reg_reg);
10036 %}
10037
10038 // special cases where one arg is zero
10039
10040 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10041 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10042
10043 ins_cost(INSN_COST * 2);
10044 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10045
10046 ins_encode %{
10047 __ csel(as_Register($dst$$reg),
10048 zr,
10049 as_Register($src$$reg),
10050 (Assembler::Condition)$cmp$$cmpcode);
10051 %}
10052
10053 ins_pipe(icond_reg);
10054 %}
10055
10056 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10057 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10058
10059 ins_cost(INSN_COST * 2);
10060 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10061
10062 ins_encode %{
10063 __ csel(as_Register($dst$$reg),
10064 zr,
10065 as_Register($src$$reg),
10066 (Assembler::Condition)$cmp$$cmpcode);
10067 %}
10068
10069 ins_pipe(icond_reg);
10070 %}
10071
10072 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10073 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10074
10075 ins_cost(INSN_COST * 2);
10076 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10077
10078 ins_encode %{
10079 __ csel(as_Register($dst$$reg),
10080 as_Register($src$$reg),
10081 zr,
10082 (Assembler::Condition)$cmp$$cmpcode);
10083 %}
10084
10085 ins_pipe(icond_reg);
10086 %}
10087
10088 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10089 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10090
10091 ins_cost(INSN_COST * 2);
10092 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10093
10094 ins_encode %{
10095 __ csel(as_Register($dst$$reg),
10096 as_Register($src$$reg),
10097 zr,
10098 (Assembler::Condition)$cmp$$cmpcode);
10099 %}
10100
10101 ins_pipe(icond_reg);
10102 %}
10103
10104 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10105 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10106
10107 ins_cost(INSN_COST * 2);
10108 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10109
10110 ins_encode %{
10111 __ cselw(as_Register($dst$$reg),
10112 as_Register($src2$$reg),
10113 as_Register($src1$$reg),
10114 (Assembler::Condition)$cmp$$cmpcode);
10115 %}
10116
10117 ins_pipe(icond_reg_reg);
10118 %}
10119
10120 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10121 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10122
10123 ins_cost(INSN_COST * 2);
10124 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10125
10126 ins_encode %{
10127 __ cselw(as_Register($dst$$reg),
10128 as_Register($src2$$reg),
10129 as_Register($src1$$reg),
10130 (Assembler::Condition)$cmp$$cmpcode);
10131 %}
10132
10133 ins_pipe(icond_reg_reg);
10134 %}
10135
10136 // special cases where one arg is zero
10137
10138 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10139 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10140
10141 ins_cost(INSN_COST * 2);
10142 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10143
10144 ins_encode %{
10145 __ cselw(as_Register($dst$$reg),
10146 zr,
10147 as_Register($src$$reg),
10148 (Assembler::Condition)$cmp$$cmpcode);
10149 %}
10150
10151 ins_pipe(icond_reg);
10152 %}
10153
10154 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10155 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10156
10157 ins_cost(INSN_COST * 2);
10158 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10159
10160 ins_encode %{
10161 __ cselw(as_Register($dst$$reg),
10162 zr,
10163 as_Register($src$$reg),
10164 (Assembler::Condition)$cmp$$cmpcode);
10165 %}
10166
10167 ins_pipe(icond_reg);
10168 %}
10169
10170 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10171 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10172
10173 ins_cost(INSN_COST * 2);
10174 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10175
10176 ins_encode %{
10177 __ cselw(as_Register($dst$$reg),
10178 as_Register($src$$reg),
10179 zr,
10180 (Assembler::Condition)$cmp$$cmpcode);
10181 %}
10182
10183 ins_pipe(icond_reg);
10184 %}
10185
10186 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10187 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10188
10189 ins_cost(INSN_COST * 2);
10190 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10191
10192 ins_encode %{
10193 __ cselw(as_Register($dst$$reg),
10194 as_Register($src$$reg),
10195 zr,
10196 (Assembler::Condition)$cmp$$cmpcode);
10197 %}
10198
10199 ins_pipe(icond_reg);
10200 %}
10201
10202 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10203 %{
10204 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10205
10206 ins_cost(INSN_COST * 3);
10207
10208 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10209 ins_encode %{
10210 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10211 __ fcsels(as_FloatRegister($dst$$reg),
10212 as_FloatRegister($src2$$reg),
10213 as_FloatRegister($src1$$reg),
10214 cond);
10215 %}
10216
10217 ins_pipe(fp_cond_reg_reg_s);
10218 %}
10219
10220 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10221 %{
10222 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10223
10224 ins_cost(INSN_COST * 3);
10225
10226 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10227 ins_encode %{
10228 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10229 __ fcsels(as_FloatRegister($dst$$reg),
10230 as_FloatRegister($src2$$reg),
10231 as_FloatRegister($src1$$reg),
10232 cond);
10233 %}
10234
10235 ins_pipe(fp_cond_reg_reg_s);
10236 %}
10237
10238 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10239 %{
10240 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10241
10242 ins_cost(INSN_COST * 3);
10243
10244 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10245 ins_encode %{
10246 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10247 __ fcseld(as_FloatRegister($dst$$reg),
10248 as_FloatRegister($src2$$reg),
10249 as_FloatRegister($src1$$reg),
10250 cond);
10251 %}
10252
10253 ins_pipe(fp_cond_reg_reg_d);
10254 %}
10255
10256 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10257 %{
10258 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10259
10260 ins_cost(INSN_COST * 3);
10261
10262 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10263 ins_encode %{
10264 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10265 __ fcseld(as_FloatRegister($dst$$reg),
10266 as_FloatRegister($src2$$reg),
10267 as_FloatRegister($src1$$reg),
10268 cond);
10269 %}
10270
10271 ins_pipe(fp_cond_reg_reg_d);
10272 %}
10273
10274 // ============================================================================
10275 // Arithmetic Instructions
10276 //
10277
10278 // Integer Addition
10279
10280 // TODO
10281 // these currently employ operations which do not set CR and hence are
10282 // not flagged as killing CR but we would like to isolate the cases
10283 // where we want to set flags from those where we don't. need to work
10284 // out how to do that.
10285
10286 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10287 match(Set dst (AddI src1 src2));
10288
10289 ins_cost(INSN_COST);
10290 format %{ "addw $dst, $src1, $src2" %}
10291
10292 ins_encode %{
10293 __ addw(as_Register($dst$$reg),
10294 as_Register($src1$$reg),
10295 as_Register($src2$$reg));
10296 %}
10297
10298 ins_pipe(ialu_reg_reg);
10299 %}
10300
10301 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10302 match(Set dst (AddI src1 src2));
10303
10304 ins_cost(INSN_COST);
10305 format %{ "addw $dst, $src1, $src2" %}
10306
10307 // use opcode to indicate that this is an add not a sub
10308 opcode(0x0);
10309
10310 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10311
10312 ins_pipe(ialu_reg_imm);
10313 %}
10314
10315 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10316 match(Set dst (AddI (ConvL2I src1) src2));
10317
10318 ins_cost(INSN_COST);
10319 format %{ "addw $dst, $src1, $src2" %}
10320
10321 // use opcode to indicate that this is an add not a sub
10322 opcode(0x0);
10323
10324 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10325
10326 ins_pipe(ialu_reg_imm);
10327 %}
10328
10329 // Pointer Addition
10330 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10331 match(Set dst (AddP src1 src2));
10332
10333 ins_cost(INSN_COST);
10334 format %{ "add $dst, $src1, $src2\t# ptr" %}
10335
10336 ins_encode %{
10337 __ add(as_Register($dst$$reg),
10338 as_Register($src1$$reg),
10339 as_Register($src2$$reg));
10340 %}
10341
10342 ins_pipe(ialu_reg_reg);
10343 %}
10344
10345 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10346 match(Set dst (AddP src1 (ConvI2L src2)));
10347
10348 ins_cost(1.9 * INSN_COST);
10349 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10350
10351 ins_encode %{
10352 __ add(as_Register($dst$$reg),
10353 as_Register($src1$$reg),
10354 as_Register($src2$$reg), ext::sxtw);
10355 %}
10356
10357 ins_pipe(ialu_reg_reg);
10358 %}
10359
10360 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10361 match(Set dst (AddP src1 (LShiftL src2 scale)));
10362
10363 ins_cost(1.9 * INSN_COST);
10364 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10365
10366 ins_encode %{
10367 __ lea(as_Register($dst$$reg),
10368 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10369 Address::lsl($scale$$constant)));
10370 %}
10371
10372 ins_pipe(ialu_reg_reg_shift);
10373 %}
10374
10375 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10376 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10377
10378 ins_cost(1.9 * INSN_COST);
10379 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10380
10381 ins_encode %{
10382 __ lea(as_Register($dst$$reg),
10383 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10384 Address::sxtw($scale$$constant)));
10385 %}
10386
10387 ins_pipe(ialu_reg_reg_shift);
10388 %}
10389
10390 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10391 match(Set dst (LShiftL (ConvI2L src) scale));
10392
10393 ins_cost(INSN_COST);
10394 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10395
10396 ins_encode %{
10397 __ sbfiz(as_Register($dst$$reg),
10398 as_Register($src$$reg),
10399 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10400 %}
10401
10402 ins_pipe(ialu_reg_shift);
10403 %}
10404
10405 // Pointer Immediate Addition
10406 // n.b. this needs to be more expensive than using an indirect memory
10407 // operand
10408 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10409 match(Set dst (AddP src1 src2));
10410
10411 ins_cost(INSN_COST);
10412 format %{ "add $dst, $src1, $src2\t# ptr" %}
10413
10414 // use opcode to indicate that this is an add not a sub
10415 opcode(0x0);
10416
10417 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10418
10419 ins_pipe(ialu_reg_imm);
10420 %}
10421
10422 // Long Addition
10423 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10424
10425 match(Set dst (AddL src1 src2));
10426
10427 ins_cost(INSN_COST);
10428 format %{ "add $dst, $src1, $src2" %}
10429
10430 ins_encode %{
10431 __ add(as_Register($dst$$reg),
10432 as_Register($src1$$reg),
10433 as_Register($src2$$reg));
10434 %}
10435
10436 ins_pipe(ialu_reg_reg);
10437 %}
10438
10439 // No constant pool entries requiredLong Immediate Addition.
10440 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10441 match(Set dst (AddL src1 src2));
10442
10443 ins_cost(INSN_COST);
10444 format %{ "add $dst, $src1, $src2" %}
10445
10446 // use opcode to indicate that this is an add not a sub
10447 opcode(0x0);
10448
10449 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10450
10451 ins_pipe(ialu_reg_imm);
10452 %}
10453
10454 // Integer Subtraction
10455 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10456 match(Set dst (SubI src1 src2));
10457
10458 ins_cost(INSN_COST);
10459 format %{ "subw $dst, $src1, $src2" %}
10460
10461 ins_encode %{
10462 __ subw(as_Register($dst$$reg),
10463 as_Register($src1$$reg),
10464 as_Register($src2$$reg));
10465 %}
10466
10467 ins_pipe(ialu_reg_reg);
10468 %}
10469
10470 // Immediate Subtraction
10471 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10472 match(Set dst (SubI src1 src2));
10473
10474 ins_cost(INSN_COST);
10475 format %{ "subw $dst, $src1, $src2" %}
10476
10477 // use opcode to indicate that this is a sub not an add
10478 opcode(0x1);
10479
10480 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10481
10482 ins_pipe(ialu_reg_imm);
10483 %}
10484
10485 // Long Subtraction
10486 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10487
10488 match(Set dst (SubL src1 src2));
10489
10490 ins_cost(INSN_COST);
10491 format %{ "sub $dst, $src1, $src2" %}
10492
10493 ins_encode %{
10494 __ sub(as_Register($dst$$reg),
10495 as_Register($src1$$reg),
10496 as_Register($src2$$reg));
10497 %}
10498
10499 ins_pipe(ialu_reg_reg);
10500 %}
10501
10502 // No constant pool entries requiredLong Immediate Subtraction.
10503 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10504 match(Set dst (SubL src1 src2));
10505
10506 ins_cost(INSN_COST);
10507 format %{ "sub$dst, $src1, $src2" %}
10508
10509 // use opcode to indicate that this is a sub not an add
10510 opcode(0x1);
10511
10512 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10513
10514 ins_pipe(ialu_reg_imm);
10515 %}
10516
10517 // Integer Negation (special case for sub)
10518
10519 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10520 match(Set dst (SubI zero src));
10521
10522 ins_cost(INSN_COST);
10523 format %{ "negw $dst, $src\t# int" %}
10524
10525 ins_encode %{
10526 __ negw(as_Register($dst$$reg),
10527 as_Register($src$$reg));
10528 %}
10529
10530 ins_pipe(ialu_reg);
10531 %}
10532
10533 // Long Negation
10534
10535 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10536 match(Set dst (SubL zero src));
10537
10538 ins_cost(INSN_COST);
10539 format %{ "neg $dst, $src\t# long" %}
10540
10541 ins_encode %{
10542 __ neg(as_Register($dst$$reg),
10543 as_Register($src$$reg));
10544 %}
10545
10546 ins_pipe(ialu_reg);
10547 %}
10548
10549 // Integer Multiply
10550
10551 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10552 match(Set dst (MulI src1 src2));
10553
10554 ins_cost(INSN_COST * 3);
10555 format %{ "mulw $dst, $src1, $src2" %}
10556
10557 ins_encode %{
10558 __ mulw(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 as_Register($src2$$reg));
10561 %}
10562
10563 ins_pipe(imul_reg_reg);
10564 %}
10565
10566 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10567 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10568
10569 ins_cost(INSN_COST * 3);
10570 format %{ "smull $dst, $src1, $src2" %}
10571
10572 ins_encode %{
10573 __ smull(as_Register($dst$$reg),
10574 as_Register($src1$$reg),
10575 as_Register($src2$$reg));
10576 %}
10577
10578 ins_pipe(imul_reg_reg);
10579 %}
10580
10581 // Long Multiply
10582
10583 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10584 match(Set dst (MulL src1 src2));
10585
10586 ins_cost(INSN_COST * 5);
10587 format %{ "mul $dst, $src1, $src2" %}
10588
10589 ins_encode %{
10590 __ mul(as_Register($dst$$reg),
10591 as_Register($src1$$reg),
10592 as_Register($src2$$reg));
10593 %}
10594
10595 ins_pipe(lmul_reg_reg);
10596 %}
10597
10598 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10599 %{
10600 match(Set dst (MulHiL src1 src2));
10601
10602 ins_cost(INSN_COST * 7);
10603 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10604
10605 ins_encode %{
10606 __ smulh(as_Register($dst$$reg),
10607 as_Register($src1$$reg),
10608 as_Register($src2$$reg));
10609 %}
10610
10611 ins_pipe(lmul_reg_reg);
10612 %}
10613
10614 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10615 %{
10616 match(Set dst (UMulHiL src1 src2));
10617
10618 ins_cost(INSN_COST * 7);
10619 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10620
10621 ins_encode %{
10622 __ umulh(as_Register($dst$$reg),
10623 as_Register($src1$$reg),
10624 as_Register($src2$$reg));
10625 %}
10626
10627 ins_pipe(lmul_reg_reg);
10628 %}
10629
10630 // Combined Integer Multiply & Add/Sub
10631
10632 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10633 match(Set dst (AddI src3 (MulI src1 src2)));
10634
10635 ins_cost(INSN_COST * 3);
10636 format %{ "madd $dst, $src1, $src2, $src3" %}
10637
10638 ins_encode %{
10639 __ maddw(as_Register($dst$$reg),
10640 as_Register($src1$$reg),
10641 as_Register($src2$$reg),
10642 as_Register($src3$$reg));
10643 %}
10644
10645 ins_pipe(imac_reg_reg);
10646 %}
10647
10648 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10649 match(Set dst (SubI src3 (MulI src1 src2)));
10650
10651 ins_cost(INSN_COST * 3);
10652 format %{ "msub $dst, $src1, $src2, $src3" %}
10653
10654 ins_encode %{
10655 __ msubw(as_Register($dst$$reg),
10656 as_Register($src1$$reg),
10657 as_Register($src2$$reg),
10658 as_Register($src3$$reg));
10659 %}
10660
10661 ins_pipe(imac_reg_reg);
10662 %}
10663
10664 // Combined Integer Multiply & Neg
10665
10666 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10667 match(Set dst (MulI (SubI zero src1) src2));
10668
10669 ins_cost(INSN_COST * 3);
10670 format %{ "mneg $dst, $src1, $src2" %}
10671
10672 ins_encode %{
10673 __ mnegw(as_Register($dst$$reg),
10674 as_Register($src1$$reg),
10675 as_Register($src2$$reg));
10676 %}
10677
10678 ins_pipe(imac_reg_reg);
10679 %}
10680
10681 // Combined Long Multiply & Add/Sub
10682
10683 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10684 match(Set dst (AddL src3 (MulL src1 src2)));
10685
10686 ins_cost(INSN_COST * 5);
10687 format %{ "madd $dst, $src1, $src2, $src3" %}
10688
10689 ins_encode %{
10690 __ madd(as_Register($dst$$reg),
10691 as_Register($src1$$reg),
10692 as_Register($src2$$reg),
10693 as_Register($src3$$reg));
10694 %}
10695
10696 ins_pipe(lmac_reg_reg);
10697 %}
10698
10699 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10700 match(Set dst (SubL src3 (MulL src1 src2)));
10701
10702 ins_cost(INSN_COST * 5);
10703 format %{ "msub $dst, $src1, $src2, $src3" %}
10704
10705 ins_encode %{
10706 __ msub(as_Register($dst$$reg),
10707 as_Register($src1$$reg),
10708 as_Register($src2$$reg),
10709 as_Register($src3$$reg));
10710 %}
10711
10712 ins_pipe(lmac_reg_reg);
10713 %}
10714
10715 // Combined Long Multiply & Neg
10716
10717 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10718 match(Set dst (MulL (SubL zero src1) src2));
10719
10720 ins_cost(INSN_COST * 5);
10721 format %{ "mneg $dst, $src1, $src2" %}
10722
10723 ins_encode %{
10724 __ mneg(as_Register($dst$$reg),
10725 as_Register($src1$$reg),
10726 as_Register($src2$$reg));
10727 %}
10728
10729 ins_pipe(lmac_reg_reg);
10730 %}
10731
10732 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10733
10734 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10735 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10736
10737 ins_cost(INSN_COST * 3);
10738 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10739
10740 ins_encode %{
10741 __ smaddl(as_Register($dst$$reg),
10742 as_Register($src1$$reg),
10743 as_Register($src2$$reg),
10744 as_Register($src3$$reg));
10745 %}
10746
10747 ins_pipe(imac_reg_reg);
10748 %}
10749
10750 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10751 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10752
10753 ins_cost(INSN_COST * 3);
10754 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10755
10756 ins_encode %{
10757 __ smsubl(as_Register($dst$$reg),
10758 as_Register($src1$$reg),
10759 as_Register($src2$$reg),
10760 as_Register($src3$$reg));
10761 %}
10762
10763 ins_pipe(imac_reg_reg);
10764 %}
10765
10766 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10767 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10768
10769 ins_cost(INSN_COST * 3);
10770 format %{ "smnegl $dst, $src1, $src2" %}
10771
10772 ins_encode %{
10773 __ smnegl(as_Register($dst$$reg),
10774 as_Register($src1$$reg),
10775 as_Register($src2$$reg));
10776 %}
10777
10778 ins_pipe(imac_reg_reg);
10779 %}
10780
10781 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10782
10783 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10784 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10785
10786 ins_cost(INSN_COST * 5);
10787 format %{ "mulw rscratch1, $src1, $src2\n\t"
10788 "maddw $dst, $src3, $src4, rscratch1" %}
10789
10790 ins_encode %{
10791 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10792 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10793
10794 ins_pipe(imac_reg_reg);
10795 %}
10796
10797 // Integer Divide
10798
10799 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10800 match(Set dst (DivI src1 src2));
10801
10802 ins_cost(INSN_COST * 19);
10803 format %{ "sdivw $dst, $src1, $src2" %}
10804
10805 ins_encode(aarch64_enc_divw(dst, src1, src2));
10806 ins_pipe(idiv_reg_reg);
10807 %}
10808
10809 // Long Divide
10810
10811 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10812 match(Set dst (DivL src1 src2));
10813
10814 ins_cost(INSN_COST * 35);
10815 format %{ "sdiv $dst, $src1, $src2" %}
10816
10817 ins_encode(aarch64_enc_div(dst, src1, src2));
10818 ins_pipe(ldiv_reg_reg);
10819 %}
10820
10821 // Integer Remainder
10822
10823 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10824 match(Set dst (ModI src1 src2));
10825
10826 ins_cost(INSN_COST * 22);
10827 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10828 "msubw $dst, rscratch1, $src2, $src1" %}
10829
10830 ins_encode(aarch64_enc_modw(dst, src1, src2));
10831 ins_pipe(idiv_reg_reg);
10832 %}
10833
10834 // Long Remainder
10835
10836 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10837 match(Set dst (ModL src1 src2));
10838
10839 ins_cost(INSN_COST * 38);
10840 format %{ "sdiv rscratch1, $src1, $src2\n"
10841 "msub $dst, rscratch1, $src2, $src1" %}
10842
10843 ins_encode(aarch64_enc_mod(dst, src1, src2));
10844 ins_pipe(ldiv_reg_reg);
10845 %}
10846
10847 // Unsigned Integer Divide
10848
10849 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10850 match(Set dst (UDivI src1 src2));
10851
10852 ins_cost(INSN_COST * 19);
10853 format %{ "udivw $dst, $src1, $src2" %}
10854
10855 ins_encode %{
10856 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10857 %}
10858
10859 ins_pipe(idiv_reg_reg);
10860 %}
10861
10862 // Unsigned Long Divide
10863
10864 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10865 match(Set dst (UDivL src1 src2));
10866
10867 ins_cost(INSN_COST * 35);
10868 format %{ "udiv $dst, $src1, $src2" %}
10869
10870 ins_encode %{
10871 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10872 %}
10873
10874 ins_pipe(ldiv_reg_reg);
10875 %}
10876
10877 // Unsigned Integer Remainder
10878
10879 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10880 match(Set dst (UModI src1 src2));
10881
10882 ins_cost(INSN_COST * 22);
10883 format %{ "udivw rscratch1, $src1, $src2\n\t"
10884 "msubw $dst, rscratch1, $src2, $src1" %}
10885
10886 ins_encode %{
10887 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10888 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10889 %}
10890
10891 ins_pipe(idiv_reg_reg);
10892 %}
10893
10894 // Unsigned Long Remainder
10895
10896 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10897 match(Set dst (UModL src1 src2));
10898
10899 ins_cost(INSN_COST * 38);
10900 format %{ "udiv rscratch1, $src1, $src2\n"
10901 "msub $dst, rscratch1, $src2, $src1" %}
10902
10903 ins_encode %{
10904 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10905 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10906 %}
10907
10908 ins_pipe(ldiv_reg_reg);
10909 %}
10910
10911 // Integer Shifts
10912
10913 // Shift Left Register
10914 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10915 match(Set dst (LShiftI src1 src2));
10916
10917 ins_cost(INSN_COST * 2);
10918 format %{ "lslvw $dst, $src1, $src2" %}
10919
10920 ins_encode %{
10921 __ lslvw(as_Register($dst$$reg),
10922 as_Register($src1$$reg),
10923 as_Register($src2$$reg));
10924 %}
10925
10926 ins_pipe(ialu_reg_reg_vshift);
10927 %}
10928
10929 // Shift Left Immediate
10930 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10931 match(Set dst (LShiftI src1 src2));
10932
10933 ins_cost(INSN_COST);
10934 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10935
10936 ins_encode %{
10937 __ lslw(as_Register($dst$$reg),
10938 as_Register($src1$$reg),
10939 $src2$$constant & 0x1f);
10940 %}
10941
10942 ins_pipe(ialu_reg_shift);
10943 %}
10944
10945 // Shift Right Logical Register
10946 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10947 match(Set dst (URShiftI src1 src2));
10948
10949 ins_cost(INSN_COST * 2);
10950 format %{ "lsrvw $dst, $src1, $src2" %}
10951
10952 ins_encode %{
10953 __ lsrvw(as_Register($dst$$reg),
10954 as_Register($src1$$reg),
10955 as_Register($src2$$reg));
10956 %}
10957
10958 ins_pipe(ialu_reg_reg_vshift);
10959 %}
10960
10961 // Shift Right Logical Immediate
10962 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10963 match(Set dst (URShiftI src1 src2));
10964
10965 ins_cost(INSN_COST);
10966 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10967
10968 ins_encode %{
10969 __ lsrw(as_Register($dst$$reg),
10970 as_Register($src1$$reg),
10971 $src2$$constant & 0x1f);
10972 %}
10973
10974 ins_pipe(ialu_reg_shift);
10975 %}
10976
10977 // Shift Right Arithmetic Register
10978 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10979 match(Set dst (RShiftI src1 src2));
10980
10981 ins_cost(INSN_COST * 2);
10982 format %{ "asrvw $dst, $src1, $src2" %}
10983
10984 ins_encode %{
10985 __ asrvw(as_Register($dst$$reg),
10986 as_Register($src1$$reg),
10987 as_Register($src2$$reg));
10988 %}
10989
10990 ins_pipe(ialu_reg_reg_vshift);
10991 %}
10992
10993 // Shift Right Arithmetic Immediate
10994 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10995 match(Set dst (RShiftI src1 src2));
10996
10997 ins_cost(INSN_COST);
10998 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10999
11000 ins_encode %{
11001 __ asrw(as_Register($dst$$reg),
11002 as_Register($src1$$reg),
11003 $src2$$constant & 0x1f);
11004 %}
11005
11006 ins_pipe(ialu_reg_shift);
11007 %}
11008
11009 // Combined Int Mask and Right Shift (using UBFM)
11010 // TODO
11011
11012 // Long Shifts
11013
11014 // Shift Left Register
11015 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11016 match(Set dst (LShiftL src1 src2));
11017
11018 ins_cost(INSN_COST * 2);
11019 format %{ "lslv $dst, $src1, $src2" %}
11020
11021 ins_encode %{
11022 __ lslv(as_Register($dst$$reg),
11023 as_Register($src1$$reg),
11024 as_Register($src2$$reg));
11025 %}
11026
11027 ins_pipe(ialu_reg_reg_vshift);
11028 %}
11029
11030 // Shift Left Immediate
11031 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11032 match(Set dst (LShiftL src1 src2));
11033
11034 ins_cost(INSN_COST);
11035 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11036
11037 ins_encode %{
11038 __ lsl(as_Register($dst$$reg),
11039 as_Register($src1$$reg),
11040 $src2$$constant & 0x3f);
11041 %}
11042
11043 ins_pipe(ialu_reg_shift);
11044 %}
11045
11046 // Shift Right Logical Register
11047 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11048 match(Set dst (URShiftL src1 src2));
11049
11050 ins_cost(INSN_COST * 2);
11051 format %{ "lsrv $dst, $src1, $src2" %}
11052
11053 ins_encode %{
11054 __ lsrv(as_Register($dst$$reg),
11055 as_Register($src1$$reg),
11056 as_Register($src2$$reg));
11057 %}
11058
11059 ins_pipe(ialu_reg_reg_vshift);
11060 %}
11061
11062 // Shift Right Logical Immediate
11063 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11064 match(Set dst (URShiftL src1 src2));
11065
11066 ins_cost(INSN_COST);
11067 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11068
11069 ins_encode %{
11070 __ lsr(as_Register($dst$$reg),
11071 as_Register($src1$$reg),
11072 $src2$$constant & 0x3f);
11073 %}
11074
11075 ins_pipe(ialu_reg_shift);
11076 %}
11077
11078 // A special-case pattern for card table stores.
11079 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11080 match(Set dst (URShiftL (CastP2X src1) src2));
11081
11082 ins_cost(INSN_COST);
11083 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11084
11085 ins_encode %{
11086 __ lsr(as_Register($dst$$reg),
11087 as_Register($src1$$reg),
11088 $src2$$constant & 0x3f);
11089 %}
11090
11091 ins_pipe(ialu_reg_shift);
11092 %}
11093
11094 // Shift Right Arithmetic Register
11095 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11096 match(Set dst (RShiftL src1 src2));
11097
11098 ins_cost(INSN_COST * 2);
11099 format %{ "asrv $dst, $src1, $src2" %}
11100
11101 ins_encode %{
11102 __ asrv(as_Register($dst$$reg),
11103 as_Register($src1$$reg),
11104 as_Register($src2$$reg));
11105 %}
11106
11107 ins_pipe(ialu_reg_reg_vshift);
11108 %}
11109
11110 // Shift Right Arithmetic Immediate
11111 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11112 match(Set dst (RShiftL src1 src2));
11113
11114 ins_cost(INSN_COST);
11115 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11116
11117 ins_encode %{
11118 __ asr(as_Register($dst$$reg),
11119 as_Register($src1$$reg),
11120 $src2$$constant & 0x3f);
11121 %}
11122
11123 ins_pipe(ialu_reg_shift);
11124 %}
11125
11126 // BEGIN This section of the file is automatically generated. Do not edit --------------
11127 // This section is generated from aarch64_ad.m4
11128
11129
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 instruct regL_not_reg(iRegLNoSp dst,
11133 iRegL src1, immL_M1 m1,
11134 rFlagsReg cr) %{
11135 match(Set dst (XorL src1 m1));
11136 ins_cost(INSN_COST);
11137 format %{ "eon $dst, $src1, zr" %}
11138
11139 ins_encode %{
11140 __ eon(as_Register($dst$$reg),
11141 as_Register($src1$$reg),
11142 zr,
11143 Assembler::LSL, 0);
11144 %}
11145
11146 ins_pipe(ialu_reg);
11147 %}
11148
11149 // This pattern is automatically generated from aarch64_ad.m4
11150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11151 instruct regI_not_reg(iRegINoSp dst,
11152 iRegIorL2I src1, immI_M1 m1,
11153 rFlagsReg cr) %{
11154 match(Set dst (XorI src1 m1));
11155 ins_cost(INSN_COST);
11156 format %{ "eonw $dst, $src1, zr" %}
11157
11158 ins_encode %{
11159 __ eonw(as_Register($dst$$reg),
11160 as_Register($src1$$reg),
11161 zr,
11162 Assembler::LSL, 0);
11163 %}
11164
11165 ins_pipe(ialu_reg);
11166 %}
11167
11168 // This pattern is automatically generated from aarch64_ad.m4
11169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11170 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11171 immI0 zero, iRegIorL2I src1, immI src2) %{
11172 match(Set dst (SubI zero (URShiftI src1 src2)));
11173
11174 ins_cost(1.9 * INSN_COST);
11175 format %{ "negw $dst, $src1, LSR $src2" %}
11176
11177 ins_encode %{
11178 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11179 Assembler::LSR, $src2$$constant & 0x1f);
11180 %}
11181
11182 ins_pipe(ialu_reg_shift);
11183 %}
11184
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11188 immI0 zero, iRegIorL2I src1, immI src2) %{
11189 match(Set dst (SubI zero (RShiftI src1 src2)));
11190
11191 ins_cost(1.9 * INSN_COST);
11192 format %{ "negw $dst, $src1, ASR $src2" %}
11193
11194 ins_encode %{
11195 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11196 Assembler::ASR, $src2$$constant & 0x1f);
11197 %}
11198
11199 ins_pipe(ialu_reg_shift);
11200 %}
11201
11202 // This pattern is automatically generated from aarch64_ad.m4
11203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11204 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11205 immI0 zero, iRegIorL2I src1, immI src2) %{
11206 match(Set dst (SubI zero (LShiftI src1 src2)));
11207
11208 ins_cost(1.9 * INSN_COST);
11209 format %{ "negw $dst, $src1, LSL $src2" %}
11210
11211 ins_encode %{
11212 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11213 Assembler::LSL, $src2$$constant & 0x1f);
11214 %}
11215
11216 ins_pipe(ialu_reg_shift);
11217 %}
11218
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11222 immL0 zero, iRegL src1, immI src2) %{
11223 match(Set dst (SubL zero (URShiftL src1 src2)));
11224
11225 ins_cost(1.9 * INSN_COST);
11226 format %{ "neg $dst, $src1, LSR $src2" %}
11227
11228 ins_encode %{
11229 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11230 Assembler::LSR, $src2$$constant & 0x3f);
11231 %}
11232
11233 ins_pipe(ialu_reg_shift);
11234 %}
11235
11236 // This pattern is automatically generated from aarch64_ad.m4
11237 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11238 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11239 immL0 zero, iRegL src1, immI src2) %{
11240 match(Set dst (SubL zero (RShiftL src1 src2)));
11241
11242 ins_cost(1.9 * INSN_COST);
11243 format %{ "neg $dst, $src1, ASR $src2" %}
11244
11245 ins_encode %{
11246 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11247 Assembler::ASR, $src2$$constant & 0x3f);
11248 %}
11249
11250 ins_pipe(ialu_reg_shift);
11251 %}
11252
11253 // This pattern is automatically generated from aarch64_ad.m4
11254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11255 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11256 immL0 zero, iRegL src1, immI src2) %{
11257 match(Set dst (SubL zero (LShiftL src1 src2)));
11258
11259 ins_cost(1.9 * INSN_COST);
11260 format %{ "neg $dst, $src1, LSL $src2" %}
11261
11262 ins_encode %{
11263 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11264 Assembler::LSL, $src2$$constant & 0x3f);
11265 %}
11266
11267 ins_pipe(ialu_reg_shift);
11268 %}
11269
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 instruct AndI_reg_not_reg(iRegINoSp dst,
11273 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11274 match(Set dst (AndI src1 (XorI src2 m1)));
11275 ins_cost(INSN_COST);
11276 format %{ "bicw $dst, $src1, $src2" %}
11277
11278 ins_encode %{
11279 __ bicw(as_Register($dst$$reg),
11280 as_Register($src1$$reg),
11281 as_Register($src2$$reg),
11282 Assembler::LSL, 0);
11283 %}
11284
11285 ins_pipe(ialu_reg_reg);
11286 %}
11287
11288 // This pattern is automatically generated from aarch64_ad.m4
11289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11290 instruct AndL_reg_not_reg(iRegLNoSp dst,
11291 iRegL src1, iRegL src2, immL_M1 m1) %{
11292 match(Set dst (AndL src1 (XorL src2 m1)));
11293 ins_cost(INSN_COST);
11294 format %{ "bic $dst, $src1, $src2" %}
11295
11296 ins_encode %{
11297 __ bic(as_Register($dst$$reg),
11298 as_Register($src1$$reg),
11299 as_Register($src2$$reg),
11300 Assembler::LSL, 0);
11301 %}
11302
11303 ins_pipe(ialu_reg_reg);
11304 %}
11305
11306 // This pattern is automatically generated from aarch64_ad.m4
11307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11308 instruct OrI_reg_not_reg(iRegINoSp dst,
11309 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11310 match(Set dst (OrI src1 (XorI src2 m1)));
11311 ins_cost(INSN_COST);
11312 format %{ "ornw $dst, $src1, $src2" %}
11313
11314 ins_encode %{
11315 __ ornw(as_Register($dst$$reg),
11316 as_Register($src1$$reg),
11317 as_Register($src2$$reg),
11318 Assembler::LSL, 0);
11319 %}
11320
11321 ins_pipe(ialu_reg_reg);
11322 %}
11323
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct OrL_reg_not_reg(iRegLNoSp dst,
11327 iRegL src1, iRegL src2, immL_M1 m1) %{
11328 match(Set dst (OrL src1 (XorL src2 m1)));
11329 ins_cost(INSN_COST);
11330 format %{ "orn $dst, $src1, $src2" %}
11331
11332 ins_encode %{
11333 __ orn(as_Register($dst$$reg),
11334 as_Register($src1$$reg),
11335 as_Register($src2$$reg),
11336 Assembler::LSL, 0);
11337 %}
11338
11339 ins_pipe(ialu_reg_reg);
11340 %}
11341
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 instruct XorI_reg_not_reg(iRegINoSp dst,
11345 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11346 match(Set dst (XorI m1 (XorI src2 src1)));
11347 ins_cost(INSN_COST);
11348 format %{ "eonw $dst, $src1, $src2" %}
11349
11350 ins_encode %{
11351 __ eonw(as_Register($dst$$reg),
11352 as_Register($src1$$reg),
11353 as_Register($src2$$reg),
11354 Assembler::LSL, 0);
11355 %}
11356
11357 ins_pipe(ialu_reg_reg);
11358 %}
11359
11360 // This pattern is automatically generated from aarch64_ad.m4
11361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11362 instruct XorL_reg_not_reg(iRegLNoSp dst,
11363 iRegL src1, iRegL src2, immL_M1 m1) %{
11364 match(Set dst (XorL m1 (XorL src2 src1)));
11365 ins_cost(INSN_COST);
11366 format %{ "eon $dst, $src1, $src2" %}
11367
11368 ins_encode %{
11369 __ eon(as_Register($dst$$reg),
11370 as_Register($src1$$reg),
11371 as_Register($src2$$reg),
11372 Assembler::LSL, 0);
11373 %}
11374
11375 ins_pipe(ialu_reg_reg);
11376 %}
11377
11378 // This pattern is automatically generated from aarch64_ad.m4
11379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11380 // val & (-1 ^ (val >>> shift)) ==> bicw
11381 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11382 iRegIorL2I src1, iRegIorL2I src2,
11383 immI src3, immI_M1 src4) %{
11384 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11385 ins_cost(1.9 * INSN_COST);
11386 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11387
11388 ins_encode %{
11389 __ bicw(as_Register($dst$$reg),
11390 as_Register($src1$$reg),
11391 as_Register($src2$$reg),
11392 Assembler::LSR,
11393 $src3$$constant & 0x1f);
11394 %}
11395
11396 ins_pipe(ialu_reg_reg_shift);
11397 %}
11398
11399 // This pattern is automatically generated from aarch64_ad.m4
11400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11401 // val & (-1 ^ (val >>> shift)) ==> bic
11402 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11403 iRegL src1, iRegL src2,
11404 immI src3, immL_M1 src4) %{
11405 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11406 ins_cost(1.9 * INSN_COST);
11407 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11408
11409 ins_encode %{
11410 __ bic(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 as_Register($src2$$reg),
11413 Assembler::LSR,
11414 $src3$$constant & 0x3f);
11415 %}
11416
11417 ins_pipe(ialu_reg_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422 // val & (-1 ^ (val >> shift)) ==> bicw
11423 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11424 iRegIorL2I src1, iRegIorL2I src2,
11425 immI src3, immI_M1 src4) %{
11426 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11427 ins_cost(1.9 * INSN_COST);
11428 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11429
11430 ins_encode %{
11431 __ bicw(as_Register($dst$$reg),
11432 as_Register($src1$$reg),
11433 as_Register($src2$$reg),
11434 Assembler::ASR,
11435 $src3$$constant & 0x1f);
11436 %}
11437
11438 ins_pipe(ialu_reg_reg_shift);
11439 %}
11440
11441 // This pattern is automatically generated from aarch64_ad.m4
11442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11443 // val & (-1 ^ (val >> shift)) ==> bic
11444 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11445 iRegL src1, iRegL src2,
11446 immI src3, immL_M1 src4) %{
11447 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11448 ins_cost(1.9 * INSN_COST);
11449 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11450
11451 ins_encode %{
11452 __ bic(as_Register($dst$$reg),
11453 as_Register($src1$$reg),
11454 as_Register($src2$$reg),
11455 Assembler::ASR,
11456 $src3$$constant & 0x3f);
11457 %}
11458
11459 ins_pipe(ialu_reg_reg_shift);
11460 %}
11461
11462 // This pattern is automatically generated from aarch64_ad.m4
11463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11464 // val & (-1 ^ (val ror shift)) ==> bicw
11465 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11466 iRegIorL2I src1, iRegIorL2I src2,
11467 immI src3, immI_M1 src4) %{
11468 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11469 ins_cost(1.9 * INSN_COST);
11470 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11471
11472 ins_encode %{
11473 __ bicw(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 as_Register($src2$$reg),
11476 Assembler::ROR,
11477 $src3$$constant & 0x1f);
11478 %}
11479
11480 ins_pipe(ialu_reg_reg_shift);
11481 %}
11482
11483 // This pattern is automatically generated from aarch64_ad.m4
11484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11485 // val & (-1 ^ (val ror shift)) ==> bic
11486 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11487 iRegL src1, iRegL src2,
11488 immI src3, immL_M1 src4) %{
11489 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11490 ins_cost(1.9 * INSN_COST);
11491 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11492
11493 ins_encode %{
11494 __ bic(as_Register($dst$$reg),
11495 as_Register($src1$$reg),
11496 as_Register($src2$$reg),
11497 Assembler::ROR,
11498 $src3$$constant & 0x3f);
11499 %}
11500
11501 ins_pipe(ialu_reg_reg_shift);
11502 %}
11503
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 // val & (-1 ^ (val << shift)) ==> bicw
11507 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11508 iRegIorL2I src1, iRegIorL2I src2,
11509 immI src3, immI_M1 src4) %{
11510 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11511 ins_cost(1.9 * INSN_COST);
11512 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11513
11514 ins_encode %{
11515 __ bicw(as_Register($dst$$reg),
11516 as_Register($src1$$reg),
11517 as_Register($src2$$reg),
11518 Assembler::LSL,
11519 $src3$$constant & 0x1f);
11520 %}
11521
11522 ins_pipe(ialu_reg_reg_shift);
11523 %}
11524
11525 // This pattern is automatically generated from aarch64_ad.m4
11526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11527 // val & (-1 ^ (val << shift)) ==> bic
11528 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11529 iRegL src1, iRegL src2,
11530 immI src3, immL_M1 src4) %{
11531 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11532 ins_cost(1.9 * INSN_COST);
11533 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11534
11535 ins_encode %{
11536 __ bic(as_Register($dst$$reg),
11537 as_Register($src1$$reg),
11538 as_Register($src2$$reg),
11539 Assembler::LSL,
11540 $src3$$constant & 0x3f);
11541 %}
11542
11543 ins_pipe(ialu_reg_reg_shift);
11544 %}
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11549 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11550 iRegIorL2I src1, iRegIorL2I src2,
11551 immI src3, immI_M1 src4) %{
11552 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11553 ins_cost(1.9 * INSN_COST);
11554 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11555
11556 ins_encode %{
11557 __ eonw(as_Register($dst$$reg),
11558 as_Register($src1$$reg),
11559 as_Register($src2$$reg),
11560 Assembler::LSR,
11561 $src3$$constant & 0x1f);
11562 %}
11563
11564 ins_pipe(ialu_reg_reg_shift);
11565 %}
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 // val ^ (-1 ^ (val >>> shift)) ==> eon
11570 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11571 iRegL src1, iRegL src2,
11572 immI src3, immL_M1 src4) %{
11573 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11574 ins_cost(1.9 * INSN_COST);
11575 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11576
11577 ins_encode %{
11578 __ eon(as_Register($dst$$reg),
11579 as_Register($src1$$reg),
11580 as_Register($src2$$reg),
11581 Assembler::LSR,
11582 $src3$$constant & 0x3f);
11583 %}
11584
11585 ins_pipe(ialu_reg_reg_shift);
11586 %}
11587
11588 // This pattern is automatically generated from aarch64_ad.m4
11589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11590 // val ^ (-1 ^ (val >> shift)) ==> eonw
11591 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11592 iRegIorL2I src1, iRegIorL2I src2,
11593 immI src3, immI_M1 src4) %{
11594 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11595 ins_cost(1.9 * INSN_COST);
11596 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11597
11598 ins_encode %{
11599 __ eonw(as_Register($dst$$reg),
11600 as_Register($src1$$reg),
11601 as_Register($src2$$reg),
11602 Assembler::ASR,
11603 $src3$$constant & 0x1f);
11604 %}
11605
11606 ins_pipe(ialu_reg_reg_shift);
11607 %}
11608
11609 // This pattern is automatically generated from aarch64_ad.m4
11610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11611 // val ^ (-1 ^ (val >> shift)) ==> eon
11612 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11613 iRegL src1, iRegL src2,
11614 immI src3, immL_M1 src4) %{
11615 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11616 ins_cost(1.9 * INSN_COST);
11617 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11618
11619 ins_encode %{
11620 __ eon(as_Register($dst$$reg),
11621 as_Register($src1$$reg),
11622 as_Register($src2$$reg),
11623 Assembler::ASR,
11624 $src3$$constant & 0x3f);
11625 %}
11626
11627 ins_pipe(ialu_reg_reg_shift);
11628 %}
11629
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 // val ^ (-1 ^ (val ror shift)) ==> eonw
11633 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11634 iRegIorL2I src1, iRegIorL2I src2,
11635 immI src3, immI_M1 src4) %{
11636 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11637 ins_cost(1.9 * INSN_COST);
11638 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11639
11640 ins_encode %{
11641 __ eonw(as_Register($dst$$reg),
11642 as_Register($src1$$reg),
11643 as_Register($src2$$reg),
11644 Assembler::ROR,
11645 $src3$$constant & 0x1f);
11646 %}
11647
11648 ins_pipe(ialu_reg_reg_shift);
11649 %}
11650
11651 // This pattern is automatically generated from aarch64_ad.m4
11652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11653 // val ^ (-1 ^ (val ror shift)) ==> eon
11654 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11655 iRegL src1, iRegL src2,
11656 immI src3, immL_M1 src4) %{
11657 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11658 ins_cost(1.9 * INSN_COST);
11659 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11660
11661 ins_encode %{
11662 __ eon(as_Register($dst$$reg),
11663 as_Register($src1$$reg),
11664 as_Register($src2$$reg),
11665 Assembler::ROR,
11666 $src3$$constant & 0x3f);
11667 %}
11668
11669 ins_pipe(ialu_reg_reg_shift);
11670 %}
11671
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674 // val ^ (-1 ^ (val << shift)) ==> eonw
11675 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11676 iRegIorL2I src1, iRegIorL2I src2,
11677 immI src3, immI_M1 src4) %{
11678 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11679 ins_cost(1.9 * INSN_COST);
11680 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11681
11682 ins_encode %{
11683 __ eonw(as_Register($dst$$reg),
11684 as_Register($src1$$reg),
11685 as_Register($src2$$reg),
11686 Assembler::LSL,
11687 $src3$$constant & 0x1f);
11688 %}
11689
11690 ins_pipe(ialu_reg_reg_shift);
11691 %}
11692
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695 // val ^ (-1 ^ (val << shift)) ==> eon
11696 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11697 iRegL src1, iRegL src2,
11698 immI src3, immL_M1 src4) %{
11699 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11700 ins_cost(1.9 * INSN_COST);
11701 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11702
11703 ins_encode %{
11704 __ eon(as_Register($dst$$reg),
11705 as_Register($src1$$reg),
11706 as_Register($src2$$reg),
11707 Assembler::LSL,
11708 $src3$$constant & 0x3f);
11709 %}
11710
11711 ins_pipe(ialu_reg_reg_shift);
11712 %}
11713
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716 // val | (-1 ^ (val >>> shift)) ==> ornw
11717 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11718 iRegIorL2I src1, iRegIorL2I src2,
11719 immI src3, immI_M1 src4) %{
11720 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11721 ins_cost(1.9 * INSN_COST);
11722 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11723
11724 ins_encode %{
11725 __ ornw(as_Register($dst$$reg),
11726 as_Register($src1$$reg),
11727 as_Register($src2$$reg),
11728 Assembler::LSR,
11729 $src3$$constant & 0x1f);
11730 %}
11731
11732 ins_pipe(ialu_reg_reg_shift);
11733 %}
11734
11735 // This pattern is automatically generated from aarch64_ad.m4
11736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11737 // val | (-1 ^ (val >>> shift)) ==> orn
11738 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11739 iRegL src1, iRegL src2,
11740 immI src3, immL_M1 src4) %{
11741 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11742 ins_cost(1.9 * INSN_COST);
11743 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11744
11745 ins_encode %{
11746 __ orn(as_Register($dst$$reg),
11747 as_Register($src1$$reg),
11748 as_Register($src2$$reg),
11749 Assembler::LSR,
11750 $src3$$constant & 0x3f);
11751 %}
11752
11753 ins_pipe(ialu_reg_reg_shift);
11754 %}
11755
11756 // This pattern is automatically generated from aarch64_ad.m4
11757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11758 // val | (-1 ^ (val >> shift)) ==> ornw
11759 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11760 iRegIorL2I src1, iRegIorL2I src2,
11761 immI src3, immI_M1 src4) %{
11762 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11763 ins_cost(1.9 * INSN_COST);
11764 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11765
11766 ins_encode %{
11767 __ ornw(as_Register($dst$$reg),
11768 as_Register($src1$$reg),
11769 as_Register($src2$$reg),
11770 Assembler::ASR,
11771 $src3$$constant & 0x1f);
11772 %}
11773
11774 ins_pipe(ialu_reg_reg_shift);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 // val | (-1 ^ (val >> shift)) ==> orn
11780 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11781 iRegL src1, iRegL src2,
11782 immI src3, immL_M1 src4) %{
11783 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11784 ins_cost(1.9 * INSN_COST);
11785 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11786
11787 ins_encode %{
11788 __ orn(as_Register($dst$$reg),
11789 as_Register($src1$$reg),
11790 as_Register($src2$$reg),
11791 Assembler::ASR,
11792 $src3$$constant & 0x3f);
11793 %}
11794
11795 ins_pipe(ialu_reg_reg_shift);
11796 %}
11797
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 // val | (-1 ^ (val ror shift)) ==> ornw
11801 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11802 iRegIorL2I src1, iRegIorL2I src2,
11803 immI src3, immI_M1 src4) %{
11804 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11805 ins_cost(1.9 * INSN_COST);
11806 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11807
11808 ins_encode %{
11809 __ ornw(as_Register($dst$$reg),
11810 as_Register($src1$$reg),
11811 as_Register($src2$$reg),
11812 Assembler::ROR,
11813 $src3$$constant & 0x1f);
11814 %}
11815
11816 ins_pipe(ialu_reg_reg_shift);
11817 %}
11818
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 // val | (-1 ^ (val ror shift)) ==> orn
11822 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11823 iRegL src1, iRegL src2,
11824 immI src3, immL_M1 src4) %{
11825 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11826 ins_cost(1.9 * INSN_COST);
11827 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11828
11829 ins_encode %{
11830 __ orn(as_Register($dst$$reg),
11831 as_Register($src1$$reg),
11832 as_Register($src2$$reg),
11833 Assembler::ROR,
11834 $src3$$constant & 0x3f);
11835 %}
11836
11837 ins_pipe(ialu_reg_reg_shift);
11838 %}
11839
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 // val | (-1 ^ (val << shift)) ==> ornw
11843 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11844 iRegIorL2I src1, iRegIorL2I src2,
11845 immI src3, immI_M1 src4) %{
11846 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11847 ins_cost(1.9 * INSN_COST);
11848 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11849
11850 ins_encode %{
11851 __ ornw(as_Register($dst$$reg),
11852 as_Register($src1$$reg),
11853 as_Register($src2$$reg),
11854 Assembler::LSL,
11855 $src3$$constant & 0x1f);
11856 %}
11857
11858 ins_pipe(ialu_reg_reg_shift);
11859 %}
11860
11861 // This pattern is automatically generated from aarch64_ad.m4
11862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11863 // val | (-1 ^ (val << shift)) ==> orn
11864 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11865 iRegL src1, iRegL src2,
11866 immI src3, immL_M1 src4) %{
11867 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11868 ins_cost(1.9 * INSN_COST);
11869 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11870
11871 ins_encode %{
11872 __ orn(as_Register($dst$$reg),
11873 as_Register($src1$$reg),
11874 as_Register($src2$$reg),
11875 Assembler::LSL,
11876 $src3$$constant & 0x3f);
11877 %}
11878
11879 ins_pipe(ialu_reg_reg_shift);
11880 %}
11881
11882 // This pattern is automatically generated from aarch64_ad.m4
11883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11884 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11885 iRegIorL2I src1, iRegIorL2I src2,
11886 immI src3) %{
11887 match(Set dst (AndI src1 (URShiftI src2 src3)));
11888
11889 ins_cost(1.9 * INSN_COST);
11890 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11891
11892 ins_encode %{
11893 __ andw(as_Register($dst$$reg),
11894 as_Register($src1$$reg),
11895 as_Register($src2$$reg),
11896 Assembler::LSR,
11897 $src3$$constant & 0x1f);
11898 %}
11899
11900 ins_pipe(ialu_reg_reg_shift);
11901 %}
11902
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11906 iRegL src1, iRegL src2,
11907 immI src3) %{
11908 match(Set dst (AndL src1 (URShiftL src2 src3)));
11909
11910 ins_cost(1.9 * INSN_COST);
11911 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11912
11913 ins_encode %{
11914 __ andr(as_Register($dst$$reg),
11915 as_Register($src1$$reg),
11916 as_Register($src2$$reg),
11917 Assembler::LSR,
11918 $src3$$constant & 0x3f);
11919 %}
11920
11921 ins_pipe(ialu_reg_reg_shift);
11922 %}
11923
11924 // This pattern is automatically generated from aarch64_ad.m4
11925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11926 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11927 iRegIorL2I src1, iRegIorL2I src2,
11928 immI src3) %{
11929 match(Set dst (AndI src1 (RShiftI src2 src3)));
11930
11931 ins_cost(1.9 * INSN_COST);
11932 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11933
11934 ins_encode %{
11935 __ andw(as_Register($dst$$reg),
11936 as_Register($src1$$reg),
11937 as_Register($src2$$reg),
11938 Assembler::ASR,
11939 $src3$$constant & 0x1f);
11940 %}
11941
11942 ins_pipe(ialu_reg_reg_shift);
11943 %}
11944
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11948 iRegL src1, iRegL src2,
11949 immI src3) %{
11950 match(Set dst (AndL src1 (RShiftL src2 src3)));
11951
11952 ins_cost(1.9 * INSN_COST);
11953 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11954
11955 ins_encode %{
11956 __ andr(as_Register($dst$$reg),
11957 as_Register($src1$$reg),
11958 as_Register($src2$$reg),
11959 Assembler::ASR,
11960 $src3$$constant & 0x3f);
11961 %}
11962
11963 ins_pipe(ialu_reg_reg_shift);
11964 %}
11965
11966 // This pattern is automatically generated from aarch64_ad.m4
11967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11968 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11969 iRegIorL2I src1, iRegIorL2I src2,
11970 immI src3) %{
11971 match(Set dst (AndI src1 (LShiftI src2 src3)));
11972
11973 ins_cost(1.9 * INSN_COST);
11974 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11975
11976 ins_encode %{
11977 __ andw(as_Register($dst$$reg),
11978 as_Register($src1$$reg),
11979 as_Register($src2$$reg),
11980 Assembler::LSL,
11981 $src3$$constant & 0x1f);
11982 %}
11983
11984 ins_pipe(ialu_reg_reg_shift);
11985 %}
11986
11987 // This pattern is automatically generated from aarch64_ad.m4
11988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11989 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11990 iRegL src1, iRegL src2,
11991 immI src3) %{
11992 match(Set dst (AndL src1 (LShiftL src2 src3)));
11993
11994 ins_cost(1.9 * INSN_COST);
11995 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11996
11997 ins_encode %{
11998 __ andr(as_Register($dst$$reg),
11999 as_Register($src1$$reg),
12000 as_Register($src2$$reg),
12001 Assembler::LSL,
12002 $src3$$constant & 0x3f);
12003 %}
12004
12005 ins_pipe(ialu_reg_reg_shift);
12006 %}
12007
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12011 iRegIorL2I src1, iRegIorL2I src2,
12012 immI src3) %{
12013 match(Set dst (AndI src1 (RotateRight src2 src3)));
12014
12015 ins_cost(1.9 * INSN_COST);
12016 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12017
12018 ins_encode %{
12019 __ andw(as_Register($dst$$reg),
12020 as_Register($src1$$reg),
12021 as_Register($src2$$reg),
12022 Assembler::ROR,
12023 $src3$$constant & 0x1f);
12024 %}
12025
12026 ins_pipe(ialu_reg_reg_shift);
12027 %}
12028
12029 // This pattern is automatically generated from aarch64_ad.m4
12030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12031 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12032 iRegL src1, iRegL src2,
12033 immI src3) %{
12034 match(Set dst (AndL src1 (RotateRight src2 src3)));
12035
12036 ins_cost(1.9 * INSN_COST);
12037 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12038
12039 ins_encode %{
12040 __ andr(as_Register($dst$$reg),
12041 as_Register($src1$$reg),
12042 as_Register($src2$$reg),
12043 Assembler::ROR,
12044 $src3$$constant & 0x3f);
12045 %}
12046
12047 ins_pipe(ialu_reg_reg_shift);
12048 %}
12049
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12053 iRegIorL2I src1, iRegIorL2I src2,
12054 immI src3) %{
12055 match(Set dst (XorI src1 (URShiftI src2 src3)));
12056
12057 ins_cost(1.9 * INSN_COST);
12058 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12059
12060 ins_encode %{
12061 __ eorw(as_Register($dst$$reg),
12062 as_Register($src1$$reg),
12063 as_Register($src2$$reg),
12064 Assembler::LSR,
12065 $src3$$constant & 0x1f);
12066 %}
12067
12068 ins_pipe(ialu_reg_reg_shift);
12069 %}
12070
12071 // This pattern is automatically generated from aarch64_ad.m4
12072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12073 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12074 iRegL src1, iRegL src2,
12075 immI src3) %{
12076 match(Set dst (XorL src1 (URShiftL src2 src3)));
12077
12078 ins_cost(1.9 * INSN_COST);
12079 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12080
12081 ins_encode %{
12082 __ eor(as_Register($dst$$reg),
12083 as_Register($src1$$reg),
12084 as_Register($src2$$reg),
12085 Assembler::LSR,
12086 $src3$$constant & 0x3f);
12087 %}
12088
12089 ins_pipe(ialu_reg_reg_shift);
12090 %}
12091
12092 // This pattern is automatically generated from aarch64_ad.m4
12093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12094 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12095 iRegIorL2I src1, iRegIorL2I src2,
12096 immI src3) %{
12097 match(Set dst (XorI src1 (RShiftI src2 src3)));
12098
12099 ins_cost(1.9 * INSN_COST);
12100 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12101
12102 ins_encode %{
12103 __ eorw(as_Register($dst$$reg),
12104 as_Register($src1$$reg),
12105 as_Register($src2$$reg),
12106 Assembler::ASR,
12107 $src3$$constant & 0x1f);
12108 %}
12109
12110 ins_pipe(ialu_reg_reg_shift);
12111 %}
12112
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12116 iRegL src1, iRegL src2,
12117 immI src3) %{
12118 match(Set dst (XorL src1 (RShiftL src2 src3)));
12119
12120 ins_cost(1.9 * INSN_COST);
12121 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12122
12123 ins_encode %{
12124 __ eor(as_Register($dst$$reg),
12125 as_Register($src1$$reg),
12126 as_Register($src2$$reg),
12127 Assembler::ASR,
12128 $src3$$constant & 0x3f);
12129 %}
12130
12131 ins_pipe(ialu_reg_reg_shift);
12132 %}
12133
12134 // This pattern is automatically generated from aarch64_ad.m4
12135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12136 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12137 iRegIorL2I src1, iRegIorL2I src2,
12138 immI src3) %{
12139 match(Set dst (XorI src1 (LShiftI src2 src3)));
12140
12141 ins_cost(1.9 * INSN_COST);
12142 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12143
12144 ins_encode %{
12145 __ eorw(as_Register($dst$$reg),
12146 as_Register($src1$$reg),
12147 as_Register($src2$$reg),
12148 Assembler::LSL,
12149 $src3$$constant & 0x1f);
12150 %}
12151
12152 ins_pipe(ialu_reg_reg_shift);
12153 %}
12154
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12158 iRegL src1, iRegL src2,
12159 immI src3) %{
12160 match(Set dst (XorL src1 (LShiftL src2 src3)));
12161
12162 ins_cost(1.9 * INSN_COST);
12163 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12164
12165 ins_encode %{
12166 __ eor(as_Register($dst$$reg),
12167 as_Register($src1$$reg),
12168 as_Register($src2$$reg),
12169 Assembler::LSL,
12170 $src3$$constant & 0x3f);
12171 %}
12172
12173 ins_pipe(ialu_reg_reg_shift);
12174 %}
12175
12176 // This pattern is automatically generated from aarch64_ad.m4
12177 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12178 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12179 iRegIorL2I src1, iRegIorL2I src2,
12180 immI src3) %{
12181 match(Set dst (XorI src1 (RotateRight src2 src3)));
12182
12183 ins_cost(1.9 * INSN_COST);
12184 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12185
12186 ins_encode %{
12187 __ eorw(as_Register($dst$$reg),
12188 as_Register($src1$$reg),
12189 as_Register($src2$$reg),
12190 Assembler::ROR,
12191 $src3$$constant & 0x1f);
12192 %}
12193
12194 ins_pipe(ialu_reg_reg_shift);
12195 %}
12196
12197 // This pattern is automatically generated from aarch64_ad.m4
12198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12199 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12200 iRegL src1, iRegL src2,
12201 immI src3) %{
12202 match(Set dst (XorL src1 (RotateRight src2 src3)));
12203
12204 ins_cost(1.9 * INSN_COST);
12205 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12206
12207 ins_encode %{
12208 __ eor(as_Register($dst$$reg),
12209 as_Register($src1$$reg),
12210 as_Register($src2$$reg),
12211 Assembler::ROR,
12212 $src3$$constant & 0x3f);
12213 %}
12214
12215 ins_pipe(ialu_reg_reg_shift);
12216 %}
12217
12218 // This pattern is automatically generated from aarch64_ad.m4
12219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12220 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12221 iRegIorL2I src1, iRegIorL2I src2,
12222 immI src3) %{
12223 match(Set dst (OrI src1 (URShiftI src2 src3)));
12224
12225 ins_cost(1.9 * INSN_COST);
12226 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12227
12228 ins_encode %{
12229 __ orrw(as_Register($dst$$reg),
12230 as_Register($src1$$reg),
12231 as_Register($src2$$reg),
12232 Assembler::LSR,
12233 $src3$$constant & 0x1f);
12234 %}
12235
12236 ins_pipe(ialu_reg_reg_shift);
12237 %}
12238
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12242 iRegL src1, iRegL src2,
12243 immI src3) %{
12244 match(Set dst (OrL src1 (URShiftL src2 src3)));
12245
12246 ins_cost(1.9 * INSN_COST);
12247 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12248
12249 ins_encode %{
12250 __ orr(as_Register($dst$$reg),
12251 as_Register($src1$$reg),
12252 as_Register($src2$$reg),
12253 Assembler::LSR,
12254 $src3$$constant & 0x3f);
12255 %}
12256
12257 ins_pipe(ialu_reg_reg_shift);
12258 %}
12259
12260 // This pattern is automatically generated from aarch64_ad.m4
12261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12262 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12263 iRegIorL2I src1, iRegIorL2I src2,
12264 immI src3) %{
12265 match(Set dst (OrI src1 (RShiftI src2 src3)));
12266
12267 ins_cost(1.9 * INSN_COST);
12268 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12269
12270 ins_encode %{
12271 __ orrw(as_Register($dst$$reg),
12272 as_Register($src1$$reg),
12273 as_Register($src2$$reg),
12274 Assembler::ASR,
12275 $src3$$constant & 0x1f);
12276 %}
12277
12278 ins_pipe(ialu_reg_reg_shift);
12279 %}
12280
12281 // This pattern is automatically generated from aarch64_ad.m4
12282 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12283 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12284 iRegL src1, iRegL src2,
12285 immI src3) %{
12286 match(Set dst (OrL src1 (RShiftL src2 src3)));
12287
12288 ins_cost(1.9 * INSN_COST);
12289 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12290
12291 ins_encode %{
12292 __ orr(as_Register($dst$$reg),
12293 as_Register($src1$$reg),
12294 as_Register($src2$$reg),
12295 Assembler::ASR,
12296 $src3$$constant & 0x3f);
12297 %}
12298
12299 ins_pipe(ialu_reg_reg_shift);
12300 %}
12301
12302 // This pattern is automatically generated from aarch64_ad.m4
12303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12304 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12305 iRegIorL2I src1, iRegIorL2I src2,
12306 immI src3) %{
12307 match(Set dst (OrI src1 (LShiftI src2 src3)));
12308
12309 ins_cost(1.9 * INSN_COST);
12310 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12311
12312 ins_encode %{
12313 __ orrw(as_Register($dst$$reg),
12314 as_Register($src1$$reg),
12315 as_Register($src2$$reg),
12316 Assembler::LSL,
12317 $src3$$constant & 0x1f);
12318 %}
12319
12320 ins_pipe(ialu_reg_reg_shift);
12321 %}
12322
12323 // This pattern is automatically generated from aarch64_ad.m4
12324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12325 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12326 iRegL src1, iRegL src2,
12327 immI src3) %{
12328 match(Set dst (OrL src1 (LShiftL src2 src3)));
12329
12330 ins_cost(1.9 * INSN_COST);
12331 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12332
12333 ins_encode %{
12334 __ orr(as_Register($dst$$reg),
12335 as_Register($src1$$reg),
12336 as_Register($src2$$reg),
12337 Assembler::LSL,
12338 $src3$$constant & 0x3f);
12339 %}
12340
12341 ins_pipe(ialu_reg_reg_shift);
12342 %}
12343
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12347 iRegIorL2I src1, iRegIorL2I src2,
12348 immI src3) %{
12349 match(Set dst (OrI src1 (RotateRight src2 src3)));
12350
12351 ins_cost(1.9 * INSN_COST);
12352 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12353
12354 ins_encode %{
12355 __ orrw(as_Register($dst$$reg),
12356 as_Register($src1$$reg),
12357 as_Register($src2$$reg),
12358 Assembler::ROR,
12359 $src3$$constant & 0x1f);
12360 %}
12361
12362 ins_pipe(ialu_reg_reg_shift);
12363 %}
12364
12365 // This pattern is automatically generated from aarch64_ad.m4
12366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12367 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12368 iRegL src1, iRegL src2,
12369 immI src3) %{
12370 match(Set dst (OrL src1 (RotateRight src2 src3)));
12371
12372 ins_cost(1.9 * INSN_COST);
12373 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12374
12375 ins_encode %{
12376 __ orr(as_Register($dst$$reg),
12377 as_Register($src1$$reg),
12378 as_Register($src2$$reg),
12379 Assembler::ROR,
12380 $src3$$constant & 0x3f);
12381 %}
12382
12383 ins_pipe(ialu_reg_reg_shift);
12384 %}
12385
12386 // This pattern is automatically generated from aarch64_ad.m4
12387 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12388 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12389 iRegIorL2I src1, iRegIorL2I src2,
12390 immI src3) %{
12391 match(Set dst (AddI src1 (URShiftI src2 src3)));
12392
12393 ins_cost(1.9 * INSN_COST);
12394 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12395
12396 ins_encode %{
12397 __ addw(as_Register($dst$$reg),
12398 as_Register($src1$$reg),
12399 as_Register($src2$$reg),
12400 Assembler::LSR,
12401 $src3$$constant & 0x1f);
12402 %}
12403
12404 ins_pipe(ialu_reg_reg_shift);
12405 %}
12406
12407 // This pattern is automatically generated from aarch64_ad.m4
12408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12409 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12410 iRegL src1, iRegL src2,
12411 immI src3) %{
12412 match(Set dst (AddL src1 (URShiftL src2 src3)));
12413
12414 ins_cost(1.9 * INSN_COST);
12415 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12416
12417 ins_encode %{
12418 __ add(as_Register($dst$$reg),
12419 as_Register($src1$$reg),
12420 as_Register($src2$$reg),
12421 Assembler::LSR,
12422 $src3$$constant & 0x3f);
12423 %}
12424
12425 ins_pipe(ialu_reg_reg_shift);
12426 %}
12427
12428 // This pattern is automatically generated from aarch64_ad.m4
12429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12430 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12431 iRegIorL2I src1, iRegIorL2I src2,
12432 immI src3) %{
12433 match(Set dst (AddI src1 (RShiftI src2 src3)));
12434
12435 ins_cost(1.9 * INSN_COST);
12436 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12437
12438 ins_encode %{
12439 __ addw(as_Register($dst$$reg),
12440 as_Register($src1$$reg),
12441 as_Register($src2$$reg),
12442 Assembler::ASR,
12443 $src3$$constant & 0x1f);
12444 %}
12445
12446 ins_pipe(ialu_reg_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12452 iRegL src1, iRegL src2,
12453 immI src3) %{
12454 match(Set dst (AddL src1 (RShiftL src2 src3)));
12455
12456 ins_cost(1.9 * INSN_COST);
12457 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12458
12459 ins_encode %{
12460 __ add(as_Register($dst$$reg),
12461 as_Register($src1$$reg),
12462 as_Register($src2$$reg),
12463 Assembler::ASR,
12464 $src3$$constant & 0x3f);
12465 %}
12466
12467 ins_pipe(ialu_reg_reg_shift);
12468 %}
12469
12470 // This pattern is automatically generated from aarch64_ad.m4
12471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12472 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12473 iRegIorL2I src1, iRegIorL2I src2,
12474 immI src3) %{
12475 match(Set dst (AddI src1 (LShiftI src2 src3)));
12476
12477 ins_cost(1.9 * INSN_COST);
12478 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12479
12480 ins_encode %{
12481 __ addw(as_Register($dst$$reg),
12482 as_Register($src1$$reg),
12483 as_Register($src2$$reg),
12484 Assembler::LSL,
12485 $src3$$constant & 0x1f);
12486 %}
12487
12488 ins_pipe(ialu_reg_reg_shift);
12489 %}
12490
12491 // This pattern is automatically generated from aarch64_ad.m4
12492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12493 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12494 iRegL src1, iRegL src2,
12495 immI src3) %{
12496 match(Set dst (AddL src1 (LShiftL src2 src3)));
12497
12498 ins_cost(1.9 * INSN_COST);
12499 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12500
12501 ins_encode %{
12502 __ add(as_Register($dst$$reg),
12503 as_Register($src1$$reg),
12504 as_Register($src2$$reg),
12505 Assembler::LSL,
12506 $src3$$constant & 0x3f);
12507 %}
12508
12509 ins_pipe(ialu_reg_reg_shift);
12510 %}
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12515 iRegIorL2I src1, iRegIorL2I src2,
12516 immI src3) %{
12517 match(Set dst (SubI src1 (URShiftI src2 src3)));
12518
12519 ins_cost(1.9 * INSN_COST);
12520 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12521
12522 ins_encode %{
12523 __ subw(as_Register($dst$$reg),
12524 as_Register($src1$$reg),
12525 as_Register($src2$$reg),
12526 Assembler::LSR,
12527 $src3$$constant & 0x1f);
12528 %}
12529
12530 ins_pipe(ialu_reg_reg_shift);
12531 %}
12532
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12536 iRegL src1, iRegL src2,
12537 immI src3) %{
12538 match(Set dst (SubL src1 (URShiftL src2 src3)));
12539
12540 ins_cost(1.9 * INSN_COST);
12541 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12542
12543 ins_encode %{
12544 __ sub(as_Register($dst$$reg),
12545 as_Register($src1$$reg),
12546 as_Register($src2$$reg),
12547 Assembler::LSR,
12548 $src3$$constant & 0x3f);
12549 %}
12550
12551 ins_pipe(ialu_reg_reg_shift);
12552 %}
12553
12554 // This pattern is automatically generated from aarch64_ad.m4
12555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12556 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12557 iRegIorL2I src1, iRegIorL2I src2,
12558 immI src3) %{
12559 match(Set dst (SubI src1 (RShiftI src2 src3)));
12560
12561 ins_cost(1.9 * INSN_COST);
12562 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12563
12564 ins_encode %{
12565 __ subw(as_Register($dst$$reg),
12566 as_Register($src1$$reg),
12567 as_Register($src2$$reg),
12568 Assembler::ASR,
12569 $src3$$constant & 0x1f);
12570 %}
12571
12572 ins_pipe(ialu_reg_reg_shift);
12573 %}
12574
12575 // This pattern is automatically generated from aarch64_ad.m4
12576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12577 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12578 iRegL src1, iRegL src2,
12579 immI src3) %{
12580 match(Set dst (SubL src1 (RShiftL src2 src3)));
12581
12582 ins_cost(1.9 * INSN_COST);
12583 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12584
12585 ins_encode %{
12586 __ sub(as_Register($dst$$reg),
12587 as_Register($src1$$reg),
12588 as_Register($src2$$reg),
12589 Assembler::ASR,
12590 $src3$$constant & 0x3f);
12591 %}
12592
12593 ins_pipe(ialu_reg_reg_shift);
12594 %}
12595
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12599 iRegIorL2I src1, iRegIorL2I src2,
12600 immI src3) %{
12601 match(Set dst (SubI src1 (LShiftI src2 src3)));
12602
12603 ins_cost(1.9 * INSN_COST);
12604 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12605
12606 ins_encode %{
12607 __ subw(as_Register($dst$$reg),
12608 as_Register($src1$$reg),
12609 as_Register($src2$$reg),
12610 Assembler::LSL,
12611 $src3$$constant & 0x1f);
12612 %}
12613
12614 ins_pipe(ialu_reg_reg_shift);
12615 %}
12616
12617 // This pattern is automatically generated from aarch64_ad.m4
12618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12619 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12620 iRegL src1, iRegL src2,
12621 immI src3) %{
12622 match(Set dst (SubL src1 (LShiftL src2 src3)));
12623
12624 ins_cost(1.9 * INSN_COST);
12625 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12626
12627 ins_encode %{
12628 __ sub(as_Register($dst$$reg),
12629 as_Register($src1$$reg),
12630 as_Register($src2$$reg),
12631 Assembler::LSL,
12632 $src3$$constant & 0x3f);
12633 %}
12634
12635 ins_pipe(ialu_reg_reg_shift);
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640
12641 // Shift Left followed by Shift Right.
12642 // This idiom is used by the compiler for the i2b bytecode etc.
12643 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12644 %{
12645 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12646 ins_cost(INSN_COST * 2);
12647 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12648 ins_encode %{
12649 int lshift = $lshift_count$$constant & 63;
12650 int rshift = $rshift_count$$constant & 63;
12651 int s = 63 - lshift;
12652 int r = (rshift - lshift) & 63;
12653 __ sbfm(as_Register($dst$$reg),
12654 as_Register($src$$reg),
12655 r, s);
12656 %}
12657
12658 ins_pipe(ialu_reg_shift);
12659 %}
12660
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663
12664 // Shift Left followed by Shift Right.
12665 // This idiom is used by the compiler for the i2b bytecode etc.
12666 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12667 %{
12668 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12669 ins_cost(INSN_COST * 2);
12670 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12671 ins_encode %{
12672 int lshift = $lshift_count$$constant & 31;
12673 int rshift = $rshift_count$$constant & 31;
12674 int s = 31 - lshift;
12675 int r = (rshift - lshift) & 31;
12676 __ sbfmw(as_Register($dst$$reg),
12677 as_Register($src$$reg),
12678 r, s);
12679 %}
12680
12681 ins_pipe(ialu_reg_shift);
12682 %}
12683
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686
12687 // Shift Left followed by Shift Right.
12688 // This idiom is used by the compiler for the i2b bytecode etc.
12689 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12690 %{
12691 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12692 ins_cost(INSN_COST * 2);
12693 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12694 ins_encode %{
12695 int lshift = $lshift_count$$constant & 63;
12696 int rshift = $rshift_count$$constant & 63;
12697 int s = 63 - lshift;
12698 int r = (rshift - lshift) & 63;
12699 __ ubfm(as_Register($dst$$reg),
12700 as_Register($src$$reg),
12701 r, s);
12702 %}
12703
12704 ins_pipe(ialu_reg_shift);
12705 %}
12706
12707 // This pattern is automatically generated from aarch64_ad.m4
12708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12709
12710 // Shift Left followed by Shift Right.
12711 // This idiom is used by the compiler for the i2b bytecode etc.
12712 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12713 %{
12714 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12715 ins_cost(INSN_COST * 2);
12716 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12717 ins_encode %{
12718 int lshift = $lshift_count$$constant & 31;
12719 int rshift = $rshift_count$$constant & 31;
12720 int s = 31 - lshift;
12721 int r = (rshift - lshift) & 31;
12722 __ ubfmw(as_Register($dst$$reg),
12723 as_Register($src$$reg),
12724 r, s);
12725 %}
12726
12727 ins_pipe(ialu_reg_shift);
12728 %}
12729
12730 // Bitfield extract with shift & mask
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12735 %{
12736 match(Set dst (AndI (URShiftI src rshift) mask));
12737 // Make sure we are not going to exceed what ubfxw can do.
12738 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12739
12740 ins_cost(INSN_COST);
12741 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12742 ins_encode %{
12743 int rshift = $rshift$$constant & 31;
12744 intptr_t mask = $mask$$constant;
12745 int width = exact_log2(mask+1);
12746 __ ubfxw(as_Register($dst$$reg),
12747 as_Register($src$$reg), rshift, width);
12748 %}
12749 ins_pipe(ialu_reg_shift);
12750 %}
12751
12752 // This pattern is automatically generated from aarch64_ad.m4
12753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12754 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12755 %{
12756 match(Set dst (AndL (URShiftL src rshift) mask));
12757 // Make sure we are not going to exceed what ubfx can do.
12758 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12759
12760 ins_cost(INSN_COST);
12761 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12762 ins_encode %{
12763 int rshift = $rshift$$constant & 63;
12764 intptr_t mask = $mask$$constant;
12765 int width = exact_log2_long(mask+1);
12766 __ ubfx(as_Register($dst$$reg),
12767 as_Register($src$$reg), rshift, width);
12768 %}
12769 ins_pipe(ialu_reg_shift);
12770 %}
12771
12772
12773 // This pattern is automatically generated from aarch64_ad.m4
12774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12775
12776 // We can use ubfx when extending an And with a mask when we know mask
12777 // is positive. We know that because immI_bitmask guarantees it.
12778 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12779 %{
12780 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12781 // Make sure we are not going to exceed what ubfxw can do.
12782 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12783
12784 ins_cost(INSN_COST * 2);
12785 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12786 ins_encode %{
12787 int rshift = $rshift$$constant & 31;
12788 intptr_t mask = $mask$$constant;
12789 int width = exact_log2(mask+1);
12790 __ ubfx(as_Register($dst$$reg),
12791 as_Register($src$$reg), rshift, width);
12792 %}
12793 ins_pipe(ialu_reg_shift);
12794 %}
12795
12796
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799
12800 // We can use ubfiz when masking by a positive number and then left shifting the result.
12801 // We know that the mask is positive because immI_bitmask guarantees it.
12802 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12803 %{
12804 match(Set dst (LShiftI (AndI src mask) lshift));
12805 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12806
12807 ins_cost(INSN_COST);
12808 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12809 ins_encode %{
12810 int lshift = $lshift$$constant & 31;
12811 intptr_t mask = $mask$$constant;
12812 int width = exact_log2(mask+1);
12813 __ ubfizw(as_Register($dst$$reg),
12814 as_Register($src$$reg), lshift, width);
12815 %}
12816 ins_pipe(ialu_reg_shift);
12817 %}
12818
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821
12822 // We can use ubfiz when masking by a positive number and then left shifting the result.
12823 // We know that the mask is positive because immL_bitmask guarantees it.
12824 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12825 %{
12826 match(Set dst (LShiftL (AndL src mask) lshift));
12827 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12828
12829 ins_cost(INSN_COST);
12830 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12831 ins_encode %{
12832 int lshift = $lshift$$constant & 63;
12833 intptr_t mask = $mask$$constant;
12834 int width = exact_log2_long(mask+1);
12835 __ ubfiz(as_Register($dst$$reg),
12836 as_Register($src$$reg), lshift, width);
12837 %}
12838 ins_pipe(ialu_reg_shift);
12839 %}
12840
12841 // This pattern is automatically generated from aarch64_ad.m4
12842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12843
12844 // We can use ubfiz when masking by a positive number and then left shifting the result.
12845 // We know that the mask is positive because immI_bitmask guarantees it.
12846 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12847 %{
12848 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12849 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12850
12851 ins_cost(INSN_COST);
12852 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12853 ins_encode %{
12854 int lshift = $lshift$$constant & 31;
12855 intptr_t mask = $mask$$constant;
12856 int width = exact_log2(mask+1);
12857 __ ubfizw(as_Register($dst$$reg),
12858 as_Register($src$$reg), lshift, width);
12859 %}
12860 ins_pipe(ialu_reg_shift);
12861 %}
12862
12863 // This pattern is automatically generated from aarch64_ad.m4
12864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12865
12866 // We can use ubfiz when masking by a positive number and then left shifting the result.
12867 // We know that the mask is positive because immL_bitmask guarantees it.
12868 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12869 %{
12870 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12871 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12872
12873 ins_cost(INSN_COST);
12874 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12875 ins_encode %{
12876 int lshift = $lshift$$constant & 63;
12877 intptr_t mask = $mask$$constant;
12878 int width = exact_log2_long(mask+1);
12879 __ ubfiz(as_Register($dst$$reg),
12880 as_Register($src$$reg), lshift, width);
12881 %}
12882 ins_pipe(ialu_reg_shift);
12883 %}
12884
12885
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888
12889 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12890 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12891 %{
12892 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12893 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12894
12895 ins_cost(INSN_COST);
12896 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12897 ins_encode %{
12898 int lshift = $lshift$$constant & 63;
12899 intptr_t mask = $mask$$constant;
12900 int width = exact_log2(mask+1);
12901 __ ubfiz(as_Register($dst$$reg),
12902 as_Register($src$$reg), lshift, width);
12903 %}
12904 ins_pipe(ialu_reg_shift);
12905 %}
12906
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909
12910 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12911 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12912 %{
12913 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12914 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12915
12916 ins_cost(INSN_COST);
12917 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12918 ins_encode %{
12919 int lshift = $lshift$$constant & 31;
12920 intptr_t mask = $mask$$constant;
12921 int width = exact_log2(mask+1);
12922 __ ubfiz(as_Register($dst$$reg),
12923 as_Register($src$$reg), lshift, width);
12924 %}
12925 ins_pipe(ialu_reg_shift);
12926 %}
12927
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930
12931 // Can skip int2long conversions after AND with small bitmask
12932 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12933 %{
12934 match(Set dst (ConvI2L (AndI src msk)));
12935 ins_cost(INSN_COST);
12936 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12937 ins_encode %{
12938 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12939 %}
12940 ins_pipe(ialu_reg_shift);
12941 %}
12942
12943
12944 // Rotations
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12948 %{
12949 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12950 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12951
12952 ins_cost(INSN_COST);
12953 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12954
12955 ins_encode %{
12956 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12957 $rshift$$constant & 63);
12958 %}
12959 ins_pipe(ialu_reg_reg_extr);
12960 %}
12961
12962
12963 // This pattern is automatically generated from aarch64_ad.m4
12964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12965 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12966 %{
12967 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12968 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12969
12970 ins_cost(INSN_COST);
12971 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12972
12973 ins_encode %{
12974 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12975 $rshift$$constant & 31);
12976 %}
12977 ins_pipe(ialu_reg_reg_extr);
12978 %}
12979
12980
12981 // This pattern is automatically generated from aarch64_ad.m4
12982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12983 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12984 %{
12985 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12986 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12987
12988 ins_cost(INSN_COST);
12989 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12990
12991 ins_encode %{
12992 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12993 $rshift$$constant & 63);
12994 %}
12995 ins_pipe(ialu_reg_reg_extr);
12996 %}
12997
12998
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13002 %{
13003 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13004 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13005
13006 ins_cost(INSN_COST);
13007 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13008
13009 ins_encode %{
13010 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13011 $rshift$$constant & 31);
13012 %}
13013 ins_pipe(ialu_reg_reg_extr);
13014 %}
13015
13016
13017 // This pattern is automatically generated from aarch64_ad.m4
13018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13019 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13020 %{
13021 match(Set dst (RotateRight src shift));
13022
13023 ins_cost(INSN_COST);
13024 format %{ "ror $dst, $src, $shift" %}
13025
13026 ins_encode %{
13027 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13028 $shift$$constant & 0x1f);
13029 %}
13030 ins_pipe(ialu_reg_reg_vshift);
13031 %}
13032
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13036 %{
13037 match(Set dst (RotateRight src shift));
13038
13039 ins_cost(INSN_COST);
13040 format %{ "ror $dst, $src, $shift" %}
13041
13042 ins_encode %{
13043 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13044 $shift$$constant & 0x3f);
13045 %}
13046 ins_pipe(ialu_reg_reg_vshift);
13047 %}
13048
13049 // This pattern is automatically generated from aarch64_ad.m4
13050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13051 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13052 %{
13053 match(Set dst (RotateRight src shift));
13054
13055 ins_cost(INSN_COST);
13056 format %{ "ror $dst, $src, $shift" %}
13057
13058 ins_encode %{
13059 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13060 %}
13061 ins_pipe(ialu_reg_reg_vshift);
13062 %}
13063
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13067 %{
13068 match(Set dst (RotateRight src shift));
13069
13070 ins_cost(INSN_COST);
13071 format %{ "ror $dst, $src, $shift" %}
13072
13073 ins_encode %{
13074 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13075 %}
13076 ins_pipe(ialu_reg_reg_vshift);
13077 %}
13078
13079 // This pattern is automatically generated from aarch64_ad.m4
13080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13081 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13082 %{
13083 match(Set dst (RotateLeft src shift));
13084
13085 ins_cost(INSN_COST);
13086 format %{ "rol $dst, $src, $shift" %}
13087
13088 ins_encode %{
13089 __ subw(rscratch1, zr, as_Register($shift$$reg));
13090 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13091 %}
13092 ins_pipe(ialu_reg_reg_vshift);
13093 %}
13094
13095 // This pattern is automatically generated from aarch64_ad.m4
13096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13097 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13098 %{
13099 match(Set dst (RotateLeft src shift));
13100
13101 ins_cost(INSN_COST);
13102 format %{ "rol $dst, $src, $shift" %}
13103
13104 ins_encode %{
13105 __ subw(rscratch1, zr, as_Register($shift$$reg));
13106 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13107 %}
13108 ins_pipe(ialu_reg_reg_vshift);
13109 %}
13110
13111
13112 // Add/subtract (extended)
13113
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13117 %{
13118 match(Set dst (AddL src1 (ConvI2L src2)));
13119 ins_cost(INSN_COST);
13120 format %{ "add $dst, $src1, $src2, sxtw" %}
13121
13122 ins_encode %{
13123 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13124 as_Register($src2$$reg), ext::sxtw);
13125 %}
13126 ins_pipe(ialu_reg_reg);
13127 %}
13128
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13132 %{
13133 match(Set dst (SubL src1 (ConvI2L src2)));
13134 ins_cost(INSN_COST);
13135 format %{ "sub $dst, $src1, $src2, sxtw" %}
13136
13137 ins_encode %{
13138 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13139 as_Register($src2$$reg), ext::sxtw);
13140 %}
13141 ins_pipe(ialu_reg_reg);
13142 %}
13143
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13147 %{
13148 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13149 ins_cost(INSN_COST);
13150 format %{ "add $dst, $src1, $src2, sxth" %}
13151
13152 ins_encode %{
13153 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13154 as_Register($src2$$reg), ext::sxth);
13155 %}
13156 ins_pipe(ialu_reg_reg);
13157 %}
13158
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13162 %{
13163 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13164 ins_cost(INSN_COST);
13165 format %{ "add $dst, $src1, $src2, sxtb" %}
13166
13167 ins_encode %{
13168 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169 as_Register($src2$$reg), ext::sxtb);
13170 %}
13171 ins_pipe(ialu_reg_reg);
13172 %}
13173
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13177 %{
13178 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13179 ins_cost(INSN_COST);
13180 format %{ "add $dst, $src1, $src2, uxtb" %}
13181
13182 ins_encode %{
13183 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13184 as_Register($src2$$reg), ext::uxtb);
13185 %}
13186 ins_pipe(ialu_reg_reg);
13187 %}
13188
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13192 %{
13193 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13194 ins_cost(INSN_COST);
13195 format %{ "add $dst, $src1, $src2, sxth" %}
13196
13197 ins_encode %{
13198 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13199 as_Register($src2$$reg), ext::sxth);
13200 %}
13201 ins_pipe(ialu_reg_reg);
13202 %}
13203
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13207 %{
13208 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13209 ins_cost(INSN_COST);
13210 format %{ "add $dst, $src1, $src2, sxtw" %}
13211
13212 ins_encode %{
13213 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13214 as_Register($src2$$reg), ext::sxtw);
13215 %}
13216 ins_pipe(ialu_reg_reg);
13217 %}
13218
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13222 %{
13223 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13224 ins_cost(INSN_COST);
13225 format %{ "add $dst, $src1, $src2, sxtb" %}
13226
13227 ins_encode %{
13228 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13229 as_Register($src2$$reg), ext::sxtb);
13230 %}
13231 ins_pipe(ialu_reg_reg);
13232 %}
13233
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13237 %{
13238 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13239 ins_cost(INSN_COST);
13240 format %{ "add $dst, $src1, $src2, uxtb" %}
13241
13242 ins_encode %{
13243 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13244 as_Register($src2$$reg), ext::uxtb);
13245 %}
13246 ins_pipe(ialu_reg_reg);
13247 %}
13248
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13252 %{
13253 match(Set dst (AddI src1 (AndI src2 mask)));
13254 ins_cost(INSN_COST);
13255 format %{ "addw $dst, $src1, $src2, uxtb" %}
13256
13257 ins_encode %{
13258 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13259 as_Register($src2$$reg), ext::uxtb);
13260 %}
13261 ins_pipe(ialu_reg_reg);
13262 %}
13263
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13267 %{
13268 match(Set dst (AddI src1 (AndI src2 mask)));
13269 ins_cost(INSN_COST);
13270 format %{ "addw $dst, $src1, $src2, uxth" %}
13271
13272 ins_encode %{
13273 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13274 as_Register($src2$$reg), ext::uxth);
13275 %}
13276 ins_pipe(ialu_reg_reg);
13277 %}
13278
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13282 %{
13283 match(Set dst (AddL src1 (AndL src2 mask)));
13284 ins_cost(INSN_COST);
13285 format %{ "add $dst, $src1, $src2, uxtb" %}
13286
13287 ins_encode %{
13288 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13289 as_Register($src2$$reg), ext::uxtb);
13290 %}
13291 ins_pipe(ialu_reg_reg);
13292 %}
13293
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13297 %{
13298 match(Set dst (AddL src1 (AndL src2 mask)));
13299 ins_cost(INSN_COST);
13300 format %{ "add $dst, $src1, $src2, uxth" %}
13301
13302 ins_encode %{
13303 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13304 as_Register($src2$$reg), ext::uxth);
13305 %}
13306 ins_pipe(ialu_reg_reg);
13307 %}
13308
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13312 %{
13313 match(Set dst (AddL src1 (AndL src2 mask)));
13314 ins_cost(INSN_COST);
13315 format %{ "add $dst, $src1, $src2, uxtw" %}
13316
13317 ins_encode %{
13318 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13319 as_Register($src2$$reg), ext::uxtw);
13320 %}
13321 ins_pipe(ialu_reg_reg);
13322 %}
13323
13324 // This pattern is automatically generated from aarch64_ad.m4
13325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13326 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13327 %{
13328 match(Set dst (SubI src1 (AndI src2 mask)));
13329 ins_cost(INSN_COST);
13330 format %{ "subw $dst, $src1, $src2, uxtb" %}
13331
13332 ins_encode %{
13333 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13334 as_Register($src2$$reg), ext::uxtb);
13335 %}
13336 ins_pipe(ialu_reg_reg);
13337 %}
13338
13339 // This pattern is automatically generated from aarch64_ad.m4
13340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13341 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13342 %{
13343 match(Set dst (SubI src1 (AndI src2 mask)));
13344 ins_cost(INSN_COST);
13345 format %{ "subw $dst, $src1, $src2, uxth" %}
13346
13347 ins_encode %{
13348 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13349 as_Register($src2$$reg), ext::uxth);
13350 %}
13351 ins_pipe(ialu_reg_reg);
13352 %}
13353
13354 // This pattern is automatically generated from aarch64_ad.m4
13355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13356 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13357 %{
13358 match(Set dst (SubL src1 (AndL src2 mask)));
13359 ins_cost(INSN_COST);
13360 format %{ "sub $dst, $src1, $src2, uxtb" %}
13361
13362 ins_encode %{
13363 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13364 as_Register($src2$$reg), ext::uxtb);
13365 %}
13366 ins_pipe(ialu_reg_reg);
13367 %}
13368
13369 // This pattern is automatically generated from aarch64_ad.m4
13370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13371 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13372 %{
13373 match(Set dst (SubL src1 (AndL src2 mask)));
13374 ins_cost(INSN_COST);
13375 format %{ "sub $dst, $src1, $src2, uxth" %}
13376
13377 ins_encode %{
13378 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13379 as_Register($src2$$reg), ext::uxth);
13380 %}
13381 ins_pipe(ialu_reg_reg);
13382 %}
13383
13384 // This pattern is automatically generated from aarch64_ad.m4
13385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13386 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13387 %{
13388 match(Set dst (SubL src1 (AndL src2 mask)));
13389 ins_cost(INSN_COST);
13390 format %{ "sub $dst, $src1, $src2, uxtw" %}
13391
13392 ins_encode %{
13393 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13394 as_Register($src2$$reg), ext::uxtw);
13395 %}
13396 ins_pipe(ialu_reg_reg);
13397 %}
13398
13399
13400 // This pattern is automatically generated from aarch64_ad.m4
13401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13402 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13403 %{
13404 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13405 ins_cost(1.9 * INSN_COST);
13406 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13407
13408 ins_encode %{
13409 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13410 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13411 %}
13412 ins_pipe(ialu_reg_reg_shift);
13413 %}
13414
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13418 %{
13419 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13420 ins_cost(1.9 * INSN_COST);
13421 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13422
13423 ins_encode %{
13424 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13425 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13426 %}
13427 ins_pipe(ialu_reg_reg_shift);
13428 %}
13429
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13433 %{
13434 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13435 ins_cost(1.9 * INSN_COST);
13436 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13437
13438 ins_encode %{
13439 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13440 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13441 %}
13442 ins_pipe(ialu_reg_reg_shift);
13443 %}
13444
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13448 %{
13449 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13450 ins_cost(1.9 * INSN_COST);
13451 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13452
13453 ins_encode %{
13454 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13455 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13456 %}
13457 ins_pipe(ialu_reg_reg_shift);
13458 %}
13459
13460 // This pattern is automatically generated from aarch64_ad.m4
13461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13462 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13463 %{
13464 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13465 ins_cost(1.9 * INSN_COST);
13466 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13467
13468 ins_encode %{
13469 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13470 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13471 %}
13472 ins_pipe(ialu_reg_reg_shift);
13473 %}
13474
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13478 %{
13479 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13480 ins_cost(1.9 * INSN_COST);
13481 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13482
13483 ins_encode %{
13484 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13485 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13486 %}
13487 ins_pipe(ialu_reg_reg_shift);
13488 %}
13489
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13493 %{
13494 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13495 ins_cost(1.9 * INSN_COST);
13496 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13497
13498 ins_encode %{
13499 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13500 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13501 %}
13502 ins_pipe(ialu_reg_reg_shift);
13503 %}
13504
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13508 %{
13509 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13510 ins_cost(1.9 * INSN_COST);
13511 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13512
13513 ins_encode %{
13514 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13515 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13516 %}
13517 ins_pipe(ialu_reg_reg_shift);
13518 %}
13519
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13523 %{
13524 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13525 ins_cost(1.9 * INSN_COST);
13526 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13527
13528 ins_encode %{
13529 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13530 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13531 %}
13532 ins_pipe(ialu_reg_reg_shift);
13533 %}
13534
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13538 %{
13539 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13540 ins_cost(1.9 * INSN_COST);
13541 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13542
13543 ins_encode %{
13544 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13545 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13546 %}
13547 ins_pipe(ialu_reg_reg_shift);
13548 %}
13549
13550 // This pattern is automatically generated from aarch64_ad.m4
13551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13552 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13553 %{
13554 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13555 ins_cost(1.9 * INSN_COST);
13556 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13557
13558 ins_encode %{
13559 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13560 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13561 %}
13562 ins_pipe(ialu_reg_reg_shift);
13563 %}
13564
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13568 %{
13569 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13570 ins_cost(1.9 * INSN_COST);
13571 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13572
13573 ins_encode %{
13574 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13575 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13576 %}
13577 ins_pipe(ialu_reg_reg_shift);
13578 %}
13579
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13583 %{
13584 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13585 ins_cost(1.9 * INSN_COST);
13586 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13587
13588 ins_encode %{
13589 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13590 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13591 %}
13592 ins_pipe(ialu_reg_reg_shift);
13593 %}
13594
13595 // This pattern is automatically generated from aarch64_ad.m4
13596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13597 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13598 %{
13599 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13600 ins_cost(1.9 * INSN_COST);
13601 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13602
13603 ins_encode %{
13604 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13605 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13606 %}
13607 ins_pipe(ialu_reg_reg_shift);
13608 %}
13609
13610 // This pattern is automatically generated from aarch64_ad.m4
13611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13612 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13613 %{
13614 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13615 ins_cost(1.9 * INSN_COST);
13616 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13617
13618 ins_encode %{
13619 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13620 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13621 %}
13622 ins_pipe(ialu_reg_reg_shift);
13623 %}
13624
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13628 %{
13629 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13630 ins_cost(1.9 * INSN_COST);
13631 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13632
13633 ins_encode %{
13634 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13635 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13636 %}
13637 ins_pipe(ialu_reg_reg_shift);
13638 %}
13639
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13643 %{
13644 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13645 ins_cost(1.9 * INSN_COST);
13646 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13647
13648 ins_encode %{
13649 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13650 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13651 %}
13652 ins_pipe(ialu_reg_reg_shift);
13653 %}
13654
13655 // This pattern is automatically generated from aarch64_ad.m4
13656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13657 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13658 %{
13659 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13660 ins_cost(1.9 * INSN_COST);
13661 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13662
13663 ins_encode %{
13664 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13665 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13666 %}
13667 ins_pipe(ialu_reg_reg_shift);
13668 %}
13669
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13673 %{
13674 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13675 ins_cost(1.9 * INSN_COST);
13676 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13677
13678 ins_encode %{
13679 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13680 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13681 %}
13682 ins_pipe(ialu_reg_reg_shift);
13683 %}
13684
13685 // This pattern is automatically generated from aarch64_ad.m4
13686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13687 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13688 %{
13689 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13690 ins_cost(1.9 * INSN_COST);
13691 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13692
13693 ins_encode %{
13694 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13695 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13696 %}
13697 ins_pipe(ialu_reg_reg_shift);
13698 %}
13699
13700 // This pattern is automatically generated from aarch64_ad.m4
13701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13702 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13703 %{
13704 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13705 ins_cost(1.9 * INSN_COST);
13706 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13707
13708 ins_encode %{
13709 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13710 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13711 %}
13712 ins_pipe(ialu_reg_reg_shift);
13713 %}
13714
13715 // This pattern is automatically generated from aarch64_ad.m4
13716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13717 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13718 %{
13719 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13720 ins_cost(1.9 * INSN_COST);
13721 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13722
13723 ins_encode %{
13724 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13725 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13726 %}
13727 ins_pipe(ialu_reg_reg_shift);
13728 %}
13729
13730
13731
13732 // END This section of the file is automatically generated. Do not edit --------------
13733
13734
13735 // ============================================================================
13736 // Floating Point Arithmetic Instructions
13737
13738 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13739 match(Set dst (AddF src1 src2));
13740
13741 ins_cost(INSN_COST * 5);
13742 format %{ "fadds $dst, $src1, $src2" %}
13743
13744 ins_encode %{
13745 __ fadds(as_FloatRegister($dst$$reg),
13746 as_FloatRegister($src1$$reg),
13747 as_FloatRegister($src2$$reg));
13748 %}
13749
13750 ins_pipe(fp_dop_reg_reg_s);
13751 %}
13752
13753 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13754 match(Set dst (AddD src1 src2));
13755
13756 ins_cost(INSN_COST * 5);
13757 format %{ "faddd $dst, $src1, $src2" %}
13758
13759 ins_encode %{
13760 __ faddd(as_FloatRegister($dst$$reg),
13761 as_FloatRegister($src1$$reg),
13762 as_FloatRegister($src2$$reg));
13763 %}
13764
13765 ins_pipe(fp_dop_reg_reg_d);
13766 %}
13767
13768 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13769 match(Set dst (SubF src1 src2));
13770
13771 ins_cost(INSN_COST * 5);
13772 format %{ "fsubs $dst, $src1, $src2" %}
13773
13774 ins_encode %{
13775 __ fsubs(as_FloatRegister($dst$$reg),
13776 as_FloatRegister($src1$$reg),
13777 as_FloatRegister($src2$$reg));
13778 %}
13779
13780 ins_pipe(fp_dop_reg_reg_s);
13781 %}
13782
13783 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13784 match(Set dst (SubD src1 src2));
13785
13786 ins_cost(INSN_COST * 5);
13787 format %{ "fsubd $dst, $src1, $src2" %}
13788
13789 ins_encode %{
13790 __ fsubd(as_FloatRegister($dst$$reg),
13791 as_FloatRegister($src1$$reg),
13792 as_FloatRegister($src2$$reg));
13793 %}
13794
13795 ins_pipe(fp_dop_reg_reg_d);
13796 %}
13797
13798 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13799 match(Set dst (MulF src1 src2));
13800
13801 ins_cost(INSN_COST * 6);
13802 format %{ "fmuls $dst, $src1, $src2" %}
13803
13804 ins_encode %{
13805 __ fmuls(as_FloatRegister($dst$$reg),
13806 as_FloatRegister($src1$$reg),
13807 as_FloatRegister($src2$$reg));
13808 %}
13809
13810 ins_pipe(fp_dop_reg_reg_s);
13811 %}
13812
13813 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13814 match(Set dst (MulD src1 src2));
13815
13816 ins_cost(INSN_COST * 6);
13817 format %{ "fmuld $dst, $src1, $src2" %}
13818
13819 ins_encode %{
13820 __ fmuld(as_FloatRegister($dst$$reg),
13821 as_FloatRegister($src1$$reg),
13822 as_FloatRegister($src2$$reg));
13823 %}
13824
13825 ins_pipe(fp_dop_reg_reg_d);
13826 %}
13827
13828 // src1 * src2 + src3
13829 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13830 predicate(UseFMA);
13831 match(Set dst (FmaF src3 (Binary src1 src2)));
13832
13833 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13834
13835 ins_encode %{
13836 __ fmadds(as_FloatRegister($dst$$reg),
13837 as_FloatRegister($src1$$reg),
13838 as_FloatRegister($src2$$reg),
13839 as_FloatRegister($src3$$reg));
13840 %}
13841
13842 ins_pipe(pipe_class_default);
13843 %}
13844
13845 // src1 * src2 + src3
13846 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13847 predicate(UseFMA);
13848 match(Set dst (FmaD src3 (Binary src1 src2)));
13849
13850 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13851
13852 ins_encode %{
13853 __ fmaddd(as_FloatRegister($dst$$reg),
13854 as_FloatRegister($src1$$reg),
13855 as_FloatRegister($src2$$reg),
13856 as_FloatRegister($src3$$reg));
13857 %}
13858
13859 ins_pipe(pipe_class_default);
13860 %}
13861
13862 // -src1 * src2 + src3
13863 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13864 predicate(UseFMA);
13865 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13866 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13867
13868 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13869
13870 ins_encode %{
13871 __ fmsubs(as_FloatRegister($dst$$reg),
13872 as_FloatRegister($src1$$reg),
13873 as_FloatRegister($src2$$reg),
13874 as_FloatRegister($src3$$reg));
13875 %}
13876
13877 ins_pipe(pipe_class_default);
13878 %}
13879
13880 // -src1 * src2 + src3
13881 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13882 predicate(UseFMA);
13883 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13884 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13885
13886 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13887
13888 ins_encode %{
13889 __ fmsubd(as_FloatRegister($dst$$reg),
13890 as_FloatRegister($src1$$reg),
13891 as_FloatRegister($src2$$reg),
13892 as_FloatRegister($src3$$reg));
13893 %}
13894
13895 ins_pipe(pipe_class_default);
13896 %}
13897
13898 // -src1 * src2 - src3
13899 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13900 predicate(UseFMA);
13901 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13902 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13903
13904 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13905
13906 ins_encode %{
13907 __ fnmadds(as_FloatRegister($dst$$reg),
13908 as_FloatRegister($src1$$reg),
13909 as_FloatRegister($src2$$reg),
13910 as_FloatRegister($src3$$reg));
13911 %}
13912
13913 ins_pipe(pipe_class_default);
13914 %}
13915
13916 // -src1 * src2 - src3
13917 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13918 predicate(UseFMA);
13919 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13920 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13921
13922 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13923
13924 ins_encode %{
13925 __ fnmaddd(as_FloatRegister($dst$$reg),
13926 as_FloatRegister($src1$$reg),
13927 as_FloatRegister($src2$$reg),
13928 as_FloatRegister($src3$$reg));
13929 %}
13930
13931 ins_pipe(pipe_class_default);
13932 %}
13933
13934 // src1 * src2 - src3
13935 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13936 predicate(UseFMA);
13937 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13938
13939 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13940
13941 ins_encode %{
13942 __ fnmsubs(as_FloatRegister($dst$$reg),
13943 as_FloatRegister($src1$$reg),
13944 as_FloatRegister($src2$$reg),
13945 as_FloatRegister($src3$$reg));
13946 %}
13947
13948 ins_pipe(pipe_class_default);
13949 %}
13950
13951 // src1 * src2 - src3
13952 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13953 predicate(UseFMA);
13954 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13955
13956 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13957
13958 ins_encode %{
13959 // n.b. insn name should be fnmsubd
13960 __ fnmsub(as_FloatRegister($dst$$reg),
13961 as_FloatRegister($src1$$reg),
13962 as_FloatRegister($src2$$reg),
13963 as_FloatRegister($src3$$reg));
13964 %}
13965
13966 ins_pipe(pipe_class_default);
13967 %}
13968
13969
13970 // Math.max(FF)F
13971 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13972 match(Set dst (MaxF src1 src2));
13973
13974 format %{ "fmaxs $dst, $src1, $src2" %}
13975 ins_encode %{
13976 __ fmaxs(as_FloatRegister($dst$$reg),
13977 as_FloatRegister($src1$$reg),
13978 as_FloatRegister($src2$$reg));
13979 %}
13980
13981 ins_pipe(fp_dop_reg_reg_s);
13982 %}
13983
13984 // Math.min(FF)F
13985 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13986 match(Set dst (MinF src1 src2));
13987
13988 format %{ "fmins $dst, $src1, $src2" %}
13989 ins_encode %{
13990 __ fmins(as_FloatRegister($dst$$reg),
13991 as_FloatRegister($src1$$reg),
13992 as_FloatRegister($src2$$reg));
13993 %}
13994
13995 ins_pipe(fp_dop_reg_reg_s);
13996 %}
13997
13998 // Math.max(DD)D
13999 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14000 match(Set dst (MaxD src1 src2));
14001
14002 format %{ "fmaxd $dst, $src1, $src2" %}
14003 ins_encode %{
14004 __ fmaxd(as_FloatRegister($dst$$reg),
14005 as_FloatRegister($src1$$reg),
14006 as_FloatRegister($src2$$reg));
14007 %}
14008
14009 ins_pipe(fp_dop_reg_reg_d);
14010 %}
14011
14012 // Math.min(DD)D
14013 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14014 match(Set dst (MinD src1 src2));
14015
14016 format %{ "fmind $dst, $src1, $src2" %}
14017 ins_encode %{
14018 __ fmind(as_FloatRegister($dst$$reg),
14019 as_FloatRegister($src1$$reg),
14020 as_FloatRegister($src2$$reg));
14021 %}
14022
14023 ins_pipe(fp_dop_reg_reg_d);
14024 %}
14025
14026
14027 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14028 match(Set dst (DivF src1 src2));
14029
14030 ins_cost(INSN_COST * 18);
14031 format %{ "fdivs $dst, $src1, $src2" %}
14032
14033 ins_encode %{
14034 __ fdivs(as_FloatRegister($dst$$reg),
14035 as_FloatRegister($src1$$reg),
14036 as_FloatRegister($src2$$reg));
14037 %}
14038
14039 ins_pipe(fp_div_s);
14040 %}
14041
14042 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14043 match(Set dst (DivD src1 src2));
14044
14045 ins_cost(INSN_COST * 32);
14046 format %{ "fdivd $dst, $src1, $src2" %}
14047
14048 ins_encode %{
14049 __ fdivd(as_FloatRegister($dst$$reg),
14050 as_FloatRegister($src1$$reg),
14051 as_FloatRegister($src2$$reg));
14052 %}
14053
14054 ins_pipe(fp_div_d);
14055 %}
14056
14057 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14058 match(Set dst (NegF src));
14059
14060 ins_cost(INSN_COST * 3);
14061 format %{ "fneg $dst, $src" %}
14062
14063 ins_encode %{
14064 __ fnegs(as_FloatRegister($dst$$reg),
14065 as_FloatRegister($src$$reg));
14066 %}
14067
14068 ins_pipe(fp_uop_s);
14069 %}
14070
14071 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14072 match(Set dst (NegD src));
14073
14074 ins_cost(INSN_COST * 3);
14075 format %{ "fnegd $dst, $src" %}
14076
14077 ins_encode %{
14078 __ fnegd(as_FloatRegister($dst$$reg),
14079 as_FloatRegister($src$$reg));
14080 %}
14081
14082 ins_pipe(fp_uop_d);
14083 %}
14084
14085 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14086 %{
14087 match(Set dst (AbsI src));
14088
14089 effect(KILL cr);
14090 ins_cost(INSN_COST * 2);
14091 format %{ "cmpw $src, zr\n\t"
14092 "cnegw $dst, $src, Assembler::LT\t# int abs"
14093 %}
14094
14095 ins_encode %{
14096 __ cmpw(as_Register($src$$reg), zr);
14097 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14098 %}
14099 ins_pipe(pipe_class_default);
14100 %}
14101
14102 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14103 %{
14104 match(Set dst (AbsL src));
14105
14106 effect(KILL cr);
14107 ins_cost(INSN_COST * 2);
14108 format %{ "cmp $src, zr\n\t"
14109 "cneg $dst, $src, Assembler::LT\t# long abs"
14110 %}
14111
14112 ins_encode %{
14113 __ cmp(as_Register($src$$reg), zr);
14114 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14115 %}
14116 ins_pipe(pipe_class_default);
14117 %}
14118
14119 instruct absF_reg(vRegF dst, vRegF src) %{
14120 match(Set dst (AbsF src));
14121
14122 ins_cost(INSN_COST * 3);
14123 format %{ "fabss $dst, $src" %}
14124 ins_encode %{
14125 __ fabss(as_FloatRegister($dst$$reg),
14126 as_FloatRegister($src$$reg));
14127 %}
14128
14129 ins_pipe(fp_uop_s);
14130 %}
14131
14132 instruct absD_reg(vRegD dst, vRegD src) %{
14133 match(Set dst (AbsD src));
14134
14135 ins_cost(INSN_COST * 3);
14136 format %{ "fabsd $dst, $src" %}
14137 ins_encode %{
14138 __ fabsd(as_FloatRegister($dst$$reg),
14139 as_FloatRegister($src$$reg));
14140 %}
14141
14142 ins_pipe(fp_uop_d);
14143 %}
14144
14145 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14146 match(Set dst (AbsF (SubF src1 src2)));
14147
14148 ins_cost(INSN_COST * 3);
14149 format %{ "fabds $dst, $src1, $src2" %}
14150 ins_encode %{
14151 __ fabds(as_FloatRegister($dst$$reg),
14152 as_FloatRegister($src1$$reg),
14153 as_FloatRegister($src2$$reg));
14154 %}
14155
14156 ins_pipe(fp_uop_s);
14157 %}
14158
14159 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14160 match(Set dst (AbsD (SubD src1 src2)));
14161
14162 ins_cost(INSN_COST * 3);
14163 format %{ "fabdd $dst, $src1, $src2" %}
14164 ins_encode %{
14165 __ fabdd(as_FloatRegister($dst$$reg),
14166 as_FloatRegister($src1$$reg),
14167 as_FloatRegister($src2$$reg));
14168 %}
14169
14170 ins_pipe(fp_uop_d);
14171 %}
14172
14173 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14174 match(Set dst (SqrtD src));
14175
14176 ins_cost(INSN_COST * 50);
14177 format %{ "fsqrtd $dst, $src" %}
14178 ins_encode %{
14179 __ fsqrtd(as_FloatRegister($dst$$reg),
14180 as_FloatRegister($src$$reg));
14181 %}
14182
14183 ins_pipe(fp_div_s);
14184 %}
14185
14186 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14187 match(Set dst (SqrtF src));
14188
14189 ins_cost(INSN_COST * 50);
14190 format %{ "fsqrts $dst, $src" %}
14191 ins_encode %{
14192 __ fsqrts(as_FloatRegister($dst$$reg),
14193 as_FloatRegister($src$$reg));
14194 %}
14195
14196 ins_pipe(fp_div_d);
14197 %}
14198
14199 // Math.rint, floor, ceil
14200 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14201 match(Set dst (RoundDoubleMode src rmode));
14202 format %{ "frint $dst, $src, $rmode" %}
14203 ins_encode %{
14204 switch ($rmode$$constant) {
14205 case RoundDoubleModeNode::rmode_rint:
14206 __ frintnd(as_FloatRegister($dst$$reg),
14207 as_FloatRegister($src$$reg));
14208 break;
14209 case RoundDoubleModeNode::rmode_floor:
14210 __ frintmd(as_FloatRegister($dst$$reg),
14211 as_FloatRegister($src$$reg));
14212 break;
14213 case RoundDoubleModeNode::rmode_ceil:
14214 __ frintpd(as_FloatRegister($dst$$reg),
14215 as_FloatRegister($src$$reg));
14216 break;
14217 }
14218 %}
14219 ins_pipe(fp_uop_d);
14220 %}
14221
14222 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14223 match(Set dst (CopySignD src1 (Binary src2 zero)));
14224 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14225 format %{ "CopySignD $dst $src1 $src2" %}
14226 ins_encode %{
14227 FloatRegister dst = as_FloatRegister($dst$$reg),
14228 src1 = as_FloatRegister($src1$$reg),
14229 src2 = as_FloatRegister($src2$$reg),
14230 zero = as_FloatRegister($zero$$reg);
14231 __ fnegd(dst, zero);
14232 __ bsl(dst, __ T8B, src2, src1);
14233 %}
14234 ins_pipe(fp_uop_d);
14235 %}
14236
14237 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14238 match(Set dst (CopySignF src1 src2));
14239 effect(TEMP_DEF dst, USE src1, USE src2);
14240 format %{ "CopySignF $dst $src1 $src2" %}
14241 ins_encode %{
14242 FloatRegister dst = as_FloatRegister($dst$$reg),
14243 src1 = as_FloatRegister($src1$$reg),
14244 src2 = as_FloatRegister($src2$$reg);
14245 __ movi(dst, __ T2S, 0x80, 24);
14246 __ bsl(dst, __ T8B, src2, src1);
14247 %}
14248 ins_pipe(fp_uop_d);
14249 %}
14250
14251 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14252 match(Set dst (SignumD src (Binary zero one)));
14253 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14254 format %{ "signumD $dst, $src" %}
14255 ins_encode %{
14256 FloatRegister src = as_FloatRegister($src$$reg),
14257 dst = as_FloatRegister($dst$$reg),
14258 zero = as_FloatRegister($zero$$reg),
14259 one = as_FloatRegister($one$$reg);
14260 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14261 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14262 // Bit selection instruction gets bit from "one" for each enabled bit in
14263 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14264 // NaN the whole "src" will be copied because "dst" is zero. For all other
14265 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14266 // from "src", and all other bits are copied from 1.0.
14267 __ bsl(dst, __ T8B, one, src);
14268 %}
14269 ins_pipe(fp_uop_d);
14270 %}
14271
14272 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14273 match(Set dst (SignumF src (Binary zero one)));
14274 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14275 format %{ "signumF $dst, $src" %}
14276 ins_encode %{
14277 FloatRegister src = as_FloatRegister($src$$reg),
14278 dst = as_FloatRegister($dst$$reg),
14279 zero = as_FloatRegister($zero$$reg),
14280 one = as_FloatRegister($one$$reg);
14281 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14282 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14283 // Bit selection instruction gets bit from "one" for each enabled bit in
14284 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14285 // NaN the whole "src" will be copied because "dst" is zero. For all other
14286 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14287 // from "src", and all other bits are copied from 1.0.
14288 __ bsl(dst, __ T8B, one, src);
14289 %}
14290 ins_pipe(fp_uop_d);
14291 %}
14292
14293 instruct onspinwait() %{
14294 match(OnSpinWait);
14295 ins_cost(INSN_COST);
14296
14297 format %{ "onspinwait" %}
14298
14299 ins_encode %{
14300 __ spin_wait();
14301 %}
14302 ins_pipe(pipe_class_empty);
14303 %}
14304
14305 // ============================================================================
14306 // Logical Instructions
14307
14308 // Integer Logical Instructions
14309
14310 // And Instructions
14311
14312
14313 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14314 match(Set dst (AndI src1 src2));
14315
14316 format %{ "andw $dst, $src1, $src2\t# int" %}
14317
14318 ins_cost(INSN_COST);
14319 ins_encode %{
14320 __ andw(as_Register($dst$$reg),
14321 as_Register($src1$$reg),
14322 as_Register($src2$$reg));
14323 %}
14324
14325 ins_pipe(ialu_reg_reg);
14326 %}
14327
14328 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14329 match(Set dst (AndI src1 src2));
14330
14331 format %{ "andsw $dst, $src1, $src2\t# int" %}
14332
14333 ins_cost(INSN_COST);
14334 ins_encode %{
14335 __ andw(as_Register($dst$$reg),
14336 as_Register($src1$$reg),
14337 (uint64_t)($src2$$constant));
14338 %}
14339
14340 ins_pipe(ialu_reg_imm);
14341 %}
14342
14343 // Or Instructions
14344
14345 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14346 match(Set dst (OrI src1 src2));
14347
14348 format %{ "orrw $dst, $src1, $src2\t# int" %}
14349
14350 ins_cost(INSN_COST);
14351 ins_encode %{
14352 __ orrw(as_Register($dst$$reg),
14353 as_Register($src1$$reg),
14354 as_Register($src2$$reg));
14355 %}
14356
14357 ins_pipe(ialu_reg_reg);
14358 %}
14359
14360 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14361 match(Set dst (OrI src1 src2));
14362
14363 format %{ "orrw $dst, $src1, $src2\t# int" %}
14364
14365 ins_cost(INSN_COST);
14366 ins_encode %{
14367 __ orrw(as_Register($dst$$reg),
14368 as_Register($src1$$reg),
14369 (uint64_t)($src2$$constant));
14370 %}
14371
14372 ins_pipe(ialu_reg_imm);
14373 %}
14374
14375 // Xor Instructions
14376
14377 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14378 match(Set dst (XorI src1 src2));
14379
14380 format %{ "eorw $dst, $src1, $src2\t# int" %}
14381
14382 ins_cost(INSN_COST);
14383 ins_encode %{
14384 __ eorw(as_Register($dst$$reg),
14385 as_Register($src1$$reg),
14386 as_Register($src2$$reg));
14387 %}
14388
14389 ins_pipe(ialu_reg_reg);
14390 %}
14391
14392 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14393 match(Set dst (XorI src1 src2));
14394
14395 format %{ "eorw $dst, $src1, $src2\t# int" %}
14396
14397 ins_cost(INSN_COST);
14398 ins_encode %{
14399 __ eorw(as_Register($dst$$reg),
14400 as_Register($src1$$reg),
14401 (uint64_t)($src2$$constant));
14402 %}
14403
14404 ins_pipe(ialu_reg_imm);
14405 %}
14406
14407 // Long Logical Instructions
14408 // TODO
14409
14410 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14411 match(Set dst (AndL src1 src2));
14412
14413 format %{ "and $dst, $src1, $src2\t# int" %}
14414
14415 ins_cost(INSN_COST);
14416 ins_encode %{
14417 __ andr(as_Register($dst$$reg),
14418 as_Register($src1$$reg),
14419 as_Register($src2$$reg));
14420 %}
14421
14422 ins_pipe(ialu_reg_reg);
14423 %}
14424
14425 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14426 match(Set dst (AndL src1 src2));
14427
14428 format %{ "and $dst, $src1, $src2\t# int" %}
14429
14430 ins_cost(INSN_COST);
14431 ins_encode %{
14432 __ andr(as_Register($dst$$reg),
14433 as_Register($src1$$reg),
14434 (uint64_t)($src2$$constant));
14435 %}
14436
14437 ins_pipe(ialu_reg_imm);
14438 %}
14439
14440 // Or Instructions
14441
14442 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14443 match(Set dst (OrL src1 src2));
14444
14445 format %{ "orr $dst, $src1, $src2\t# int" %}
14446
14447 ins_cost(INSN_COST);
14448 ins_encode %{
14449 __ orr(as_Register($dst$$reg),
14450 as_Register($src1$$reg),
14451 as_Register($src2$$reg));
14452 %}
14453
14454 ins_pipe(ialu_reg_reg);
14455 %}
14456
14457 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14458 match(Set dst (OrL src1 src2));
14459
14460 format %{ "orr $dst, $src1, $src2\t# int" %}
14461
14462 ins_cost(INSN_COST);
14463 ins_encode %{
14464 __ orr(as_Register($dst$$reg),
14465 as_Register($src1$$reg),
14466 (uint64_t)($src2$$constant));
14467 %}
14468
14469 ins_pipe(ialu_reg_imm);
14470 %}
14471
14472 // Xor Instructions
14473
14474 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14475 match(Set dst (XorL src1 src2));
14476
14477 format %{ "eor $dst, $src1, $src2\t# int" %}
14478
14479 ins_cost(INSN_COST);
14480 ins_encode %{
14481 __ eor(as_Register($dst$$reg),
14482 as_Register($src1$$reg),
14483 as_Register($src2$$reg));
14484 %}
14485
14486 ins_pipe(ialu_reg_reg);
14487 %}
14488
14489 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14490 match(Set dst (XorL src1 src2));
14491
14492 ins_cost(INSN_COST);
14493 format %{ "eor $dst, $src1, $src2\t# int" %}
14494
14495 ins_encode %{
14496 __ eor(as_Register($dst$$reg),
14497 as_Register($src1$$reg),
14498 (uint64_t)($src2$$constant));
14499 %}
14500
14501 ins_pipe(ialu_reg_imm);
14502 %}
14503
14504 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14505 %{
14506 match(Set dst (ConvI2L src));
14507
14508 ins_cost(INSN_COST);
14509 format %{ "sxtw $dst, $src\t# i2l" %}
14510 ins_encode %{
14511 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14512 %}
14513 ins_pipe(ialu_reg_shift);
14514 %}
14515
14516 // this pattern occurs in bigmath arithmetic
14517 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14518 %{
14519 match(Set dst (AndL (ConvI2L src) mask));
14520
14521 ins_cost(INSN_COST);
14522 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14523 ins_encode %{
14524 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14525 %}
14526
14527 ins_pipe(ialu_reg_shift);
14528 %}
14529
14530 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14531 match(Set dst (ConvL2I src));
14532
14533 ins_cost(INSN_COST);
14534 format %{ "movw $dst, $src \t// l2i" %}
14535
14536 ins_encode %{
14537 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14538 %}
14539
14540 ins_pipe(ialu_reg);
14541 %}
14542
14543 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14544 %{
14545 match(Set dst (Conv2B src));
14546 effect(KILL cr);
14547
14548 format %{
14549 "cmpw $src, zr\n\t"
14550 "cset $dst, ne"
14551 %}
14552
14553 ins_encode %{
14554 __ cmpw(as_Register($src$$reg), zr);
14555 __ cset(as_Register($dst$$reg), Assembler::NE);
14556 %}
14557
14558 ins_pipe(ialu_reg);
14559 %}
14560
14561 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14562 %{
14563 match(Set dst (Conv2B src));
14564 effect(KILL cr);
14565
14566 format %{
14567 "cmp $src, zr\n\t"
14568 "cset $dst, ne"
14569 %}
14570
14571 ins_encode %{
14572 __ cmp(as_Register($src$$reg), zr);
14573 __ cset(as_Register($dst$$reg), Assembler::NE);
14574 %}
14575
14576 ins_pipe(ialu_reg);
14577 %}
14578
14579 instruct convD2F_reg(vRegF dst, vRegD src) %{
14580 match(Set dst (ConvD2F src));
14581
14582 ins_cost(INSN_COST * 5);
14583 format %{ "fcvtd $dst, $src \t// d2f" %}
14584
14585 ins_encode %{
14586 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14587 %}
14588
14589 ins_pipe(fp_d2f);
14590 %}
14591
14592 instruct convF2D_reg(vRegD dst, vRegF src) %{
14593 match(Set dst (ConvF2D src));
14594
14595 ins_cost(INSN_COST * 5);
14596 format %{ "fcvts $dst, $src \t// f2d" %}
14597
14598 ins_encode %{
14599 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14600 %}
14601
14602 ins_pipe(fp_f2d);
14603 %}
14604
14605 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14606 match(Set dst (ConvF2I src));
14607
14608 ins_cost(INSN_COST * 5);
14609 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14610
14611 ins_encode %{
14612 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14613 %}
14614
14615 ins_pipe(fp_f2i);
14616 %}
14617
14618 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14619 match(Set dst (ConvF2L src));
14620
14621 ins_cost(INSN_COST * 5);
14622 format %{ "fcvtzs $dst, $src \t// f2l" %}
14623
14624 ins_encode %{
14625 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14626 %}
14627
14628 ins_pipe(fp_f2l);
14629 %}
14630
14631 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14632 match(Set dst (ConvI2F src));
14633
14634 ins_cost(INSN_COST * 5);
14635 format %{ "scvtfws $dst, $src \t// i2f" %}
14636
14637 ins_encode %{
14638 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14639 %}
14640
14641 ins_pipe(fp_i2f);
14642 %}
14643
14644 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14645 match(Set dst (ConvL2F src));
14646
14647 ins_cost(INSN_COST * 5);
14648 format %{ "scvtfs $dst, $src \t// l2f" %}
14649
14650 ins_encode %{
14651 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14652 %}
14653
14654 ins_pipe(fp_l2f);
14655 %}
14656
14657 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14658 match(Set dst (ConvD2I src));
14659
14660 ins_cost(INSN_COST * 5);
14661 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14662
14663 ins_encode %{
14664 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14665 %}
14666
14667 ins_pipe(fp_d2i);
14668 %}
14669
14670 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14671 match(Set dst (ConvD2L src));
14672
14673 ins_cost(INSN_COST * 5);
14674 format %{ "fcvtzd $dst, $src \t// d2l" %}
14675
14676 ins_encode %{
14677 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14678 %}
14679
14680 ins_pipe(fp_d2l);
14681 %}
14682
14683 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14684 match(Set dst (ConvI2D src));
14685
14686 ins_cost(INSN_COST * 5);
14687 format %{ "scvtfwd $dst, $src \t// i2d" %}
14688
14689 ins_encode %{
14690 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14691 %}
14692
14693 ins_pipe(fp_i2d);
14694 %}
14695
14696 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14697 match(Set dst (ConvL2D src));
14698
14699 ins_cost(INSN_COST * 5);
14700 format %{ "scvtfd $dst, $src \t// l2d" %}
14701
14702 ins_encode %{
14703 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14704 %}
14705
14706 ins_pipe(fp_l2d);
14707 %}
14708
14709 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14710 %{
14711 match(Set dst (RoundD src));
14712 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14713 format %{ "java_round_double $dst,$src"%}
14714 ins_encode %{
14715 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14716 as_FloatRegister($ftmp$$reg));
14717 %}
14718 ins_pipe(pipe_slow);
14719 %}
14720
14721 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14722 %{
14723 match(Set dst (RoundF src));
14724 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14725 format %{ "java_round_float $dst,$src"%}
14726 ins_encode %{
14727 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14728 as_FloatRegister($ftmp$$reg));
14729 %}
14730 ins_pipe(pipe_slow);
14731 %}
14732
14733 // stack <-> reg and reg <-> reg shuffles with no conversion
14734
14735 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14736
14737 match(Set dst (MoveF2I src));
14738
14739 effect(DEF dst, USE src);
14740
14741 ins_cost(4 * INSN_COST);
14742
14743 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14744
14745 ins_encode %{
14746 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14747 %}
14748
14749 ins_pipe(iload_reg_reg);
14750
14751 %}
14752
14753 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14754
14755 match(Set dst (MoveI2F src));
14756
14757 effect(DEF dst, USE src);
14758
14759 ins_cost(4 * INSN_COST);
14760
14761 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14762
14763 ins_encode %{
14764 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14765 %}
14766
14767 ins_pipe(pipe_class_memory);
14768
14769 %}
14770
14771 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14772
14773 match(Set dst (MoveD2L src));
14774
14775 effect(DEF dst, USE src);
14776
14777 ins_cost(4 * INSN_COST);
14778
14779 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14780
14781 ins_encode %{
14782 __ ldr($dst$$Register, Address(sp, $src$$disp));
14783 %}
14784
14785 ins_pipe(iload_reg_reg);
14786
14787 %}
14788
14789 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14790
14791 match(Set dst (MoveL2D src));
14792
14793 effect(DEF dst, USE src);
14794
14795 ins_cost(4 * INSN_COST);
14796
14797 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14798
14799 ins_encode %{
14800 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14801 %}
14802
14803 ins_pipe(pipe_class_memory);
14804
14805 %}
14806
14807 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14808
14809 match(Set dst (MoveF2I src));
14810
14811 effect(DEF dst, USE src);
14812
14813 ins_cost(INSN_COST);
14814
14815 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14816
14817 ins_encode %{
14818 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14819 %}
14820
14821 ins_pipe(pipe_class_memory);
14822
14823 %}
14824
14825 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14826
14827 match(Set dst (MoveI2F src));
14828
14829 effect(DEF dst, USE src);
14830
14831 ins_cost(INSN_COST);
14832
14833 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14834
14835 ins_encode %{
14836 __ strw($src$$Register, Address(sp, $dst$$disp));
14837 %}
14838
14839 ins_pipe(istore_reg_reg);
14840
14841 %}
14842
14843 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14844
14845 match(Set dst (MoveD2L src));
14846
14847 effect(DEF dst, USE src);
14848
14849 ins_cost(INSN_COST);
14850
14851 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14852
14853 ins_encode %{
14854 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14855 %}
14856
14857 ins_pipe(pipe_class_memory);
14858
14859 %}
14860
14861 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14862
14863 match(Set dst (MoveL2D src));
14864
14865 effect(DEF dst, USE src);
14866
14867 ins_cost(INSN_COST);
14868
14869 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14870
14871 ins_encode %{
14872 __ str($src$$Register, Address(sp, $dst$$disp));
14873 %}
14874
14875 ins_pipe(istore_reg_reg);
14876
14877 %}
14878
14879 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14880
14881 match(Set dst (MoveF2I src));
14882
14883 effect(DEF dst, USE src);
14884
14885 ins_cost(INSN_COST);
14886
14887 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14888
14889 ins_encode %{
14890 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14891 %}
14892
14893 ins_pipe(fp_f2i);
14894
14895 %}
14896
14897 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14898
14899 match(Set dst (MoveI2F src));
14900
14901 effect(DEF dst, USE src);
14902
14903 ins_cost(INSN_COST);
14904
14905 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14906
14907 ins_encode %{
14908 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14909 %}
14910
14911 ins_pipe(fp_i2f);
14912
14913 %}
14914
14915 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14916
14917 match(Set dst (MoveD2L src));
14918
14919 effect(DEF dst, USE src);
14920
14921 ins_cost(INSN_COST);
14922
14923 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14924
14925 ins_encode %{
14926 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14927 %}
14928
14929 ins_pipe(fp_d2l);
14930
14931 %}
14932
14933 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14934
14935 match(Set dst (MoveL2D src));
14936
14937 effect(DEF dst, USE src);
14938
14939 ins_cost(INSN_COST);
14940
14941 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14942
14943 ins_encode %{
14944 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14945 %}
14946
14947 ins_pipe(fp_l2d);
14948
14949 %}
14950
14951 // ============================================================================
14952 // clearing of an array
14953
14954 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14955 %{
14956 match(Set dummy (ClearArray (Binary cnt base) zero));
14957 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14958
14959 ins_cost(4 * INSN_COST);
14960 format %{ "ClearArray $cnt, $base" %}
14961
14962 ins_encode %{
14963 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14964 if (tpc == NULL) {
14965 ciEnv::current()->record_failure("CodeCache is full");
14966 return;
14967 }
14968 %}
14969
14970 ins_pipe(pipe_class_memory);
14971 %}
14972
14973 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14974 %{
14975 predicate(((ClearArrayNode*)n)->word_copy_only());
14976 match(Set dummy (ClearArray (Binary cnt base) val));
14977 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14978
14979 ins_cost(4 * INSN_COST);
14980 format %{ "ClearArray $cnt, $base, $val" %}
14981
14982 ins_encode %{
14983 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14984 %}
14985
14986 ins_pipe(pipe_class_memory);
14987 %}
14988
14989 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14990 %{
14991 predicate((uint64_t)n->in(2)->get_long()
14992 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14993 && !((ClearArrayNode*)n)->word_copy_only());
14994 match(Set dummy (ClearArray cnt base));
14995 effect(TEMP temp, USE_KILL base, KILL cr);
14996
14997 ins_cost(4 * INSN_COST);
14998 format %{ "ClearArray $cnt, $base" %}
14999
15000 ins_encode %{
15001 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15002 if (tpc == NULL) {
15003 ciEnv::current()->record_failure("CodeCache is full");
15004 return;
15005 }
15006 %}
15007
15008 ins_pipe(pipe_class_memory);
15009 %}
15010
15011 // ============================================================================
15012 // Overflow Math Instructions
15013
15014 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15015 %{
15016 match(Set cr (OverflowAddI op1 op2));
15017
15018 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15019 ins_cost(INSN_COST);
15020 ins_encode %{
15021 __ cmnw($op1$$Register, $op2$$Register);
15022 %}
15023
15024 ins_pipe(icmp_reg_reg);
15025 %}
15026
15027 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15028 %{
15029 match(Set cr (OverflowAddI op1 op2));
15030
15031 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15032 ins_cost(INSN_COST);
15033 ins_encode %{
15034 __ cmnw($op1$$Register, $op2$$constant);
15035 %}
15036
15037 ins_pipe(icmp_reg_imm);
15038 %}
15039
15040 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15041 %{
15042 match(Set cr (OverflowAddL op1 op2));
15043
15044 format %{ "cmn $op1, $op2\t# overflow check long" %}
15045 ins_cost(INSN_COST);
15046 ins_encode %{
15047 __ cmn($op1$$Register, $op2$$Register);
15048 %}
15049
15050 ins_pipe(icmp_reg_reg);
15051 %}
15052
15053 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15054 %{
15055 match(Set cr (OverflowAddL op1 op2));
15056
15057 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15058 ins_cost(INSN_COST);
15059 ins_encode %{
15060 __ adds(zr, $op1$$Register, $op2$$constant);
15061 %}
15062
15063 ins_pipe(icmp_reg_imm);
15064 %}
15065
15066 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15067 %{
15068 match(Set cr (OverflowSubI op1 op2));
15069
15070 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15071 ins_cost(INSN_COST);
15072 ins_encode %{
15073 __ cmpw($op1$$Register, $op2$$Register);
15074 %}
15075
15076 ins_pipe(icmp_reg_reg);
15077 %}
15078
15079 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15080 %{
15081 match(Set cr (OverflowSubI op1 op2));
15082
15083 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15084 ins_cost(INSN_COST);
15085 ins_encode %{
15086 __ cmpw($op1$$Register, $op2$$constant);
15087 %}
15088
15089 ins_pipe(icmp_reg_imm);
15090 %}
15091
15092 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15093 %{
15094 match(Set cr (OverflowSubL op1 op2));
15095
15096 format %{ "cmp $op1, $op2\t# overflow check long" %}
15097 ins_cost(INSN_COST);
15098 ins_encode %{
15099 __ cmp($op1$$Register, $op2$$Register);
15100 %}
15101
15102 ins_pipe(icmp_reg_reg);
15103 %}
15104
15105 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15106 %{
15107 match(Set cr (OverflowSubL op1 op2));
15108
15109 format %{ "cmp $op1, $op2\t# overflow check long" %}
15110 ins_cost(INSN_COST);
15111 ins_encode %{
15112 __ subs(zr, $op1$$Register, $op2$$constant);
15113 %}
15114
15115 ins_pipe(icmp_reg_imm);
15116 %}
15117
15118 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15119 %{
15120 match(Set cr (OverflowSubI zero op1));
15121
15122 format %{ "cmpw zr, $op1\t# overflow check int" %}
15123 ins_cost(INSN_COST);
15124 ins_encode %{
15125 __ cmpw(zr, $op1$$Register);
15126 %}
15127
15128 ins_pipe(icmp_reg_imm);
15129 %}
15130
15131 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15132 %{
15133 match(Set cr (OverflowSubL zero op1));
15134
15135 format %{ "cmp zr, $op1\t# overflow check long" %}
15136 ins_cost(INSN_COST);
15137 ins_encode %{
15138 __ cmp(zr, $op1$$Register);
15139 %}
15140
15141 ins_pipe(icmp_reg_imm);
15142 %}
15143
15144 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15145 %{
15146 match(Set cr (OverflowMulI op1 op2));
15147
15148 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15149 "cmp rscratch1, rscratch1, sxtw\n\t"
15150 "movw rscratch1, #0x80000000\n\t"
15151 "cselw rscratch1, rscratch1, zr, NE\n\t"
15152 "cmpw rscratch1, #1" %}
15153 ins_cost(5 * INSN_COST);
15154 ins_encode %{
15155 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15156 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15157 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15158 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15159 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15160 %}
15161
15162 ins_pipe(pipe_slow);
15163 %}
15164
15165 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15166 %{
15167 match(If cmp (OverflowMulI op1 op2));
15168 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15169 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15170 effect(USE labl, KILL cr);
15171
15172 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15173 "cmp rscratch1, rscratch1, sxtw\n\t"
15174 "b$cmp $labl" %}
15175 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15176 ins_encode %{
15177 Label* L = $labl$$label;
15178 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15179 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15180 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15181 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15182 %}
15183
15184 ins_pipe(pipe_serial);
15185 %}
15186
15187 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15188 %{
15189 match(Set cr (OverflowMulL op1 op2));
15190
15191 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15192 "smulh rscratch2, $op1, $op2\n\t"
15193 "cmp rscratch2, rscratch1, ASR #63\n\t"
15194 "movw rscratch1, #0x80000000\n\t"
15195 "cselw rscratch1, rscratch1, zr, NE\n\t"
15196 "cmpw rscratch1, #1" %}
15197 ins_cost(6 * INSN_COST);
15198 ins_encode %{
15199 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15200 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15201 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15202 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15203 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15204 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15205 %}
15206
15207 ins_pipe(pipe_slow);
15208 %}
15209
15210 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15211 %{
15212 match(If cmp (OverflowMulL op1 op2));
15213 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15214 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15215 effect(USE labl, KILL cr);
15216
15217 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15218 "smulh rscratch2, $op1, $op2\n\t"
15219 "cmp rscratch2, rscratch1, ASR #63\n\t"
15220 "b$cmp $labl" %}
15221 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15222 ins_encode %{
15223 Label* L = $labl$$label;
15224 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15225 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15226 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15227 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15228 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15229 %}
15230
15231 ins_pipe(pipe_serial);
15232 %}
15233
15234 // ============================================================================
15235 // Compare Instructions
15236
15237 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15238 %{
15239 match(Set cr (CmpI op1 op2));
15240
15241 effect(DEF cr, USE op1, USE op2);
15242
15243 ins_cost(INSN_COST);
15244 format %{ "cmpw $op1, $op2" %}
15245
15246 ins_encode(aarch64_enc_cmpw(op1, op2));
15247
15248 ins_pipe(icmp_reg_reg);
15249 %}
15250
15251 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15252 %{
15253 match(Set cr (CmpI op1 zero));
15254
15255 effect(DEF cr, USE op1);
15256
15257 ins_cost(INSN_COST);
15258 format %{ "cmpw $op1, 0" %}
15259
15260 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15261
15262 ins_pipe(icmp_reg_imm);
15263 %}
15264
15265 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15266 %{
15267 match(Set cr (CmpI op1 op2));
15268
15269 effect(DEF cr, USE op1);
15270
15271 ins_cost(INSN_COST);
15272 format %{ "cmpw $op1, $op2" %}
15273
15274 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15275
15276 ins_pipe(icmp_reg_imm);
15277 %}
15278
15279 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15280 %{
15281 match(Set cr (CmpI op1 op2));
15282
15283 effect(DEF cr, USE op1);
15284
15285 ins_cost(INSN_COST * 2);
15286 format %{ "cmpw $op1, $op2" %}
15287
15288 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15289
15290 ins_pipe(icmp_reg_imm);
15291 %}
15292
15293 // Unsigned compare Instructions; really, same as signed compare
15294 // except it should only be used to feed an If or a CMovI which takes a
15295 // cmpOpU.
15296
15297 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15298 %{
15299 match(Set cr (CmpU op1 op2));
15300
15301 effect(DEF cr, USE op1, USE op2);
15302
15303 ins_cost(INSN_COST);
15304 format %{ "cmpw $op1, $op2\t# unsigned" %}
15305
15306 ins_encode(aarch64_enc_cmpw(op1, op2));
15307
15308 ins_pipe(icmp_reg_reg);
15309 %}
15310
15311 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15312 %{
15313 match(Set cr (CmpU op1 zero));
15314
15315 effect(DEF cr, USE op1);
15316
15317 ins_cost(INSN_COST);
15318 format %{ "cmpw $op1, #0\t# unsigned" %}
15319
15320 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15321
15322 ins_pipe(icmp_reg_imm);
15323 %}
15324
15325 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15326 %{
15327 match(Set cr (CmpU op1 op2));
15328
15329 effect(DEF cr, USE op1);
15330
15331 ins_cost(INSN_COST);
15332 format %{ "cmpw $op1, $op2\t# unsigned" %}
15333
15334 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15335
15336 ins_pipe(icmp_reg_imm);
15337 %}
15338
15339 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15340 %{
15341 match(Set cr (CmpU op1 op2));
15342
15343 effect(DEF cr, USE op1);
15344
15345 ins_cost(INSN_COST * 2);
15346 format %{ "cmpw $op1, $op2\t# unsigned" %}
15347
15348 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15349
15350 ins_pipe(icmp_reg_imm);
15351 %}
15352
15353 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15354 %{
15355 match(Set cr (CmpL op1 op2));
15356
15357 effect(DEF cr, USE op1, USE op2);
15358
15359 ins_cost(INSN_COST);
15360 format %{ "cmp $op1, $op2" %}
15361
15362 ins_encode(aarch64_enc_cmp(op1, op2));
15363
15364 ins_pipe(icmp_reg_reg);
15365 %}
15366
15367 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15368 %{
15369 match(Set cr (CmpL op1 zero));
15370
15371 effect(DEF cr, USE op1);
15372
15373 ins_cost(INSN_COST);
15374 format %{ "tst $op1" %}
15375
15376 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15377
15378 ins_pipe(icmp_reg_imm);
15379 %}
15380
15381 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15382 %{
15383 match(Set cr (CmpL op1 op2));
15384
15385 effect(DEF cr, USE op1);
15386
15387 ins_cost(INSN_COST);
15388 format %{ "cmp $op1, $op2" %}
15389
15390 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15391
15392 ins_pipe(icmp_reg_imm);
15393 %}
15394
15395 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15396 %{
15397 match(Set cr (CmpL op1 op2));
15398
15399 effect(DEF cr, USE op1);
15400
15401 ins_cost(INSN_COST * 2);
15402 format %{ "cmp $op1, $op2" %}
15403
15404 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15405
15406 ins_pipe(icmp_reg_imm);
15407 %}
15408
15409 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15410 %{
15411 match(Set cr (CmpUL op1 op2));
15412
15413 effect(DEF cr, USE op1, USE op2);
15414
15415 ins_cost(INSN_COST);
15416 format %{ "cmp $op1, $op2" %}
15417
15418 ins_encode(aarch64_enc_cmp(op1, op2));
15419
15420 ins_pipe(icmp_reg_reg);
15421 %}
15422
15423 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15424 %{
15425 match(Set cr (CmpUL op1 zero));
15426
15427 effect(DEF cr, USE op1);
15428
15429 ins_cost(INSN_COST);
15430 format %{ "tst $op1" %}
15431
15432 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15433
15434 ins_pipe(icmp_reg_imm);
15435 %}
15436
15437 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15438 %{
15439 match(Set cr (CmpUL op1 op2));
15440
15441 effect(DEF cr, USE op1);
15442
15443 ins_cost(INSN_COST);
15444 format %{ "cmp $op1, $op2" %}
15445
15446 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15447
15448 ins_pipe(icmp_reg_imm);
15449 %}
15450
15451 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15452 %{
15453 match(Set cr (CmpUL op1 op2));
15454
15455 effect(DEF cr, USE op1);
15456
15457 ins_cost(INSN_COST * 2);
15458 format %{ "cmp $op1, $op2" %}
15459
15460 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15461
15462 ins_pipe(icmp_reg_imm);
15463 %}
15464
15465 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15466 %{
15467 match(Set cr (CmpP op1 op2));
15468
15469 effect(DEF cr, USE op1, USE op2);
15470
15471 ins_cost(INSN_COST);
15472 format %{ "cmp $op1, $op2\t // ptr" %}
15473
15474 ins_encode(aarch64_enc_cmpp(op1, op2));
15475
15476 ins_pipe(icmp_reg_reg);
15477 %}
15478
15479 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15480 %{
15481 match(Set cr (CmpN op1 op2));
15482
15483 effect(DEF cr, USE op1, USE op2);
15484
15485 ins_cost(INSN_COST);
15486 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15487
15488 ins_encode(aarch64_enc_cmpn(op1, op2));
15489
15490 ins_pipe(icmp_reg_reg);
15491 %}
15492
15493 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15494 %{
15495 match(Set cr (CmpP op1 zero));
15496
15497 effect(DEF cr, USE op1, USE zero);
15498
15499 ins_cost(INSN_COST);
15500 format %{ "cmp $op1, 0\t // ptr" %}
15501
15502 ins_encode(aarch64_enc_testp(op1));
15503
15504 ins_pipe(icmp_reg_imm);
15505 %}
15506
15507 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15508 %{
15509 match(Set cr (CmpN op1 zero));
15510
15511 effect(DEF cr, USE op1, USE zero);
15512
15513 ins_cost(INSN_COST);
15514 format %{ "cmp $op1, 0\t // compressed ptr" %}
15515
15516 ins_encode(aarch64_enc_testn(op1));
15517
15518 ins_pipe(icmp_reg_imm);
15519 %}
15520
15521 // FP comparisons
15522 //
15523 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15524 // using normal cmpOp. See declaration of rFlagsReg for details.
15525
15526 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15527 %{
15528 match(Set cr (CmpF src1 src2));
15529
15530 ins_cost(3 * INSN_COST);
15531 format %{ "fcmps $src1, $src2" %}
15532
15533 ins_encode %{
15534 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15535 %}
15536
15537 ins_pipe(pipe_class_compare);
15538 %}
15539
15540 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15541 %{
15542 match(Set cr (CmpF src1 src2));
15543
15544 ins_cost(3 * INSN_COST);
15545 format %{ "fcmps $src1, 0.0" %}
15546
15547 ins_encode %{
15548 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15549 %}
15550
15551 ins_pipe(pipe_class_compare);
15552 %}
15553 // FROM HERE
15554
15555 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15556 %{
15557 match(Set cr (CmpD src1 src2));
15558
15559 ins_cost(3 * INSN_COST);
15560 format %{ "fcmpd $src1, $src2" %}
15561
15562 ins_encode %{
15563 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15564 %}
15565
15566 ins_pipe(pipe_class_compare);
15567 %}
15568
15569 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15570 %{
15571 match(Set cr (CmpD src1 src2));
15572
15573 ins_cost(3 * INSN_COST);
15574 format %{ "fcmpd $src1, 0.0" %}
15575
15576 ins_encode %{
15577 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15578 %}
15579
15580 ins_pipe(pipe_class_compare);
15581 %}
15582
15583 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15584 %{
15585 match(Set dst (CmpF3 src1 src2));
15586 effect(KILL cr);
15587
15588 ins_cost(5 * INSN_COST);
15589 format %{ "fcmps $src1, $src2\n\t"
15590 "csinvw($dst, zr, zr, eq\n\t"
15591 "csnegw($dst, $dst, $dst, lt)"
15592 %}
15593
15594 ins_encode %{
15595 Label done;
15596 FloatRegister s1 = as_FloatRegister($src1$$reg);
15597 FloatRegister s2 = as_FloatRegister($src2$$reg);
15598 Register d = as_Register($dst$$reg);
15599 __ fcmps(s1, s2);
15600 // installs 0 if EQ else -1
15601 __ csinvw(d, zr, zr, Assembler::EQ);
15602 // keeps -1 if less or unordered else installs 1
15603 __ csnegw(d, d, d, Assembler::LT);
15604 __ bind(done);
15605 %}
15606
15607 ins_pipe(pipe_class_default);
15608
15609 %}
15610
15611 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15612 %{
15613 match(Set dst (CmpD3 src1 src2));
15614 effect(KILL cr);
15615
15616 ins_cost(5 * INSN_COST);
15617 format %{ "fcmpd $src1, $src2\n\t"
15618 "csinvw($dst, zr, zr, eq\n\t"
15619 "csnegw($dst, $dst, $dst, lt)"
15620 %}
15621
15622 ins_encode %{
15623 Label done;
15624 FloatRegister s1 = as_FloatRegister($src1$$reg);
15625 FloatRegister s2 = as_FloatRegister($src2$$reg);
15626 Register d = as_Register($dst$$reg);
15627 __ fcmpd(s1, s2);
15628 // installs 0 if EQ else -1
15629 __ csinvw(d, zr, zr, Assembler::EQ);
15630 // keeps -1 if less or unordered else installs 1
15631 __ csnegw(d, d, d, Assembler::LT);
15632 __ bind(done);
15633 %}
15634 ins_pipe(pipe_class_default);
15635
15636 %}
15637
15638 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15639 %{
15640 match(Set dst (CmpF3 src1 zero));
15641 effect(KILL cr);
15642
15643 ins_cost(5 * INSN_COST);
15644 format %{ "fcmps $src1, 0.0\n\t"
15645 "csinvw($dst, zr, zr, eq\n\t"
15646 "csnegw($dst, $dst, $dst, lt)"
15647 %}
15648
15649 ins_encode %{
15650 Label done;
15651 FloatRegister s1 = as_FloatRegister($src1$$reg);
15652 Register d = as_Register($dst$$reg);
15653 __ fcmps(s1, 0.0);
15654 // installs 0 if EQ else -1
15655 __ csinvw(d, zr, zr, Assembler::EQ);
15656 // keeps -1 if less or unordered else installs 1
15657 __ csnegw(d, d, d, Assembler::LT);
15658 __ bind(done);
15659 %}
15660
15661 ins_pipe(pipe_class_default);
15662
15663 %}
15664
15665 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15666 %{
15667 match(Set dst (CmpD3 src1 zero));
15668 effect(KILL cr);
15669
15670 ins_cost(5 * INSN_COST);
15671 format %{ "fcmpd $src1, 0.0\n\t"
15672 "csinvw($dst, zr, zr, eq\n\t"
15673 "csnegw($dst, $dst, $dst, lt)"
15674 %}
15675
15676 ins_encode %{
15677 Label done;
15678 FloatRegister s1 = as_FloatRegister($src1$$reg);
15679 Register d = as_Register($dst$$reg);
15680 __ fcmpd(s1, 0.0);
15681 // installs 0 if EQ else -1
15682 __ csinvw(d, zr, zr, Assembler::EQ);
15683 // keeps -1 if less or unordered else installs 1
15684 __ csnegw(d, d, d, Assembler::LT);
15685 __ bind(done);
15686 %}
15687 ins_pipe(pipe_class_default);
15688
15689 %}
15690
15691 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15692 %{
15693 match(Set dst (CmpLTMask p q));
15694 effect(KILL cr);
15695
15696 ins_cost(3 * INSN_COST);
15697
15698 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15699 "csetw $dst, lt\n\t"
15700 "subw $dst, zr, $dst"
15701 %}
15702
15703 ins_encode %{
15704 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15705 __ csetw(as_Register($dst$$reg), Assembler::LT);
15706 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15707 %}
15708
15709 ins_pipe(ialu_reg_reg);
15710 %}
15711
15712 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15713 %{
15714 match(Set dst (CmpLTMask src zero));
15715 effect(KILL cr);
15716
15717 ins_cost(INSN_COST);
15718
15719 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15720
15721 ins_encode %{
15722 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15723 %}
15724
15725 ins_pipe(ialu_reg_shift);
15726 %}
15727
15728 // ============================================================================
15729 // Max and Min
15730
15731 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15732 %{
15733 effect( DEF dst, USE src1, USE src2, USE cr );
15734
15735 ins_cost(INSN_COST * 2);
15736 format %{ "cselw $dst, $src1, $src2 lt\t" %}
15737
15738 ins_encode %{
15739 __ cselw(as_Register($dst$$reg),
15740 as_Register($src1$$reg),
15741 as_Register($src2$$reg),
15742 Assembler::LT);
15743 %}
15744
15745 ins_pipe(icond_reg_reg);
15746 %}
15747
15748 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15749 %{
15750 match(Set dst (MinI src1 src2));
15751 ins_cost(INSN_COST * 3);
15752
15753 expand %{
15754 rFlagsReg cr;
15755 compI_reg_reg(cr, src1, src2);
15756 cmovI_reg_reg_lt(dst, src1, src2, cr);
15757 %}
15758
15759 %}
15760 // FROM HERE
15761
15762 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15763 %{
15764 effect( DEF dst, USE src1, USE src2, USE cr );
15765
15766 ins_cost(INSN_COST * 2);
15767 format %{ "cselw $dst, $src1, $src2 gt\t" %}
15768
15769 ins_encode %{
15770 __ cselw(as_Register($dst$$reg),
15771 as_Register($src1$$reg),
15772 as_Register($src2$$reg),
15773 Assembler::GT);
15774 %}
15775
15776 ins_pipe(icond_reg_reg);
15777 %}
15778
15779 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15780 %{
15781 match(Set dst (MaxI src1 src2));
15782 ins_cost(INSN_COST * 3);
15783 expand %{
15784 rFlagsReg cr;
15785 compI_reg_reg(cr, src1, src2);
15786 cmovI_reg_reg_gt(dst, src1, src2, cr);
15787 %}
15788 %}
15789
15790 // ============================================================================
15791 // Branch Instructions
15792
15793 // Direct Branch.
15794 instruct branch(label lbl)
15795 %{
15796 match(Goto);
15797
15798 effect(USE lbl);
15799
15800 ins_cost(BRANCH_COST);
15801 format %{ "b $lbl" %}
15802
15803 ins_encode(aarch64_enc_b(lbl));
15804
15805 ins_pipe(pipe_branch);
15806 %}
15807
15808 // Conditional Near Branch
15809 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15810 %{
15811 // Same match rule as `branchConFar'.
15812 match(If cmp cr);
15813
15814 effect(USE lbl);
15815
15816 ins_cost(BRANCH_COST);
15817 // If set to 1 this indicates that the current instruction is a
15818 // short variant of a long branch. This avoids using this
15819 // instruction in first-pass matching. It will then only be used in
15820 // the `Shorten_branches' pass.
15821 // ins_short_branch(1);
15822 format %{ "b$cmp $lbl" %}
15823
15824 ins_encode(aarch64_enc_br_con(cmp, lbl));
15825
15826 ins_pipe(pipe_branch_cond);
15827 %}
15828
15829 // Conditional Near Branch Unsigned
15830 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15831 %{
15832 // Same match rule as `branchConFar'.
15833 match(If cmp cr);
15834
15835 effect(USE lbl);
15836
15837 ins_cost(BRANCH_COST);
15838 // If set to 1 this indicates that the current instruction is a
15839 // short variant of a long branch. This avoids using this
15840 // instruction in first-pass matching. It will then only be used in
15841 // the `Shorten_branches' pass.
15842 // ins_short_branch(1);
15843 format %{ "b$cmp $lbl\t# unsigned" %}
15844
15845 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15846
15847 ins_pipe(pipe_branch_cond);
15848 %}
15849
15850 // Make use of CBZ and CBNZ. These instructions, as well as being
15851 // shorter than (cmp; branch), have the additional benefit of not
15852 // killing the flags.
15853
15854 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15855 match(If cmp (CmpI op1 op2));
15856 effect(USE labl);
15857
15858 ins_cost(BRANCH_COST);
15859 format %{ "cbw$cmp $op1, $labl" %}
15860 ins_encode %{
15861 Label* L = $labl$$label;
15862 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15863 if (cond == Assembler::EQ)
15864 __ cbzw($op1$$Register, *L);
15865 else
15866 __ cbnzw($op1$$Register, *L);
15867 %}
15868 ins_pipe(pipe_cmp_branch);
15869 %}
15870
15871 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15872 match(If cmp (CmpL op1 op2));
15873 effect(USE labl);
15874
15875 ins_cost(BRANCH_COST);
15876 format %{ "cb$cmp $op1, $labl" %}
15877 ins_encode %{
15878 Label* L = $labl$$label;
15879 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15880 if (cond == Assembler::EQ)
15881 __ cbz($op1$$Register, *L);
15882 else
15883 __ cbnz($op1$$Register, *L);
15884 %}
15885 ins_pipe(pipe_cmp_branch);
15886 %}
15887
15888 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15889 match(If cmp (CmpP op1 op2));
15890 effect(USE labl);
15891
15892 ins_cost(BRANCH_COST);
15893 format %{ "cb$cmp $op1, $labl" %}
15894 ins_encode %{
15895 Label* L = $labl$$label;
15896 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15897 if (cond == Assembler::EQ)
15898 __ cbz($op1$$Register, *L);
15899 else
15900 __ cbnz($op1$$Register, *L);
15901 %}
15902 ins_pipe(pipe_cmp_branch);
15903 %}
15904
15905 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15906 match(If cmp (CmpN op1 op2));
15907 effect(USE labl);
15908
15909 ins_cost(BRANCH_COST);
15910 format %{ "cbw$cmp $op1, $labl" %}
15911 ins_encode %{
15912 Label* L = $labl$$label;
15913 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15914 if (cond == Assembler::EQ)
15915 __ cbzw($op1$$Register, *L);
15916 else
15917 __ cbnzw($op1$$Register, *L);
15918 %}
15919 ins_pipe(pipe_cmp_branch);
15920 %}
15921
15922 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15923 match(If cmp (CmpP (DecodeN oop) zero));
15924 effect(USE labl);
15925
15926 ins_cost(BRANCH_COST);
15927 format %{ "cb$cmp $oop, $labl" %}
15928 ins_encode %{
15929 Label* L = $labl$$label;
15930 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15931 if (cond == Assembler::EQ)
15932 __ cbzw($oop$$Register, *L);
15933 else
15934 __ cbnzw($oop$$Register, *L);
15935 %}
15936 ins_pipe(pipe_cmp_branch);
15937 %}
15938
15939 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15940 match(If cmp (CmpU op1 op2));
15941 effect(USE labl);
15942
15943 ins_cost(BRANCH_COST);
15944 format %{ "cbw$cmp $op1, $labl" %}
15945 ins_encode %{
15946 Label* L = $labl$$label;
15947 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15948 if (cond == Assembler::EQ || cond == Assembler::LS)
15949 __ cbzw($op1$$Register, *L);
15950 else
15951 __ cbnzw($op1$$Register, *L);
15952 %}
15953 ins_pipe(pipe_cmp_branch);
15954 %}
15955
15956 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15957 match(If cmp (CmpUL op1 op2));
15958 effect(USE labl);
15959
15960 ins_cost(BRANCH_COST);
15961 format %{ "cb$cmp $op1, $labl" %}
15962 ins_encode %{
15963 Label* L = $labl$$label;
15964 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15965 if (cond == Assembler::EQ || cond == Assembler::LS)
15966 __ cbz($op1$$Register, *L);
15967 else
15968 __ cbnz($op1$$Register, *L);
15969 %}
15970 ins_pipe(pipe_cmp_branch);
15971 %}
15972
15973 // Test bit and Branch
15974
15975 // Patterns for short (< 32KiB) variants
15976 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15977 match(If cmp (CmpL op1 op2));
15978 effect(USE labl);
15979
15980 ins_cost(BRANCH_COST);
15981 format %{ "cb$cmp $op1, $labl # long" %}
15982 ins_encode %{
15983 Label* L = $labl$$label;
15984 Assembler::Condition cond =
15985 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15986 __ tbr(cond, $op1$$Register, 63, *L);
15987 %}
15988 ins_pipe(pipe_cmp_branch);
15989 ins_short_branch(1);
15990 %}
15991
15992 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15993 match(If cmp (CmpI op1 op2));
15994 effect(USE labl);
15995
15996 ins_cost(BRANCH_COST);
15997 format %{ "cb$cmp $op1, $labl # int" %}
15998 ins_encode %{
15999 Label* L = $labl$$label;
16000 Assembler::Condition cond =
16001 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16002 __ tbr(cond, $op1$$Register, 31, *L);
16003 %}
16004 ins_pipe(pipe_cmp_branch);
16005 ins_short_branch(1);
16006 %}
16007
16008 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16009 match(If cmp (CmpL (AndL op1 op2) op3));
16010 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16011 effect(USE labl);
16012
16013 ins_cost(BRANCH_COST);
16014 format %{ "tb$cmp $op1, $op2, $labl" %}
16015 ins_encode %{
16016 Label* L = $labl$$label;
16017 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16018 int bit = exact_log2_long($op2$$constant);
16019 __ tbr(cond, $op1$$Register, bit, *L);
16020 %}
16021 ins_pipe(pipe_cmp_branch);
16022 ins_short_branch(1);
16023 %}
16024
16025 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16026 match(If cmp (CmpI (AndI op1 op2) op3));
16027 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16028 effect(USE labl);
16029
16030 ins_cost(BRANCH_COST);
16031 format %{ "tb$cmp $op1, $op2, $labl" %}
16032 ins_encode %{
16033 Label* L = $labl$$label;
16034 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16035 int bit = exact_log2((juint)$op2$$constant);
16036 __ tbr(cond, $op1$$Register, bit, *L);
16037 %}
16038 ins_pipe(pipe_cmp_branch);
16039 ins_short_branch(1);
16040 %}
16041
16042 // And far variants
16043 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16044 match(If cmp (CmpL op1 op2));
16045 effect(USE labl);
16046
16047 ins_cost(BRANCH_COST);
16048 format %{ "cb$cmp $op1, $labl # long" %}
16049 ins_encode %{
16050 Label* L = $labl$$label;
16051 Assembler::Condition cond =
16052 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16053 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16054 %}
16055 ins_pipe(pipe_cmp_branch);
16056 %}
16057
16058 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16059 match(If cmp (CmpI op1 op2));
16060 effect(USE labl);
16061
16062 ins_cost(BRANCH_COST);
16063 format %{ "cb$cmp $op1, $labl # int" %}
16064 ins_encode %{
16065 Label* L = $labl$$label;
16066 Assembler::Condition cond =
16067 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16068 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16069 %}
16070 ins_pipe(pipe_cmp_branch);
16071 %}
16072
16073 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16074 match(If cmp (CmpL (AndL op1 op2) op3));
16075 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16076 effect(USE labl);
16077
16078 ins_cost(BRANCH_COST);
16079 format %{ "tb$cmp $op1, $op2, $labl" %}
16080 ins_encode %{
16081 Label* L = $labl$$label;
16082 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16083 int bit = exact_log2_long($op2$$constant);
16084 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16085 %}
16086 ins_pipe(pipe_cmp_branch);
16087 %}
16088
16089 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16090 match(If cmp (CmpI (AndI op1 op2) op3));
16091 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16092 effect(USE labl);
16093
16094 ins_cost(BRANCH_COST);
16095 format %{ "tb$cmp $op1, $op2, $labl" %}
16096 ins_encode %{
16097 Label* L = $labl$$label;
16098 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16099 int bit = exact_log2((juint)$op2$$constant);
16100 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16101 %}
16102 ins_pipe(pipe_cmp_branch);
16103 %}
16104
16105 // Test bits
16106
16107 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16108 match(Set cr (CmpL (AndL op1 op2) op3));
16109 predicate(Assembler::operand_valid_for_logical_immediate
16110 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16111
16112 ins_cost(INSN_COST);
16113 format %{ "tst $op1, $op2 # long" %}
16114 ins_encode %{
16115 __ tst($op1$$Register, $op2$$constant);
16116 %}
16117 ins_pipe(ialu_reg_reg);
16118 %}
16119
16120 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16121 match(Set cr (CmpI (AndI op1 op2) op3));
16122 predicate(Assembler::operand_valid_for_logical_immediate
16123 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16124
16125 ins_cost(INSN_COST);
16126 format %{ "tst $op1, $op2 # int" %}
16127 ins_encode %{
16128 __ tstw($op1$$Register, $op2$$constant);
16129 %}
16130 ins_pipe(ialu_reg_reg);
16131 %}
16132
16133 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16134 match(Set cr (CmpL (AndL op1 op2) op3));
16135
16136 ins_cost(INSN_COST);
16137 format %{ "tst $op1, $op2 # long" %}
16138 ins_encode %{
16139 __ tst($op1$$Register, $op2$$Register);
16140 %}
16141 ins_pipe(ialu_reg_reg);
16142 %}
16143
16144 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16145 match(Set cr (CmpI (AndI op1 op2) op3));
16146
16147 ins_cost(INSN_COST);
16148 format %{ "tstw $op1, $op2 # int" %}
16149 ins_encode %{
16150 __ tstw($op1$$Register, $op2$$Register);
16151 %}
16152 ins_pipe(ialu_reg_reg);
16153 %}
16154
16155
16156 // Conditional Far Branch
16157 // Conditional Far Branch Unsigned
16158 // TODO: fixme
16159
16160 // counted loop end branch near
16161 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16162 %{
16163 match(CountedLoopEnd cmp cr);
16164
16165 effect(USE lbl);
16166
16167 ins_cost(BRANCH_COST);
16168 // short variant.
16169 // ins_short_branch(1);
16170 format %{ "b$cmp $lbl \t// counted loop end" %}
16171
16172 ins_encode(aarch64_enc_br_con(cmp, lbl));
16173
16174 ins_pipe(pipe_branch);
16175 %}
16176
16177 // counted loop end branch near Unsigned
16178 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16179 %{
16180 match(CountedLoopEnd cmp cr);
16181
16182 effect(USE lbl);
16183
16184 ins_cost(BRANCH_COST);
16185 // short variant.
16186 // ins_short_branch(1);
16187 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16188
16189 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16190
16191 ins_pipe(pipe_branch);
16192 %}
16193
16194 // counted loop end branch far
16195 // counted loop end branch far unsigned
16196 // TODO: fixme
16197
16198 // ============================================================================
16199 // inlined locking and unlocking
16200
16201 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16202 %{
16203 match(Set cr (FastLock object box));
16204 effect(TEMP tmp, TEMP tmp2);
16205
16206 // TODO
16207 // identify correct cost
16208 ins_cost(5 * INSN_COST);
16209 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16210
16211 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16212
16213 ins_pipe(pipe_serial);
16214 %}
16215
16216 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16217 %{
16218 match(Set cr (FastUnlock object box));
16219 effect(TEMP tmp, TEMP tmp2);
16220
16221 ins_cost(5 * INSN_COST);
16222 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16223
16224 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16225
16226 ins_pipe(pipe_serial);
16227 %}
16228
16229
16230 // ============================================================================
16231 // Safepoint Instructions
16232
16233 // TODO
16234 // provide a near and far version of this code
16235
16236 instruct safePoint(rFlagsReg cr, iRegP poll)
16237 %{
16238 match(SafePoint poll);
16239 effect(KILL cr);
16240
16241 format %{
16242 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16243 %}
16244 ins_encode %{
16245 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16246 %}
16247 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16248 %}
16249
16250
16251 // ============================================================================
16252 // Procedure Call/Return Instructions
16253
16254 // Call Java Static Instruction
16255
16256 instruct CallStaticJavaDirect(method meth)
16257 %{
16258 match(CallStaticJava);
16259
16260 effect(USE meth);
16261
16262 ins_cost(CALL_COST);
16263
16264 format %{ "call,static $meth \t// ==> " %}
16265
16266 ins_encode(aarch64_enc_java_static_call(meth),
16267 aarch64_enc_call_epilog);
16268
16269 ins_pipe(pipe_class_call);
16270 %}
16271
16272 // TO HERE
16273
16274 // Call Java Dynamic Instruction
16275 instruct CallDynamicJavaDirect(method meth)
16276 %{
16277 match(CallDynamicJava);
16278
16279 effect(USE meth);
16280
16281 ins_cost(CALL_COST);
16282
16283 format %{ "CALL,dynamic $meth \t// ==> " %}
16284
16285 ins_encode(aarch64_enc_java_dynamic_call(meth),
16286 aarch64_enc_call_epilog);
16287
16288 ins_pipe(pipe_class_call);
16289 %}
16290
16291 // Call Runtime Instruction
16292
16293 instruct CallRuntimeDirect(method meth)
16294 %{
16295 match(CallRuntime);
16296
16297 effect(USE meth);
16298
16299 ins_cost(CALL_COST);
16300
16301 format %{ "CALL, runtime $meth" %}
16302
16303 ins_encode( aarch64_enc_java_to_runtime(meth) );
16304
16305 ins_pipe(pipe_class_call);
16306 %}
16307
16308 // Call Runtime Instruction
16309
16310 instruct CallLeafDirect(method meth)
16311 %{
16312 match(CallLeaf);
16313
16314 effect(USE meth);
16315
16316 ins_cost(CALL_COST);
16317
16318 format %{ "CALL, runtime leaf $meth" %}
16319
16320 ins_encode( aarch64_enc_java_to_runtime(meth) );
16321
16322 ins_pipe(pipe_class_call);
16323 %}
16324
16325 // Call Runtime Instruction
16326
16327 // entry point is null, target holds the address to call
16328 instruct CallLeafNoFPIndirect(iRegP target)
16329 %{
16330 predicate(n->as_Call()->entry_point() == NULL);
16331
16332 match(CallLeafNoFP target);
16333
16334 ins_cost(CALL_COST);
16335
16336 format %{ "CALL, runtime leaf nofp indirect $target" %}
16337
16338 ins_encode %{
16339 __ blr($target$$Register);
16340 %}
16341
16342 ins_pipe(pipe_class_call);
16343 %}
16344
16345 instruct CallLeafNoFPDirect(method meth)
16346 %{
16347 predicate(n->as_Call()->entry_point() != NULL);
16348
16349 match(CallLeafNoFP);
16350
16351 effect(USE meth);
16352
16353 ins_cost(CALL_COST);
16354
16355 format %{ "CALL, runtime leaf nofp $meth" %}
16356
16357 ins_encode( aarch64_enc_java_to_runtime(meth) );
16358
16359 ins_pipe(pipe_class_call);
16360 %}
16361
16362 // Tail Call; Jump from runtime stub to Java code.
16363 // Also known as an 'interprocedural jump'.
16364 // Target of jump will eventually return to caller.
16365 // TailJump below removes the return address.
16366 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16367 %{
16368 match(TailCall jump_target method_ptr);
16369
16370 ins_cost(CALL_COST);
16371
16372 format %{ "br $jump_target\t# $method_ptr holds method" %}
16373
16374 ins_encode(aarch64_enc_tail_call(jump_target));
16375
16376 ins_pipe(pipe_class_call);
16377 %}
16378
16379 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16380 %{
16381 match(TailJump jump_target ex_oop);
16382
16383 ins_cost(CALL_COST);
16384
16385 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16386
16387 ins_encode(aarch64_enc_tail_jmp(jump_target));
16388
16389 ins_pipe(pipe_class_call);
16390 %}
16391
16392 // Create exception oop: created by stack-crawling runtime code.
16393 // Created exception is now available to this handler, and is setup
16394 // just prior to jumping to this handler. No code emitted.
16395 // TODO check
16396 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16397 instruct CreateException(iRegP_R0 ex_oop)
16398 %{
16399 match(Set ex_oop (CreateEx));
16400
16401 format %{ " -- \t// exception oop; no code emitted" %}
16402
16403 size(0);
16404
16405 ins_encode( /*empty*/ );
16406
16407 ins_pipe(pipe_class_empty);
16408 %}
16409
16410 // Rethrow exception: The exception oop will come in the first
16411 // argument position. Then JUMP (not call) to the rethrow stub code.
16412 instruct RethrowException() %{
16413 match(Rethrow);
16414 ins_cost(CALL_COST);
16415
16416 format %{ "b rethrow_stub" %}
16417
16418 ins_encode( aarch64_enc_rethrow() );
16419
16420 ins_pipe(pipe_class_call);
16421 %}
16422
16423
16424 // Return Instruction
16425 // epilog node loads ret address into lr as part of frame pop
16426 instruct Ret()
16427 %{
16428 match(Return);
16429
16430 format %{ "ret\t// return register" %}
16431
16432 ins_encode( aarch64_enc_ret() );
16433
16434 ins_pipe(pipe_branch);
16435 %}
16436
16437 // Die now.
16438 instruct ShouldNotReachHere() %{
16439 match(Halt);
16440
16441 ins_cost(CALL_COST);
16442 format %{ "ShouldNotReachHere" %}
16443
16444 ins_encode %{
16445 if (is_reachable()) {
16446 __ stop(_halt_reason);
16447 }
16448 %}
16449
16450 ins_pipe(pipe_class_default);
16451 %}
16452
16453 // ============================================================================
16454 // Partial Subtype Check
16455 //
16456 // superklass array for an instance of the superklass. Set a hidden
16457 // internal cache on a hit (cache is checked with exposed code in
16458 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16459 // encoding ALSO sets flags.
16460
16461 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16462 %{
16463 match(Set result (PartialSubtypeCheck sub super));
16464 effect(KILL cr, KILL temp);
16465
16466 ins_cost(1100); // slightly larger than the next version
16467 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16468
16469 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16470
16471 opcode(0x1); // Force zero of result reg on hit
16472
16473 ins_pipe(pipe_class_memory);
16474 %}
16475
16476 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16477 %{
16478 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16479 effect(KILL temp, KILL result);
16480
16481 ins_cost(1100); // slightly larger than the next version
16482 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16483
16484 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16485
16486 opcode(0x0); // Don't zero result reg on hit
16487
16488 ins_pipe(pipe_class_memory);
16489 %}
16490
16491 // Intrisics for String.compareTo()
16492
16493 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16494 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16495 %{
16496 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16497 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16498 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16499
16500 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16501 ins_encode %{
16502 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16503 __ string_compare($str1$$Register, $str2$$Register,
16504 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16505 $tmp1$$Register, $tmp2$$Register,
16506 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16507 %}
16508 ins_pipe(pipe_class_memory);
16509 %}
16510
16511 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16512 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16513 %{
16514 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16515 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16516 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16517
16518 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16519 ins_encode %{
16520 __ string_compare($str1$$Register, $str2$$Register,
16521 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16522 $tmp1$$Register, $tmp2$$Register,
16523 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16524 %}
16525 ins_pipe(pipe_class_memory);
16526 %}
16527
16528 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16529 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16530 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16531 %{
16532 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16533 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16534 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16535 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16536
16537 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16538 ins_encode %{
16539 __ string_compare($str1$$Register, $str2$$Register,
16540 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16541 $tmp1$$Register, $tmp2$$Register,
16542 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16543 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16544 %}
16545 ins_pipe(pipe_class_memory);
16546 %}
16547
16548 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16549 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16550 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16551 %{
16552 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16553 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16554 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16555 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16556
16557 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16558 ins_encode %{
16559 __ string_compare($str1$$Register, $str2$$Register,
16560 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16561 $tmp1$$Register, $tmp2$$Register,
16562 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16563 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16564 %}
16565 ins_pipe(pipe_class_memory);
16566 %}
16567
16568 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16569 // these string_compare variants as NEON register type for convenience so that the prototype of
16570 // string_compare can be shared with all variants.
16571
16572 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16573 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16574 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16575 pRegGov_P1 pgtmp2, rFlagsReg cr)
16576 %{
16577 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16578 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16579 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16580 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16581
16582 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16583 ins_encode %{
16584 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16585 __ string_compare($str1$$Register, $str2$$Register,
16586 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16587 $tmp1$$Register, $tmp2$$Register,
16588 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16589 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16590 StrIntrinsicNode::LL);
16591 %}
16592 ins_pipe(pipe_class_memory);
16593 %}
16594
16595 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16596 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16597 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16598 pRegGov_P1 pgtmp2, rFlagsReg cr)
16599 %{
16600 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16601 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16602 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16603 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16604
16605 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16606 ins_encode %{
16607 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16608 __ string_compare($str1$$Register, $str2$$Register,
16609 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16610 $tmp1$$Register, $tmp2$$Register,
16611 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16612 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16613 StrIntrinsicNode::LU);
16614 %}
16615 ins_pipe(pipe_class_memory);
16616 %}
16617
16618 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16619 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16620 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16621 pRegGov_P1 pgtmp2, rFlagsReg cr)
16622 %{
16623 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16624 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16625 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16626 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16627
16628 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16629 ins_encode %{
16630 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16631 __ string_compare($str1$$Register, $str2$$Register,
16632 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16633 $tmp1$$Register, $tmp2$$Register,
16634 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16635 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16636 StrIntrinsicNode::UL);
16637 %}
16638 ins_pipe(pipe_class_memory);
16639 %}
16640
16641 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16642 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16643 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16644 pRegGov_P1 pgtmp2, rFlagsReg cr)
16645 %{
16646 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16647 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16648 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16649 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16650
16651 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16652 ins_encode %{
16653 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16654 __ string_compare($str1$$Register, $str2$$Register,
16655 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16656 $tmp1$$Register, $tmp2$$Register,
16657 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16658 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16659 StrIntrinsicNode::UU);
16660 %}
16661 ins_pipe(pipe_class_memory);
16662 %}
16663
16664 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16665 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16666 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16667 %{
16668 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16669 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16670 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16671 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16672 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16673
16674 ins_encode %{
16675 __ string_indexof($str1$$Register, $str2$$Register,
16676 $cnt1$$Register, $cnt2$$Register,
16677 $tmp1$$Register, $tmp2$$Register,
16678 $tmp3$$Register, $tmp4$$Register,
16679 $tmp5$$Register, $tmp6$$Register,
16680 -1, $result$$Register, StrIntrinsicNode::UU);
16681 %}
16682 ins_pipe(pipe_class_memory);
16683 %}
16684
16685 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16686 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16687 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16688 %{
16689 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16690 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16691 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16692 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16693 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16694
16695 ins_encode %{
16696 __ string_indexof($str1$$Register, $str2$$Register,
16697 $cnt1$$Register, $cnt2$$Register,
16698 $tmp1$$Register, $tmp2$$Register,
16699 $tmp3$$Register, $tmp4$$Register,
16700 $tmp5$$Register, $tmp6$$Register,
16701 -1, $result$$Register, StrIntrinsicNode::LL);
16702 %}
16703 ins_pipe(pipe_class_memory);
16704 %}
16705
16706 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16707 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16708 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16709 %{
16710 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16711 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16712 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16713 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16714 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16715
16716 ins_encode %{
16717 __ string_indexof($str1$$Register, $str2$$Register,
16718 $cnt1$$Register, $cnt2$$Register,
16719 $tmp1$$Register, $tmp2$$Register,
16720 $tmp3$$Register, $tmp4$$Register,
16721 $tmp5$$Register, $tmp6$$Register,
16722 -1, $result$$Register, StrIntrinsicNode::UL);
16723 %}
16724 ins_pipe(pipe_class_memory);
16725 %}
16726
16727 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16728 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16729 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16730 %{
16731 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16732 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16733 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16734 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16735 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16736
16737 ins_encode %{
16738 int icnt2 = (int)$int_cnt2$$constant;
16739 __ string_indexof($str1$$Register, $str2$$Register,
16740 $cnt1$$Register, zr,
16741 $tmp1$$Register, $tmp2$$Register,
16742 $tmp3$$Register, $tmp4$$Register, zr, zr,
16743 icnt2, $result$$Register, StrIntrinsicNode::UU);
16744 %}
16745 ins_pipe(pipe_class_memory);
16746 %}
16747
16748 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16749 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16750 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16751 %{
16752 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16753 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16754 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16755 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16756 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16757
16758 ins_encode %{
16759 int icnt2 = (int)$int_cnt2$$constant;
16760 __ string_indexof($str1$$Register, $str2$$Register,
16761 $cnt1$$Register, zr,
16762 $tmp1$$Register, $tmp2$$Register,
16763 $tmp3$$Register, $tmp4$$Register, zr, zr,
16764 icnt2, $result$$Register, StrIntrinsicNode::LL);
16765 %}
16766 ins_pipe(pipe_class_memory);
16767 %}
16768
16769 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16770 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16771 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16772 %{
16773 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16774 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16775 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16776 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16777 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16778
16779 ins_encode %{
16780 int icnt2 = (int)$int_cnt2$$constant;
16781 __ string_indexof($str1$$Register, $str2$$Register,
16782 $cnt1$$Register, zr,
16783 $tmp1$$Register, $tmp2$$Register,
16784 $tmp3$$Register, $tmp4$$Register, zr, zr,
16785 icnt2, $result$$Register, StrIntrinsicNode::UL);
16786 %}
16787 ins_pipe(pipe_class_memory);
16788 %}
16789
16790 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16791 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16792 iRegINoSp tmp3, rFlagsReg cr)
16793 %{
16794 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16795 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16796 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16797 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16798
16799 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16800
16801 ins_encode %{
16802 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16803 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16804 $tmp3$$Register);
16805 %}
16806 ins_pipe(pipe_class_memory);
16807 %}
16808
16809 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16810 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16811 iRegINoSp tmp3, rFlagsReg cr)
16812 %{
16813 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16814 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16815 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16816 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16817
16818 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16819
16820 ins_encode %{
16821 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16822 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16823 $tmp3$$Register);
16824 %}
16825 ins_pipe(pipe_class_memory);
16826 %}
16827
16828 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16829 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16830 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16831 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16832 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16833 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16834 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16835 ins_encode %{
16836 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16837 $result$$Register, $ztmp1$$FloatRegister,
16838 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16839 $ptmp$$PRegister, true /* isL */);
16840 %}
16841 ins_pipe(pipe_class_memory);
16842 %}
16843
16844 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16845 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16846 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16847 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16848 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16849 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16850 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16851 ins_encode %{
16852 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16853 $result$$Register, $ztmp1$$FloatRegister,
16854 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16855 $ptmp$$PRegister, false /* isL */);
16856 %}
16857 ins_pipe(pipe_class_memory);
16858 %}
16859
16860 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16861 iRegI_R0 result, rFlagsReg cr)
16862 %{
16863 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16864 match(Set result (StrEquals (Binary str1 str2) cnt));
16865 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16866
16867 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16868 ins_encode %{
16869 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16870 __ string_equals($str1$$Register, $str2$$Register,
16871 $result$$Register, $cnt$$Register, 1);
16872 %}
16873 ins_pipe(pipe_class_memory);
16874 %}
16875
16876 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16877 iRegI_R0 result, rFlagsReg cr)
16878 %{
16879 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16880 match(Set result (StrEquals (Binary str1 str2) cnt));
16881 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16882
16883 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16884 ins_encode %{
16885 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16886 __ string_equals($str1$$Register, $str2$$Register,
16887 $result$$Register, $cnt$$Register, 2);
16888 %}
16889 ins_pipe(pipe_class_memory);
16890 %}
16891
16892 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16893 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16894 iRegP_R10 tmp, rFlagsReg cr)
16895 %{
16896 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16897 match(Set result (AryEq ary1 ary2));
16898 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16899
16900 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
16901 ins_encode %{
16902 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16903 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16904 $result$$Register, $tmp$$Register, 1);
16905 if (tpc == NULL) {
16906 ciEnv::current()->record_failure("CodeCache is full");
16907 return;
16908 }
16909 %}
16910 ins_pipe(pipe_class_memory);
16911 %}
16912
16913 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16914 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16915 iRegP_R10 tmp, rFlagsReg cr)
16916 %{
16917 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16918 match(Set result (AryEq ary1 ary2));
16919 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16920
16921 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
16922 ins_encode %{
16923 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16924 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16925 $result$$Register, $tmp$$Register, 2);
16926 if (tpc == NULL) {
16927 ciEnv::current()->record_failure("CodeCache is full");
16928 return;
16929 }
16930 %}
16931 ins_pipe(pipe_class_memory);
16932 %}
16933
16934 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16935 %{
16936 match(Set result (CountPositives ary1 len));
16937 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16938 format %{ "count positives byte[] $ary1,$len -> $result" %}
16939 ins_encode %{
16940 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16941 if (tpc == NULL) {
16942 ciEnv::current()->record_failure("CodeCache is full");
16943 return;
16944 }
16945 %}
16946 ins_pipe( pipe_slow );
16947 %}
16948
16949 // fast char[] to byte[] compression
16950 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16951 vRegD_V0 tmp1, vRegD_V1 tmp2,
16952 vRegD_V2 tmp3, vRegD_V3 tmp4,
16953 iRegI_R0 result, rFlagsReg cr)
16954 %{
16955 match(Set result (StrCompressedCopy src (Binary dst len)));
16956 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
16957 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16958
16959 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
16960 ins_encode %{
16961 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16962 $result$$Register,
16963 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16964 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
16965 %}
16966 ins_pipe(pipe_slow);
16967 %}
16968
16969 // fast byte[] to char[] inflation
16970 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16971 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16972 %{
16973 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16974 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16975
16976 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
16977 ins_encode %{
16978 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16979 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16980 $tmp3$$FloatRegister, $tmp4$$Register);
16981 if (tpc == NULL) {
16982 ciEnv::current()->record_failure("CodeCache is full");
16983 return;
16984 }
16985 %}
16986 ins_pipe(pipe_class_memory);
16987 %}
16988
16989 // encode char[] to byte[] in ISO_8859_1
16990 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16991 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16992 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16993 iRegI_R0 result, rFlagsReg cr)
16994 %{
16995 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16996 match(Set result (EncodeISOArray src (Binary dst len)));
16997 effect(USE_KILL src, USE_KILL dst, USE len,
16998 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16999
17000 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17001 ins_encode %{
17002 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17003 $result$$Register, false,
17004 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17005 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17006 %}
17007 ins_pipe(pipe_class_memory);
17008 %}
17009
17010 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17011 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17012 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17013 iRegI_R0 result, rFlagsReg cr)
17014 %{
17015 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17016 match(Set result (EncodeISOArray src (Binary dst len)));
17017 effect(USE_KILL src, USE_KILL dst, USE len,
17018 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17019
17020 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17021 ins_encode %{
17022 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17023 $result$$Register, true,
17024 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17025 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17026 %}
17027 ins_pipe(pipe_class_memory);
17028 %}
17029
17030 // ============================================================================
17031 // This name is KNOWN by the ADLC and cannot be changed.
17032 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17033 // for this guy.
17034 instruct tlsLoadP(thread_RegP dst)
17035 %{
17036 match(Set dst (ThreadLocal));
17037
17038 ins_cost(0);
17039
17040 format %{ " -- \t// $dst=Thread::current(), empty" %}
17041
17042 size(0);
17043
17044 ins_encode( /*empty*/ );
17045
17046 ins_pipe(pipe_class_empty);
17047 %}
17048
17049 //----------PEEPHOLE RULES-----------------------------------------------------
17050 // These must follow all instruction definitions as they use the names
17051 // defined in the instructions definitions.
17052 //
17053 // peepmatch ( root_instr_name [preceding_instruction]* );
17054 //
17055 // peepconstraint %{
17056 // (instruction_number.operand_name relational_op instruction_number.operand_name
17057 // [, ...] );
17058 // // instruction numbers are zero-based using left to right order in peepmatch
17059 //
17060 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17061 // // provide an instruction_number.operand_name for each operand that appears
17062 // // in the replacement instruction's match rule
17063 //
17064 // ---------VM FLAGS---------------------------------------------------------
17065 //
17066 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17067 //
17068 // Each peephole rule is given an identifying number starting with zero and
17069 // increasing by one in the order seen by the parser. An individual peephole
17070 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17071 // on the command-line.
17072 //
17073 // ---------CURRENT LIMITATIONS----------------------------------------------
17074 //
17075 // Only match adjacent instructions in same basic block
17076 // Only equality constraints
17077 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17078 // Only one replacement instruction
17079 //
17080 // ---------EXAMPLE----------------------------------------------------------
17081 //
17082 // // pertinent parts of existing instructions in architecture description
17083 // instruct movI(iRegINoSp dst, iRegI src)
17084 // %{
17085 // match(Set dst (CopyI src));
17086 // %}
17087 //
17088 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17089 // %{
17090 // match(Set dst (AddI dst src));
17091 // effect(KILL cr);
17092 // %}
17093 //
17094 // // Change (inc mov) to lea
17095 // peephole %{
17096 // // increment preceded by register-register move
17097 // peepmatch ( incI_iReg movI );
17098 // // require that the destination register of the increment
17099 // // match the destination register of the move
17100 // peepconstraint ( 0.dst == 1.dst );
17101 // // construct a replacement instruction that sets
17102 // // the destination to ( move's source register + one )
17103 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17104 // %}
17105 //
17106
17107 // Implementation no longer uses movX instructions since
17108 // machine-independent system no longer uses CopyX nodes.
17109 //
17110 // peephole
17111 // %{
17112 // peepmatch (incI_iReg movI);
17113 // peepconstraint (0.dst == 1.dst);
17114 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17115 // %}
17116
17117 // peephole
17118 // %{
17119 // peepmatch (decI_iReg movI);
17120 // peepconstraint (0.dst == 1.dst);
17121 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17122 // %}
17123
17124 // peephole
17125 // %{
17126 // peepmatch (addI_iReg_imm movI);
17127 // peepconstraint (0.dst == 1.dst);
17128 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17129 // %}
17130
17131 // peephole
17132 // %{
17133 // peepmatch (incL_iReg movL);
17134 // peepconstraint (0.dst == 1.dst);
17135 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17136 // %}
17137
17138 // peephole
17139 // %{
17140 // peepmatch (decL_iReg movL);
17141 // peepconstraint (0.dst == 1.dst);
17142 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17143 // %}
17144
17145 // peephole
17146 // %{
17147 // peepmatch (addL_iReg_imm movL);
17148 // peepconstraint (0.dst == 1.dst);
17149 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17150 // %}
17151
17152 // peephole
17153 // %{
17154 // peepmatch (addP_iReg_imm movP);
17155 // peepconstraint (0.dst == 1.dst);
17156 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17157 // %}
17158
17159 // // Change load of spilled value to only a spill
17160 // instruct storeI(memory mem, iRegI src)
17161 // %{
17162 // match(Set mem (StoreI mem src));
17163 // %}
17164 //
17165 // instruct loadI(iRegINoSp dst, memory mem)
17166 // %{
17167 // match(Set dst (LoadI mem));
17168 // %}
17169 //
17170
17171 //----------SMARTSPILL RULES---------------------------------------------------
17172 // These must follow all instruction definitions as they use the names
17173 // defined in the instructions definitions.
17174
17175 // Local Variables:
17176 // mode: c++
17177 // End: