1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // lea(rscratch1, RuntimeAddress(addr)
1652 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2311 return new TypeVectMask(elemTy, length);
2312 }
2313
2314 // Vector calling convention not yet implemented.
2315 bool Matcher::supports_vector_calling_convention(void) {
2316 return false;
2317 }
2318
2319 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2320 Unimplemented();
2321 return OptoRegPair(0, 0);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m) ||
2624 is_encode_and_store_pattern(n, m)) {
2625 mstack.push(m, Visit);
2626 return true;
2627 }
2628 return false;
2629 }
2630
2631 // Should the Matcher clone shifts on addressing modes, expecting them
2632 // to be subsumed into complex addressing expressions or compute them
2633 // into registers?
2634 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2635 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2636 return true;
2637 }
2638
2639 Node *off = m->in(AddPNode::Offset);
2640 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2641 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2642 // Are there other uses besides address expressions?
2643 !is_visited(off)) {
2644 address_visited.set(off->_idx); // Flag as address_visited
2645 mstack.push(off->in(2), Visit);
2646 Node *conv = off->in(1);
2647 if (conv->Opcode() == Op_ConvI2L &&
2648 // Are there other uses besides address expressions?
2649 !is_visited(conv)) {
2650 address_visited.set(conv->_idx); // Flag as address_visited
2651 mstack.push(conv->in(1), Pre_Visit);
2652 } else {
2653 mstack.push(conv, Pre_Visit);
2654 }
2655 address_visited.test_set(m->_idx); // Flag as address_visited
2656 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2657 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2658 return true;
2659 } else if (off->Opcode() == Op_ConvI2L &&
2660 // Are there other uses besides address expressions?
2661 !is_visited(off)) {
2662 address_visited.test_set(m->_idx); // Flag as address_visited
2663 address_visited.set(off->_idx); // Flag as address_visited
2664 mstack.push(off->in(1), Pre_Visit);
2665 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2666 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2667 return true;
2668 }
2669 return false;
2670 }
2671
2672 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2673 { \
2674 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2675 guarantee(DISP == 0, "mode not permitted for volatile"); \
2676 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2677 __ INSN(REG, as_Register(BASE)); \
2678 }
2679
2680
2681 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2682 {
2683 Address::extend scale;
2684
2685 // Hooboy, this is fugly. We need a way to communicate to the
2686 // encoder that the index needs to be sign extended, so we have to
2687 // enumerate all the cases.
2688 switch (opcode) {
2689 case INDINDEXSCALEDI2L:
2690 case INDINDEXSCALEDI2LN:
2691 case INDINDEXI2L:
2692 case INDINDEXI2LN:
2693 scale = Address::sxtw(size);
2694 break;
2695 default:
2696 scale = Address::lsl(size);
2697 }
2698
2699 if (index == -1) {
2700 return Address(base, disp);
2701 } else {
2702 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2703 return Address(base, as_Register(index), scale);
2704 }
2705 }
2706
2707
2708 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2709 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2710 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2711 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2712 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2713
2714 // Used for all non-volatile memory accesses. The use of
2715 // $mem->opcode() to discover whether this pattern uses sign-extended
2716 // offsets is something of a kludge.
2717 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2718 Register reg, int opcode,
2719 Register base, int index, int scale, int disp,
2720 int size_in_memory)
2721 {
2722 Address addr = mem2address(opcode, base, index, scale, disp);
2723 if (addr.getMode() == Address::base_plus_offset) {
2724 /* Fix up any out-of-range offsets. */
2725 assert_different_registers(rscratch1, base);
2726 assert_different_registers(rscratch1, reg);
2727 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2728 }
2729 (masm->*insn)(reg, addr);
2730 }
2731
2732 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2733 FloatRegister reg, int opcode,
2734 Register base, int index, int size, int disp,
2735 int size_in_memory)
2736 {
2737 Address::extend scale;
2738
2739 switch (opcode) {
2740 case INDINDEXSCALEDI2L:
2741 case INDINDEXSCALEDI2LN:
2742 scale = Address::sxtw(size);
2743 break;
2744 default:
2745 scale = Address::lsl(size);
2746 }
2747
2748 if (index == -1) {
2749 // Fix up any out-of-range offsets.
2750 assert_different_registers(rscratch1, base);
2751 Address addr = Address(base, disp);
2752 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2753 (masm->*insn)(reg, addr);
2754 } else {
2755 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2756 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2757 }
2758 }
2759
2760 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2761 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2762 int opcode, Register base, int index, int size, int disp)
2763 {
2764 if (index == -1) {
2765 (masm->*insn)(reg, T, Address(base, disp));
2766 } else {
2767 assert(disp == 0, "unsupported address mode");
2768 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2769 }
2770 }
2771
2772 %}
2773
2774
2775
2776 //----------ENCODING BLOCK-----------------------------------------------------
2777 // This block specifies the encoding classes used by the compiler to
2778 // output byte streams. Encoding classes are parameterized macros
2779 // used by Machine Instruction Nodes in order to generate the bit
2780 // encoding of the instruction. Operands specify their base encoding
2781 // interface with the interface keyword. There are currently
2782 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2783 // COND_INTER. REG_INTER causes an operand to generate a function
2784 // which returns its register number when queried. CONST_INTER causes
2785 // an operand to generate a function which returns the value of the
2786 // constant when queried. MEMORY_INTER causes an operand to generate
2787 // four functions which return the Base Register, the Index Register,
2788 // the Scale Value, and the Offset Value of the operand when queried.
2789 // COND_INTER causes an operand to generate six functions which return
2790 // the encoding code (ie - encoding bits for the instruction)
2791 // associated with each basic boolean condition for a conditional
2792 // instruction.
2793 //
2794 // Instructions specify two basic values for encoding. Again, a
2795 // function is available to check if the constant displacement is an
2796 // oop. They use the ins_encode keyword to specify their encoding
2797 // classes (which must be a sequence of enc_class names, and their
2798 // parameters, specified in the encoding block), and they use the
2799 // opcode keyword to specify, in order, their primary, secondary, and
2800 // tertiary opcode. Only the opcode sections which a particular
2801 // instruction needs for encoding need to be specified.
2802 encode %{
2803 // Build emit functions for each basic byte or larger field in the
2804 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2805 // from C++ code in the enc_class source block. Emit functions will
2806 // live in the main source block for now. In future, we can
2807 // generalize this by adding a syntax that specifies the sizes of
2808 // fields in an order, so that the adlc can build the emit functions
2809 // automagically
2810
2811 // catch all for unimplemented encodings
2812 enc_class enc_unimplemented %{
2813 __ unimplemented("C2 catch all");
2814 %}
2815
2816 // BEGIN Non-volatile memory access
2817
2818 // This encoding class is generated automatically from ad_encode.m4.
2819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2820 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2821 Register dst_reg = as_Register($dst$$reg);
2822 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2823 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2824 %}
2825
2826 // This encoding class is generated automatically from ad_encode.m4.
2827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2828 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2829 Register dst_reg = as_Register($dst$$reg);
2830 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2831 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2832 %}
2833
2834 // This encoding class is generated automatically from ad_encode.m4.
2835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2836 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2837 Register dst_reg = as_Register($dst$$reg);
2838 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2839 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2840 %}
2841
2842 // This encoding class is generated automatically from ad_encode.m4.
2843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2844 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2845 Register dst_reg = as_Register($dst$$reg);
2846 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2847 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2848 %}
2849
2850 // This encoding class is generated automatically from ad_encode.m4.
2851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2852 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2853 Register dst_reg = as_Register($dst$$reg);
2854 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2855 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2856 %}
2857
2858 // This encoding class is generated automatically from ad_encode.m4.
2859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2860 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2861 Register dst_reg = as_Register($dst$$reg);
2862 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2863 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2864 %}
2865
2866 // This encoding class is generated automatically from ad_encode.m4.
2867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2868 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2869 Register dst_reg = as_Register($dst$$reg);
2870 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2872 %}
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2917 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2925 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2933 Register src_reg = as_Register($src$$reg);
2934 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_strb0(memory1 mem) %{
2941 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2948 Register src_reg = as_Register($src$$reg);
2949 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strh0(memory2 mem) %{
2956 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2963 Register src_reg = as_Register($src$$reg);
2964 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_strw0(memory4 mem) %{
2971 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2978 Register src_reg = as_Register($src$$reg);
2979 // we sometimes get asked to store the stack pointer into the
2980 // current thread -- we cannot do that directly on AArch64
2981 if (src_reg == r31_sp) {
2982 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2983 __ mov(rscratch2, sp);
2984 src_reg = rscratch2;
2985 }
2986 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_str0(memory8 mem) %{
2993 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3000 FloatRegister src_reg = as_FloatRegister($src$$reg);
3001 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3008 FloatRegister src_reg = as_FloatRegister($src$$reg);
3009 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3016 __ membar(Assembler::StoreStore);
3017 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3018 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3019 %}
3020
3021 // END Non-volatile memory access
3022
3023 // Vector loads and stores
3024 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3025 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3026 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3027 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3028 %}
3029
3030 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3031 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3032 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3033 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3034 %}
3035
3036 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3037 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3038 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3039 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3040 %}
3041
3042 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3043 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3044 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3045 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3046 %}
3047
3048 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3049 FloatRegister src_reg = as_FloatRegister($src$$reg);
3050 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3051 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3052 %}
3053
3054 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3055 FloatRegister src_reg = as_FloatRegister($src$$reg);
3056 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3057 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3058 %}
3059
3060 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3061 FloatRegister src_reg = as_FloatRegister($src$$reg);
3062 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3063 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3064 %}
3065
3066 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3069 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3070 %}
3071
3072 // volatile loads and stores
3073
3074 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3075 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3076 rscratch1, stlrb);
3077 %}
3078
3079 enc_class aarch64_enc_stlrb0(memory mem) %{
3080 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3081 rscratch1, stlrb);
3082 %}
3083
3084 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3085 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3086 rscratch1, stlrh);
3087 %}
3088
3089 enc_class aarch64_enc_stlrh0(memory mem) %{
3090 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3091 rscratch1, stlrh);
3092 %}
3093
3094 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3095 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3096 rscratch1, stlrw);
3097 %}
3098
3099 enc_class aarch64_enc_stlrw0(memory mem) %{
3100 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3101 rscratch1, stlrw);
3102 %}
3103
3104 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3105 Register dst_reg = as_Register($dst$$reg);
3106 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3107 rscratch1, ldarb);
3108 __ sxtbw(dst_reg, dst_reg);
3109 %}
3110
3111 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3112 Register dst_reg = as_Register($dst$$reg);
3113 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3114 rscratch1, ldarb);
3115 __ sxtb(dst_reg, dst_reg);
3116 %}
3117
3118 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3119 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, ldarb);
3121 %}
3122
3123 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3124 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, ldarb);
3126 %}
3127
3128 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3129 Register dst_reg = as_Register($dst$$reg);
3130 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarh);
3132 __ sxthw(dst_reg, dst_reg);
3133 %}
3134
3135 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3136 Register dst_reg = as_Register($dst$$reg);
3137 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, ldarh);
3139 __ sxth(dst_reg, dst_reg);
3140 %}
3141
3142 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3143 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3144 rscratch1, ldarh);
3145 %}
3146
3147 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarh);
3150 %}
3151
3152 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarw);
3155 %}
3156
3157 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3158 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, ldarw);
3160 %}
3161
3162 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3163 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, ldar);
3165 %}
3166
3167 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3168 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarw);
3170 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3171 %}
3172
3173 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3174 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldar);
3176 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3177 %}
3178
3179 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3180 Register src_reg = as_Register($src$$reg);
3181 // we sometimes get asked to store the stack pointer into the
3182 // current thread -- we cannot do that directly on AArch64
3183 if (src_reg == r31_sp) {
3184 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3185 __ mov(rscratch2, sp);
3186 src_reg = rscratch2;
3187 }
3188 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, stlr);
3190 %}
3191
3192 enc_class aarch64_enc_stlr0(memory mem) %{
3193 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, stlr);
3195 %}
3196
3197 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3198 {
3199 FloatRegister src_reg = as_FloatRegister($src$$reg);
3200 __ fmovs(rscratch2, src_reg);
3201 }
3202 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, stlrw);
3204 %}
3205
3206 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3207 {
3208 FloatRegister src_reg = as_FloatRegister($src$$reg);
3209 __ fmovd(rscratch2, src_reg);
3210 }
3211 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3212 rscratch1, stlr);
3213 %}
3214
3215 // synchronized read/update encodings
3216
3217 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3218 Register dst_reg = as_Register($dst$$reg);
3219 Register base = as_Register($mem$$base);
3220 int index = $mem$$index;
3221 int scale = $mem$$scale;
3222 int disp = $mem$$disp;
3223 if (index == -1) {
3224 if (disp != 0) {
3225 __ lea(rscratch1, Address(base, disp));
3226 __ ldaxr(dst_reg, rscratch1);
3227 } else {
3228 // TODO
3229 // should we ever get anything other than this case?
3230 __ ldaxr(dst_reg, base);
3231 }
3232 } else {
3233 Register index_reg = as_Register(index);
3234 if (disp == 0) {
3235 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3236 __ ldaxr(dst_reg, rscratch1);
3237 } else {
3238 __ lea(rscratch1, Address(base, disp));
3239 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3240 __ ldaxr(dst_reg, rscratch1);
3241 }
3242 }
3243 %}
3244
3245 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3246 Register src_reg = as_Register($src$$reg);
3247 Register base = as_Register($mem$$base);
3248 int index = $mem$$index;
3249 int scale = $mem$$scale;
3250 int disp = $mem$$disp;
3251 if (index == -1) {
3252 if (disp != 0) {
3253 __ lea(rscratch2, Address(base, disp));
3254 __ stlxr(rscratch1, src_reg, rscratch2);
3255 } else {
3256 // TODO
3257 // should we ever get anything other than this case?
3258 __ stlxr(rscratch1, src_reg, base);
3259 }
3260 } else {
3261 Register index_reg = as_Register(index);
3262 if (disp == 0) {
3263 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3264 __ stlxr(rscratch1, src_reg, rscratch2);
3265 } else {
3266 __ lea(rscratch2, Address(base, disp));
3267 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3268 __ stlxr(rscratch1, src_reg, rscratch2);
3269 }
3270 }
3271 __ cmpw(rscratch1, zr);
3272 %}
3273
3274 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3275 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3276 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3277 Assembler::xword, /*acquire*/ false, /*release*/ true,
3278 /*weak*/ false, noreg);
3279 %}
3280
3281 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3282 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3283 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3284 Assembler::word, /*acquire*/ false, /*release*/ true,
3285 /*weak*/ false, noreg);
3286 %}
3287
3288 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3289 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3290 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3291 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3292 /*weak*/ false, noreg);
3293 %}
3294
3295 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::byte, /*acquire*/ false, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302
3303 // The only difference between aarch64_enc_cmpxchg and
3304 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3305 // CompareAndSwap sequence to serve as a barrier on acquiring a
3306 // lock.
3307 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3308 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3309 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3310 Assembler::xword, /*acquire*/ true, /*release*/ true,
3311 /*weak*/ false, noreg);
3312 %}
3313
3314 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3315 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3316 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3317 Assembler::word, /*acquire*/ true, /*release*/ true,
3318 /*weak*/ false, noreg);
3319 %}
3320
3321 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3322 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3323 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3324 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3325 /*weak*/ false, noreg);
3326 %}
3327
3328 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3329 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3330 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3331 Assembler::byte, /*acquire*/ true, /*release*/ true,
3332 /*weak*/ false, noreg);
3333 %}
3334
3335 // auxiliary used for CompareAndSwapX to set result register
3336 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3337 Register res_reg = as_Register($res$$reg);
3338 __ cset(res_reg, Assembler::EQ);
3339 %}
3340
3341 // prefetch encodings
3342
3343 enc_class aarch64_enc_prefetchw(memory mem) %{
3344 Register base = as_Register($mem$$base);
3345 int index = $mem$$index;
3346 int scale = $mem$$scale;
3347 int disp = $mem$$disp;
3348 if (index == -1) {
3349 // Fix up any out-of-range offsets.
3350 assert_different_registers(rscratch1, base);
3351 Address addr = Address(base, disp);
3352 addr = __ legitimize_address(addr, 8, rscratch1);
3353 __ prfm(addr, PSTL1KEEP);
3354 } else {
3355 Register index_reg = as_Register(index);
3356 if (disp == 0) {
3357 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3358 } else {
3359 __ lea(rscratch1, Address(base, disp));
3360 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3361 }
3362 }
3363 %}
3364
3365 // mov encodings
3366
3367 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3368 uint32_t con = (uint32_t)$src$$constant;
3369 Register dst_reg = as_Register($dst$$reg);
3370 if (con == 0) {
3371 __ movw(dst_reg, zr);
3372 } else {
3373 __ movw(dst_reg, con);
3374 }
3375 %}
3376
3377 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3378 Register dst_reg = as_Register($dst$$reg);
3379 uint64_t con = (uint64_t)$src$$constant;
3380 if (con == 0) {
3381 __ mov(dst_reg, zr);
3382 } else {
3383 __ mov(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 address con = (address)$src$$constant;
3390 if (con == nullptr || con == (address)1) {
3391 ShouldNotReachHere();
3392 } else {
3393 relocInfo::relocType rtype = $src->constant_reloc();
3394 if (rtype == relocInfo::oop_type) {
3395 __ movoop(dst_reg, (jobject)con);
3396 } else if (rtype == relocInfo::metadata_type) {
3397 __ mov_metadata(dst_reg, (Metadata*)con);
3398 } else {
3399 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3400 if (! __ is_valid_AArch64_address(con) ||
3401 con < (address)(uintptr_t)os::vm_page_size()) {
3402 __ mov(dst_reg, con);
3403 } else {
3404 uint64_t offset;
3405 __ adrp(dst_reg, con, offset);
3406 __ add(dst_reg, dst_reg, offset);
3407 }
3408 }
3409 }
3410 %}
3411
3412 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3413 Register dst_reg = as_Register($dst$$reg);
3414 __ mov(dst_reg, zr);
3415 %}
3416
3417 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3418 Register dst_reg = as_Register($dst$$reg);
3419 __ mov(dst_reg, (uint64_t)1);
3420 %}
3421
3422 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3430 __ set_narrow_oop(dst_reg, (jobject)con);
3431 }
3432 %}
3433
3434 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, zr);
3437 %}
3438
3439 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 address con = (address)$src$$constant;
3442 if (con == nullptr) {
3443 ShouldNotReachHere();
3444 } else {
3445 relocInfo::relocType rtype = $src->constant_reloc();
3446 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3447 __ set_narrow_klass(dst_reg, (Klass *)con);
3448 }
3449 %}
3450
3451 // arithmetic encodings
3452
3453 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3454 Register dst_reg = as_Register($dst$$reg);
3455 Register src_reg = as_Register($src1$$reg);
3456 int32_t con = (int32_t)$src2$$constant;
3457 // add has primary == 0, subtract has primary == 1
3458 if ($primary) { con = -con; }
3459 if (con < 0) {
3460 __ subw(dst_reg, src_reg, -con);
3461 } else {
3462 __ addw(dst_reg, src_reg, con);
3463 }
3464 %}
3465
3466 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ sub(dst_reg, src_reg, -con);
3474 } else {
3475 __ add(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3491 %}
3492
3493 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3498 %}
3499
3500 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3505 %}
3506
3507 // compare instruction encodings
3508
3509 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3510 Register reg1 = as_Register($src1$$reg);
3511 Register reg2 = as_Register($src2$$reg);
3512 __ cmpw(reg1, reg2);
3513 %}
3514
3515 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3516 Register reg = as_Register($src1$$reg);
3517 int32_t val = $src2$$constant;
3518 if (val >= 0) {
3519 __ subsw(zr, reg, val);
3520 } else {
3521 __ addsw(zr, reg, -val);
3522 }
3523 %}
3524
3525 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3526 Register reg1 = as_Register($src1$$reg);
3527 uint32_t val = (uint32_t)$src2$$constant;
3528 __ movw(rscratch1, val);
3529 __ cmpw(reg1, rscratch1);
3530 %}
3531
3532 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3533 Register reg1 = as_Register($src1$$reg);
3534 Register reg2 = as_Register($src2$$reg);
3535 __ cmp(reg1, reg2);
3536 %}
3537
3538 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3539 Register reg = as_Register($src1$$reg);
3540 int64_t val = $src2$$constant;
3541 if (val >= 0) {
3542 __ subs(zr, reg, val);
3543 } else if (val != -val) {
3544 __ adds(zr, reg, -val);
3545 } else {
3546 // aargh, Long.MIN_VALUE is a special case
3547 __ orr(rscratch1, zr, (uint64_t)val);
3548 __ subs(zr, reg, rscratch1);
3549 }
3550 %}
3551
3552 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3553 Register reg1 = as_Register($src1$$reg);
3554 uint64_t val = (uint64_t)$src2$$constant;
3555 __ mov(rscratch1, val);
3556 __ cmp(reg1, rscratch1);
3557 %}
3558
3559 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3560 Register reg1 = as_Register($src1$$reg);
3561 Register reg2 = as_Register($src2$$reg);
3562 __ cmp(reg1, reg2);
3563 %}
3564
3565 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmpw(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_testp(iRegP src) %{
3572 Register reg = as_Register($src$$reg);
3573 __ cmp(reg, zr);
3574 %}
3575
3576 enc_class aarch64_enc_testn(iRegN src) %{
3577 Register reg = as_Register($src$$reg);
3578 __ cmpw(reg, zr);
3579 %}
3580
3581 enc_class aarch64_enc_b(label lbl) %{
3582 Label *L = $lbl$$label;
3583 __ b(*L);
3584 %}
3585
3586 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3587 Label *L = $lbl$$label;
3588 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3589 %}
3590
3591 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3592 Label *L = $lbl$$label;
3593 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3594 %}
3595
3596 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3597 %{
3598 Register sub_reg = as_Register($sub$$reg);
3599 Register super_reg = as_Register($super$$reg);
3600 Register temp_reg = as_Register($temp$$reg);
3601 Register result_reg = as_Register($result$$reg);
3602
3603 Label miss;
3604 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3605 nullptr, &miss,
3606 /*set_cond_codes:*/ true);
3607 if ($primary) {
3608 __ mov(result_reg, zr);
3609 }
3610 __ bind(miss);
3611 %}
3612
3613 enc_class aarch64_enc_java_static_call(method meth) %{
3614 address addr = (address)$meth$$method;
3615 address call;
3616 if (!_method) {
3617 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3618 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3619 if (call == nullptr) {
3620 ciEnv::current()->record_failure("CodeCache is full");
3621 return;
3622 }
3623 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3624 // The NOP here is purely to ensure that eliding a call to
3625 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3626 __ nop();
3627 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3628 } else {
3629 int method_index = resolved_method_index(masm);
3630 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3631 : static_call_Relocation::spec(method_index);
3632 call = __ trampoline_call(Address(addr, rspec));
3633 if (call == nullptr) {
3634 ciEnv::current()->record_failure("CodeCache is full");
3635 return;
3636 }
3637 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3638 // Calls of the same statically bound method can share
3639 // a stub to the interpreter.
3640 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3641 } else {
3642 // Emit stub for static call
3643 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3644 if (stub == nullptr) {
3645 ciEnv::current()->record_failure("CodeCache is full");
3646 return;
3647 }
3648 }
3649 }
3650
3651 __ post_call_nop();
3652
3653 // Only non uncommon_trap calls need to reinitialize ptrue.
3654 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3655 __ reinitialize_ptrue();
3656 }
3657 %}
3658
3659 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3660 int method_index = resolved_method_index(masm);
3661 address call = __ ic_call((address)$meth$$method, method_index);
3662 if (call == nullptr) {
3663 ciEnv::current()->record_failure("CodeCache is full");
3664 return;
3665 }
3666 __ post_call_nop();
3667 if (Compile::current()->max_vector_size() > 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_call_epilog() %{
3673 if (VerifyStackAtCalls) {
3674 // Check that stack depth is unchanged: find majik cookie on stack
3675 __ call_Unimplemented();
3676 }
3677 %}
3678
3679 enc_class aarch64_enc_java_to_runtime(method meth) %{
3680 // some calls to generated routines (arraycopy code) are scheduled
3681 // by C2 as runtime calls. if so we can call them using a br (they
3682 // will be in a reachable segment) otherwise we have to use a blr
3683 // which loads the absolute address into a register.
3684 address entry = (address)$meth$$method;
3685 CodeBlob *cb = CodeCache::find_blob(entry);
3686 if (cb) {
3687 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3688 if (call == nullptr) {
3689 ciEnv::current()->record_failure("CodeCache is full");
3690 return;
3691 }
3692 __ post_call_nop();
3693 } else {
3694 Label retaddr;
3695 __ adr(rscratch2, retaddr);
3696 __ lea(rscratch1, RuntimeAddress(entry));
3697 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3698 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3699 __ blr(rscratch1);
3700 __ bind(retaddr);
3701 __ post_call_nop();
3702 __ add(sp, sp, 2 * wordSize);
3703 }
3704 if (Compile::current()->max_vector_size() > 0) {
3705 __ reinitialize_ptrue();
3706 }
3707 %}
3708
3709 enc_class aarch64_enc_rethrow() %{
3710 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3711 %}
3712
3713 enc_class aarch64_enc_ret() %{
3714 #ifdef ASSERT
3715 if (Compile::current()->max_vector_size() > 0) {
3716 __ verify_ptrue();
3717 }
3718 #endif
3719 __ ret(lr);
3720 %}
3721
3722 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3723 Register target_reg = as_Register($jump_target$$reg);
3724 __ br(target_reg);
3725 %}
3726
3727 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3728 Register target_reg = as_Register($jump_target$$reg);
3729 // exception oop should be in r0
3730 // ret addr has been popped into lr
3731 // callee expects it in r3
3732 __ mov(r3, lr);
3733 __ br(target_reg);
3734 %}
3735
3736 %}
3737
3738 //----------FRAME--------------------------------------------------------------
3739 // Definition of frame structure and management information.
3740 //
3741 // S T A C K L A Y O U T Allocators stack-slot number
3742 // | (to get allocators register number
3743 // G Owned by | | v add OptoReg::stack0())
3744 // r CALLER | |
3745 // o | +--------+ pad to even-align allocators stack-slot
3746 // w V | pad0 | numbers; owned by CALLER
3747 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3748 // h ^ | in | 5
3749 // | | args | 4 Holes in incoming args owned by SELF
3750 // | | | | 3
3751 // | | +--------+
3752 // V | | old out| Empty on Intel, window on Sparc
3753 // | old |preserve| Must be even aligned.
3754 // | SP-+--------+----> Matcher::_old_SP, even aligned
3755 // | | in | 3 area for Intel ret address
3756 // Owned by |preserve| Empty on Sparc.
3757 // SELF +--------+
3758 // | | pad2 | 2 pad to align old SP
3759 // | +--------+ 1
3760 // | | locks | 0
3761 // | +--------+----> OptoReg::stack0(), even aligned
3762 // | | pad1 | 11 pad to align new SP
3763 // | +--------+
3764 // | | | 10
3765 // | | spills | 9 spills
3766 // V | | 8 (pad0 slot for callee)
3767 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3768 // ^ | out | 7
3769 // | | args | 6 Holes in outgoing args owned by CALLEE
3770 // Owned by +--------+
3771 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3772 // | new |preserve| Must be even-aligned.
3773 // | SP-+--------+----> Matcher::_new_SP, even aligned
3774 // | | |
3775 //
3776 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3777 // known from SELF's arguments and the Java calling convention.
3778 // Region 6-7 is determined per call site.
3779 // Note 2: If the calling convention leaves holes in the incoming argument
3780 // area, those holes are owned by SELF. Holes in the outgoing area
3781 // are owned by the CALLEE. Holes should not be necessary in the
3782 // incoming area, as the Java calling convention is completely under
3783 // the control of the AD file. Doubles can be sorted and packed to
3784 // avoid holes. Holes in the outgoing arguments may be necessary for
3785 // varargs C calling conventions.
3786 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3787 // even aligned with pad0 as needed.
3788 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3789 // (the latter is true on Intel but is it false on AArch64?)
3790 // region 6-11 is even aligned; it may be padded out more so that
3791 // the region from SP to FP meets the minimum stack alignment.
3792 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3793 // alignment. Region 11, pad1, may be dynamically extended so that
3794 // SP meets the minimum alignment.
3795
3796 frame %{
3797 // These three registers define part of the calling convention
3798 // between compiled code and the interpreter.
3799
3800 // Inline Cache Register or Method for I2C.
3801 inline_cache_reg(R12);
3802
3803 // Number of stack slots consumed by locking an object
3804 sync_stack_slots(2);
3805
3806 // Compiled code's Frame Pointer
3807 frame_pointer(R31);
3808
3809 // Interpreter stores its frame pointer in a register which is
3810 // stored to the stack by I2CAdaptors.
3811 // I2CAdaptors convert from interpreted java to compiled java.
3812 interpreter_frame_pointer(R29);
3813
3814 // Stack alignment requirement
3815 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3816
3817 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3818 // for calls to C. Supports the var-args backing area for register parms.
3819 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3820
3821 // The after-PROLOG location of the return address. Location of
3822 // return address specifies a type (REG or STACK) and a number
3823 // representing the register number (i.e. - use a register name) or
3824 // stack slot.
3825 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3826 // Otherwise, it is above the locks and verification slot and alignment word
3827 // TODO this may well be correct but need to check why that - 2 is there
3828 // ppc port uses 0 but we definitely need to allow for fixed_slots
3829 // which folds in the space used for monitors
3830 return_addr(STACK - 2 +
3831 align_up((Compile::current()->in_preserve_stack_slots() +
3832 Compile::current()->fixed_slots()),
3833 stack_alignment_in_slots()));
3834
3835 // Location of compiled Java return values. Same as C for now.
3836 return_value
3837 %{
3838 // TODO do we allow ideal_reg == Op_RegN???
3839 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3840 "only return normal values");
3841
3842 static const int lo[Op_RegL + 1] = { // enum name
3843 0, // Op_Node
3844 0, // Op_Set
3845 R0_num, // Op_RegN
3846 R0_num, // Op_RegI
3847 R0_num, // Op_RegP
3848 V0_num, // Op_RegF
3849 V0_num, // Op_RegD
3850 R0_num // Op_RegL
3851 };
3852
3853 static const int hi[Op_RegL + 1] = { // enum name
3854 0, // Op_Node
3855 0, // Op_Set
3856 OptoReg::Bad, // Op_RegN
3857 OptoReg::Bad, // Op_RegI
3858 R0_H_num, // Op_RegP
3859 OptoReg::Bad, // Op_RegF
3860 V0_H_num, // Op_RegD
3861 R0_H_num // Op_RegL
3862 };
3863
3864 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3865 %}
3866 %}
3867
3868 //----------ATTRIBUTES---------------------------------------------------------
3869 //----------Operand Attributes-------------------------------------------------
3870 op_attrib op_cost(1); // Required cost attribute
3871
3872 //----------Instruction Attributes---------------------------------------------
3873 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3874 ins_attrib ins_size(32); // Required size attribute (in bits)
3875 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3876 // a non-matching short branch variant
3877 // of some long branch?
3878 ins_attrib ins_alignment(4); // Required alignment attribute (must
3879 // be a power of 2) specifies the
3880 // alignment that some part of the
3881 // instruction (not necessarily the
3882 // start) requires. If > 1, a
3883 // compute_padding() function must be
3884 // provided for the instruction
3885
3886 //----------OPERANDS-----------------------------------------------------------
3887 // Operand definitions must precede instruction definitions for correct parsing
3888 // in the ADLC because operands constitute user defined types which are used in
3889 // instruction definitions.
3890
3891 //----------Simple Operands----------------------------------------------------
3892
3893 // Integer operands 32 bit
3894 // 32 bit immediate
3895 operand immI()
3896 %{
3897 match(ConI);
3898
3899 op_cost(0);
3900 format %{ %}
3901 interface(CONST_INTER);
3902 %}
3903
3904 // 32 bit zero
3905 operand immI0()
3906 %{
3907 predicate(n->get_int() == 0);
3908 match(ConI);
3909
3910 op_cost(0);
3911 format %{ %}
3912 interface(CONST_INTER);
3913 %}
3914
3915 // 32 bit unit increment
3916 operand immI_1()
3917 %{
3918 predicate(n->get_int() == 1);
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit unit decrement
3927 operand immI_M1()
3928 %{
3929 predicate(n->get_int() == -1);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // Shift values for add/sub extension shift
3938 operand immIExt()
3939 %{
3940 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_gt_1()
3949 %{
3950 predicate(n->get_int() > 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_le_4()
3959 %{
3960 predicate(n->get_int() <= 4);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_16()
3969 %{
3970 predicate(n->get_int() == 16);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_24()
3979 %{
3980 predicate(n->get_int() == 24);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_32()
3989 %{
3990 predicate(n->get_int() == 32);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_48()
3999 %{
4000 predicate(n->get_int() == 48);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_56()
4009 %{
4010 predicate(n->get_int() == 56);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_255()
4019 %{
4020 predicate(n->get_int() == 255);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_65535()
4029 %{
4030 predicate(n->get_int() == 65535);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_positive()
4039 %{
4040 predicate(n->get_int() > 0);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 // BoolTest condition for signed compare
4049 operand immI_cmp_cond()
4050 %{
4051 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 // BoolTest condition for unsigned compare
4060 operand immI_cmpU_cond()
4061 %{
4062 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_255()
4071 %{
4072 predicate(n->get_long() == 255L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_65535()
4081 %{
4082 predicate(n->get_long() == 65535L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_4294967295()
4091 %{
4092 predicate(n->get_long() == 4294967295L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_bitmask()
4101 %{
4102 predicate((n->get_long() != 0)
4103 && ((n->get_long() & 0xc000000000000000l) == 0)
4104 && is_power_of_2(n->get_long() + 1));
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immI_bitmask()
4113 %{
4114 predicate((n->get_int() != 0)
4115 && ((n->get_int() & 0xc0000000) == 0)
4116 && is_power_of_2(n->get_int() + 1));
4117 match(ConI);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_positive_bitmaskI()
4125 %{
4126 predicate((n->get_long() != 0)
4127 && ((julong)n->get_long() < 0x80000000ULL)
4128 && is_power_of_2(n->get_long() + 1));
4129 match(ConL);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 // Scale values for scaled offset addressing modes (up to long but not quad)
4137 operand immIScale()
4138 %{
4139 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // 5 bit signed integer
4148 operand immI5()
4149 %{
4150 predicate(Assembler::is_simm(n->get_int(), 5));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // 7 bit unsigned integer
4159 operand immIU7()
4160 %{
4161 predicate(Assembler::is_uimm(n->get_int(), 7));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // Offset for scaled or unscaled immediate loads and stores
4170 operand immIOffset()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset1()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset2()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset4()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset8()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset16()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLOffset()
4231 %{
4232 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset1()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset2()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset4()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset8()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset16()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 // 5 bit signed long integer
4291 operand immL5()
4292 %{
4293 predicate(Assembler::is_simm(n->get_long(), 5));
4294 match(ConL);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // 7 bit unsigned long integer
4302 operand immLU7()
4303 %{
4304 predicate(Assembler::is_uimm(n->get_long(), 7));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 8 bit signed value.
4313 operand immI8()
4314 %{
4315 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4316 match(ConI);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 8 bit signed value (simm8), or #simm8 LSL 8.
4324 operand immI8_shift8()
4325 %{
4326 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4327 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4328 match(ConI);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value (simm8), or #simm8 LSL 8.
4336 operand immL8_shift8()
4337 %{
4338 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4339 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4340 match(ConL);
4341
4342 op_cost(0);
4343 format %{ %}
4344 interface(CONST_INTER);
4345 %}
4346
4347 // 8 bit integer valid for vector add sub immediate
4348 operand immBAddSubV()
4349 %{
4350 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4351 match(ConI);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 // 32 bit integer valid for add sub immediate
4359 operand immIAddSub()
4360 %{
4361 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4362 match(ConI);
4363 op_cost(0);
4364 format %{ %}
4365 interface(CONST_INTER);
4366 %}
4367
4368 // 32 bit integer valid for vector add sub immediate
4369 operand immIAddSubV()
4370 %{
4371 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4372 match(ConI);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 // 32 bit unsigned integer valid for logical immediate
4380
4381 operand immBLog()
4382 %{
4383 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 operand immSLog()
4392 %{
4393 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immILog()
4402 %{
4403 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // Integer operands 64 bit
4412 // 64 bit immediate
4413 operand immL()
4414 %{
4415 match(ConL);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 // 64 bit zero
4423 operand immL0()
4424 %{
4425 predicate(n->get_long() == 0);
4426 match(ConL);
4427
4428 op_cost(0);
4429 format %{ %}
4430 interface(CONST_INTER);
4431 %}
4432
4433 // 64 bit unit decrement
4434 operand immL_M1()
4435 %{
4436 predicate(n->get_long() == -1);
4437 match(ConL);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 // 64 bit integer valid for add sub immediate
4445 operand immLAddSub()
4446 %{
4447 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4448 match(ConL);
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // 64 bit integer valid for addv subv immediate
4455 operand immLAddSubV()
4456 %{
4457 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit integer valid for logical immediate
4466 operand immLLog()
4467 %{
4468 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4469 match(ConL);
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // Long Immediate: low 32-bit mask
4476 operand immL_32bits()
4477 %{
4478 predicate(n->get_long() == 0xFFFFFFFFL);
4479 match(ConL);
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // Pointer operands
4486 // Pointer Immediate
4487 operand immP()
4488 %{
4489 match(ConP);
4490
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // nullptr Pointer Immediate
4497 operand immP0()
4498 %{
4499 predicate(n->get_ptr() == 0);
4500 match(ConP);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // Pointer Immediate One
4508 // this is used in object initialization (initial object header)
4509 operand immP_1()
4510 %{
4511 predicate(n->get_ptr() == 1);
4512 match(ConP);
4513
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 // Card Table Byte Map Base
4520 operand immByteMapBase()
4521 %{
4522 // Get base of card map
4523 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4524 is_card_table_address((address)(n->get_ptr())));
4525 match(ConP);
4526
4527 op_cost(0);
4528 format %{ %}
4529 interface(CONST_INTER);
4530 %}
4531
4532 // AOT Runtime Constants Address
4533 operand immAOTRuntimeConstantsAddress()
4534 %{
4535 // Check if the address is in the range of AOT Runtime Constants
4536 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4537 match(ConP);
4538
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // Float and Double operands
4545 // Double Immediate
4546 operand immD()
4547 %{
4548 match(ConD);
4549 op_cost(0);
4550 format %{ %}
4551 interface(CONST_INTER);
4552 %}
4553
4554 // Double Immediate: +0.0d
4555 operand immD0()
4556 %{
4557 predicate(jlong_cast(n->getd()) == 0);
4558 match(ConD);
4559
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // constant 'double +0.0'.
4566 operand immDPacked()
4567 %{
4568 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4569 match(ConD);
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // Float Immediate
4576 operand immF()
4577 %{
4578 match(ConF);
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // Float Immediate: +0.0f.
4585 operand immF0()
4586 %{
4587 predicate(jint_cast(n->getf()) == 0);
4588 match(ConF);
4589
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 //
4596 operand immFPacked()
4597 %{
4598 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4599 match(ConF);
4600 op_cost(0);
4601 format %{ %}
4602 interface(CONST_INTER);
4603 %}
4604
4605 // Narrow pointer operands
4606 // Narrow Pointer Immediate
4607 operand immN()
4608 %{
4609 match(ConN);
4610
4611 op_cost(0);
4612 format %{ %}
4613 interface(CONST_INTER);
4614 %}
4615
4616 // Narrow nullptr Pointer Immediate
4617 operand immN0()
4618 %{
4619 predicate(n->get_narrowcon() == 0);
4620 match(ConN);
4621
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 operand immNKlass()
4628 %{
4629 match(ConNKlass);
4630
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // Integer 32 bit Register Operands
4637 // Integer 32 bitRegister (excludes SP)
4638 operand iRegI()
4639 %{
4640 constraint(ALLOC_IN_RC(any_reg32));
4641 match(RegI);
4642 match(iRegINoSp);
4643 op_cost(0);
4644 format %{ %}
4645 interface(REG_INTER);
4646 %}
4647
4648 // Integer 32 bit Register not Special
4649 operand iRegINoSp()
4650 %{
4651 constraint(ALLOC_IN_RC(no_special_reg32));
4652 match(RegI);
4653 op_cost(0);
4654 format %{ %}
4655 interface(REG_INTER);
4656 %}
4657
4658 // Integer 64 bit Register Operands
4659 // Integer 64 bit Register (includes SP)
4660 operand iRegL()
4661 %{
4662 constraint(ALLOC_IN_RC(any_reg));
4663 match(RegL);
4664 match(iRegLNoSp);
4665 op_cost(0);
4666 format %{ %}
4667 interface(REG_INTER);
4668 %}
4669
4670 // Integer 64 bit Register not Special
4671 operand iRegLNoSp()
4672 %{
4673 constraint(ALLOC_IN_RC(no_special_reg));
4674 match(RegL);
4675 match(iRegL_R0);
4676 format %{ %}
4677 interface(REG_INTER);
4678 %}
4679
4680 // Pointer Register Operands
4681 // Pointer Register
4682 operand iRegP()
4683 %{
4684 constraint(ALLOC_IN_RC(ptr_reg));
4685 match(RegP);
4686 match(iRegPNoSp);
4687 match(iRegP_R0);
4688 //match(iRegP_R2);
4689 //match(iRegP_R4);
4690 match(iRegP_R5);
4691 match(thread_RegP);
4692 op_cost(0);
4693 format %{ %}
4694 interface(REG_INTER);
4695 %}
4696
4697 // Pointer 64 bit Register not Special
4698 operand iRegPNoSp()
4699 %{
4700 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4701 match(RegP);
4702 // match(iRegP);
4703 // match(iRegP_R0);
4704 // match(iRegP_R2);
4705 // match(iRegP_R4);
4706 // match(iRegP_R5);
4707 // match(thread_RegP);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // This operand is not allowed to use rfp even if
4714 // rfp is not used to hold the frame pointer.
4715 operand iRegPNoSpNoRfp()
4716 %{
4717 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4718 match(RegP);
4719 match(iRegPNoSp);
4720 op_cost(0);
4721 format %{ %}
4722 interface(REG_INTER);
4723 %}
4724
4725 // Pointer 64 bit Register R0 only
4726 operand iRegP_R0()
4727 %{
4728 constraint(ALLOC_IN_RC(r0_reg));
4729 match(RegP);
4730 // match(iRegP);
4731 match(iRegPNoSp);
4732 op_cost(0);
4733 format %{ %}
4734 interface(REG_INTER);
4735 %}
4736
4737 // Pointer 64 bit Register R1 only
4738 operand iRegP_R1()
4739 %{
4740 constraint(ALLOC_IN_RC(r1_reg));
4741 match(RegP);
4742 // match(iRegP);
4743 match(iRegPNoSp);
4744 op_cost(0);
4745 format %{ %}
4746 interface(REG_INTER);
4747 %}
4748
4749 // Pointer 64 bit Register R2 only
4750 operand iRegP_R2()
4751 %{
4752 constraint(ALLOC_IN_RC(r2_reg));
4753 match(RegP);
4754 // match(iRegP);
4755 match(iRegPNoSp);
4756 op_cost(0);
4757 format %{ %}
4758 interface(REG_INTER);
4759 %}
4760
4761 // Pointer 64 bit Register R3 only
4762 operand iRegP_R3()
4763 %{
4764 constraint(ALLOC_IN_RC(r3_reg));
4765 match(RegP);
4766 // match(iRegP);
4767 match(iRegPNoSp);
4768 op_cost(0);
4769 format %{ %}
4770 interface(REG_INTER);
4771 %}
4772
4773 // Pointer 64 bit Register R4 only
4774 operand iRegP_R4()
4775 %{
4776 constraint(ALLOC_IN_RC(r4_reg));
4777 match(RegP);
4778 // match(iRegP);
4779 match(iRegPNoSp);
4780 op_cost(0);
4781 format %{ %}
4782 interface(REG_INTER);
4783 %}
4784
4785 // Pointer 64 bit Register R5 only
4786 operand iRegP_R5()
4787 %{
4788 constraint(ALLOC_IN_RC(r5_reg));
4789 match(RegP);
4790 // match(iRegP);
4791 match(iRegPNoSp);
4792 op_cost(0);
4793 format %{ %}
4794 interface(REG_INTER);
4795 %}
4796
4797 // Pointer 64 bit Register R10 only
4798 operand iRegP_R10()
4799 %{
4800 constraint(ALLOC_IN_RC(r10_reg));
4801 match(RegP);
4802 // match(iRegP);
4803 match(iRegPNoSp);
4804 op_cost(0);
4805 format %{ %}
4806 interface(REG_INTER);
4807 %}
4808
4809 // Long 64 bit Register R0 only
4810 operand iRegL_R0()
4811 %{
4812 constraint(ALLOC_IN_RC(r0_reg));
4813 match(RegL);
4814 match(iRegLNoSp);
4815 op_cost(0);
4816 format %{ %}
4817 interface(REG_INTER);
4818 %}
4819
4820 // Long 64 bit Register R11 only
4821 operand iRegL_R11()
4822 %{
4823 constraint(ALLOC_IN_RC(r11_reg));
4824 match(RegL);
4825 match(iRegLNoSp);
4826 op_cost(0);
4827 format %{ %}
4828 interface(REG_INTER);
4829 %}
4830
4831 // Register R0 only
4832 operand iRegI_R0()
4833 %{
4834 constraint(ALLOC_IN_RC(int_r0_reg));
4835 match(RegI);
4836 match(iRegINoSp);
4837 op_cost(0);
4838 format %{ %}
4839 interface(REG_INTER);
4840 %}
4841
4842 // Register R2 only
4843 operand iRegI_R2()
4844 %{
4845 constraint(ALLOC_IN_RC(int_r2_reg));
4846 match(RegI);
4847 match(iRegINoSp);
4848 op_cost(0);
4849 format %{ %}
4850 interface(REG_INTER);
4851 %}
4852
4853 // Register R3 only
4854 operand iRegI_R3()
4855 %{
4856 constraint(ALLOC_IN_RC(int_r3_reg));
4857 match(RegI);
4858 match(iRegINoSp);
4859 op_cost(0);
4860 format %{ %}
4861 interface(REG_INTER);
4862 %}
4863
4864
4865 // Register R4 only
4866 operand iRegI_R4()
4867 %{
4868 constraint(ALLOC_IN_RC(int_r4_reg));
4869 match(RegI);
4870 match(iRegINoSp);
4871 op_cost(0);
4872 format %{ %}
4873 interface(REG_INTER);
4874 %}
4875
4876
4877 // Pointer Register Operands
4878 // Narrow Pointer Register
4879 operand iRegN()
4880 %{
4881 constraint(ALLOC_IN_RC(any_reg32));
4882 match(RegN);
4883 match(iRegNNoSp);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Integer 64 bit Register not Special
4890 operand iRegNNoSp()
4891 %{
4892 constraint(ALLOC_IN_RC(no_special_reg32));
4893 match(RegN);
4894 op_cost(0);
4895 format %{ %}
4896 interface(REG_INTER);
4897 %}
4898
4899 // Float Register
4900 // Float register operands
4901 operand vRegF()
4902 %{
4903 constraint(ALLOC_IN_RC(float_reg));
4904 match(RegF);
4905
4906 op_cost(0);
4907 format %{ %}
4908 interface(REG_INTER);
4909 %}
4910
4911 // Double Register
4912 // Double register operands
4913 operand vRegD()
4914 %{
4915 constraint(ALLOC_IN_RC(double_reg));
4916 match(RegD);
4917
4918 op_cost(0);
4919 format %{ %}
4920 interface(REG_INTER);
4921 %}
4922
4923 // Generic vector class. This will be used for
4924 // all vector operands, including NEON and SVE.
4925 operand vReg()
4926 %{
4927 constraint(ALLOC_IN_RC(dynamic));
4928 match(VecA);
4929 match(VecD);
4930 match(VecX);
4931
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937 operand vecA()
4938 %{
4939 constraint(ALLOC_IN_RC(vectora_reg));
4940 match(VecA);
4941
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947 operand vecD()
4948 %{
4949 constraint(ALLOC_IN_RC(vectord_reg));
4950 match(VecD);
4951
4952 op_cost(0);
4953 format %{ %}
4954 interface(REG_INTER);
4955 %}
4956
4957 operand vecX()
4958 %{
4959 constraint(ALLOC_IN_RC(vectorx_reg));
4960 match(VecX);
4961
4962 op_cost(0);
4963 format %{ %}
4964 interface(REG_INTER);
4965 %}
4966
4967 operand vRegD_V0()
4968 %{
4969 constraint(ALLOC_IN_RC(v0_reg));
4970 match(RegD);
4971 op_cost(0);
4972 format %{ %}
4973 interface(REG_INTER);
4974 %}
4975
4976 operand vRegD_V1()
4977 %{
4978 constraint(ALLOC_IN_RC(v1_reg));
4979 match(RegD);
4980 op_cost(0);
4981 format %{ %}
4982 interface(REG_INTER);
4983 %}
4984
4985 operand vRegD_V2()
4986 %{
4987 constraint(ALLOC_IN_RC(v2_reg));
4988 match(RegD);
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 operand vRegD_V3()
4995 %{
4996 constraint(ALLOC_IN_RC(v3_reg));
4997 match(RegD);
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vRegD_V4()
5004 %{
5005 constraint(ALLOC_IN_RC(v4_reg));
5006 match(RegD);
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vRegD_V5()
5013 %{
5014 constraint(ALLOC_IN_RC(v5_reg));
5015 match(RegD);
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 operand vRegD_V6()
5022 %{
5023 constraint(ALLOC_IN_RC(v6_reg));
5024 match(RegD);
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vRegD_V7()
5031 %{
5032 constraint(ALLOC_IN_RC(v7_reg));
5033 match(RegD);
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand pReg()
5040 %{
5041 constraint(ALLOC_IN_RC(pr_reg));
5042 match(RegVectMask);
5043 match(pRegGov);
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 operand pRegGov()
5050 %{
5051 constraint(ALLOC_IN_RC(gov_pr));
5052 match(RegVectMask);
5053 match(pReg);
5054 op_cost(0);
5055 format %{ %}
5056 interface(REG_INTER);
5057 %}
5058
5059 operand pRegGov_P0()
5060 %{
5061 constraint(ALLOC_IN_RC(p0_reg));
5062 match(RegVectMask);
5063 op_cost(0);
5064 format %{ %}
5065 interface(REG_INTER);
5066 %}
5067
5068 operand pRegGov_P1()
5069 %{
5070 constraint(ALLOC_IN_RC(p1_reg));
5071 match(RegVectMask);
5072 op_cost(0);
5073 format %{ %}
5074 interface(REG_INTER);
5075 %}
5076
5077 // Flags register, used as output of signed compare instructions
5078
5079 // note that on AArch64 we also use this register as the output for
5080 // for floating point compare instructions (CmpF CmpD). this ensures
5081 // that ordered inequality tests use GT, GE, LT or LE none of which
5082 // pass through cases where the result is unordered i.e. one or both
5083 // inputs to the compare is a NaN. this means that the ideal code can
5084 // replace e.g. a GT with an LE and not end up capturing the NaN case
5085 // (where the comparison should always fail). EQ and NE tests are
5086 // always generated in ideal code so that unordered folds into the NE
5087 // case, matching the behaviour of AArch64 NE.
5088 //
5089 // This differs from x86 where the outputs of FP compares use a
5090 // special FP flags registers and where compares based on this
5091 // register are distinguished into ordered inequalities (cmpOpUCF) and
5092 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5093 // to explicitly handle the unordered case in branches. x86 also has
5094 // to include extra CMoveX rules to accept a cmpOpUCF input.
5095
5096 operand rFlagsReg()
5097 %{
5098 constraint(ALLOC_IN_RC(int_flags));
5099 match(RegFlags);
5100
5101 op_cost(0);
5102 format %{ "RFLAGS" %}
5103 interface(REG_INTER);
5104 %}
5105
5106 // Flags register, used as output of unsigned compare instructions
5107 operand rFlagsRegU()
5108 %{
5109 constraint(ALLOC_IN_RC(int_flags));
5110 match(RegFlags);
5111
5112 op_cost(0);
5113 format %{ "RFLAGSU" %}
5114 interface(REG_INTER);
5115 %}
5116
5117 // Special Registers
5118
5119 // Method Register
5120 operand inline_cache_RegP(iRegP reg)
5121 %{
5122 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5123 match(reg);
5124 match(iRegPNoSp);
5125 op_cost(0);
5126 format %{ %}
5127 interface(REG_INTER);
5128 %}
5129
5130 // Thread Register
5131 operand thread_RegP(iRegP reg)
5132 %{
5133 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5134 match(reg);
5135 op_cost(0);
5136 format %{ %}
5137 interface(REG_INTER);
5138 %}
5139
5140 //----------Memory Operands----------------------------------------------------
5141
5142 operand indirect(iRegP reg)
5143 %{
5144 constraint(ALLOC_IN_RC(ptr_reg));
5145 match(reg);
5146 op_cost(0);
5147 format %{ "[$reg]" %}
5148 interface(MEMORY_INTER) %{
5149 base($reg);
5150 index(0xffffffff);
5151 scale(0x0);
5152 disp(0x0);
5153 %}
5154 %}
5155
5156 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5157 %{
5158 constraint(ALLOC_IN_RC(ptr_reg));
5159 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5160 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5161 op_cost(0);
5162 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5163 interface(MEMORY_INTER) %{
5164 base($reg);
5165 index($ireg);
5166 scale($scale);
5167 disp(0x0);
5168 %}
5169 %}
5170
5171 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5172 %{
5173 constraint(ALLOC_IN_RC(ptr_reg));
5174 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5175 match(AddP reg (LShiftL lreg scale));
5176 op_cost(0);
5177 format %{ "$reg, $lreg lsl($scale)" %}
5178 interface(MEMORY_INTER) %{
5179 base($reg);
5180 index($lreg);
5181 scale($scale);
5182 disp(0x0);
5183 %}
5184 %}
5185
5186 operand indIndexI2L(iRegP reg, iRegI ireg)
5187 %{
5188 constraint(ALLOC_IN_RC(ptr_reg));
5189 match(AddP reg (ConvI2L ireg));
5190 op_cost(0);
5191 format %{ "$reg, $ireg, 0, I2L" %}
5192 interface(MEMORY_INTER) %{
5193 base($reg);
5194 index($ireg);
5195 scale(0x0);
5196 disp(0x0);
5197 %}
5198 %}
5199
5200 operand indIndex(iRegP reg, iRegL lreg)
5201 %{
5202 constraint(ALLOC_IN_RC(ptr_reg));
5203 match(AddP reg lreg);
5204 op_cost(0);
5205 format %{ "$reg, $lreg" %}
5206 interface(MEMORY_INTER) %{
5207 base($reg);
5208 index($lreg);
5209 scale(0x0);
5210 disp(0x0);
5211 %}
5212 %}
5213
5214 operand indOffI1(iRegP reg, immIOffset1 off)
5215 %{
5216 constraint(ALLOC_IN_RC(ptr_reg));
5217 match(AddP reg off);
5218 op_cost(0);
5219 format %{ "[$reg, $off]" %}
5220 interface(MEMORY_INTER) %{
5221 base($reg);
5222 index(0xffffffff);
5223 scale(0x0);
5224 disp($off);
5225 %}
5226 %}
5227
5228 operand indOffI2(iRegP reg, immIOffset2 off)
5229 %{
5230 constraint(ALLOC_IN_RC(ptr_reg));
5231 match(AddP reg off);
5232 op_cost(0);
5233 format %{ "[$reg, $off]" %}
5234 interface(MEMORY_INTER) %{
5235 base($reg);
5236 index(0xffffffff);
5237 scale(0x0);
5238 disp($off);
5239 %}
5240 %}
5241
5242 operand indOffI4(iRegP reg, immIOffset4 off)
5243 %{
5244 constraint(ALLOC_IN_RC(ptr_reg));
5245 match(AddP reg off);
5246 op_cost(0);
5247 format %{ "[$reg, $off]" %}
5248 interface(MEMORY_INTER) %{
5249 base($reg);
5250 index(0xffffffff);
5251 scale(0x0);
5252 disp($off);
5253 %}
5254 %}
5255
5256 operand indOffI8(iRegP reg, immIOffset8 off)
5257 %{
5258 constraint(ALLOC_IN_RC(ptr_reg));
5259 match(AddP reg off);
5260 op_cost(0);
5261 format %{ "[$reg, $off]" %}
5262 interface(MEMORY_INTER) %{
5263 base($reg);
5264 index(0xffffffff);
5265 scale(0x0);
5266 disp($off);
5267 %}
5268 %}
5269
5270 operand indOffI16(iRegP reg, immIOffset16 off)
5271 %{
5272 constraint(ALLOC_IN_RC(ptr_reg));
5273 match(AddP reg off);
5274 op_cost(0);
5275 format %{ "[$reg, $off]" %}
5276 interface(MEMORY_INTER) %{
5277 base($reg);
5278 index(0xffffffff);
5279 scale(0x0);
5280 disp($off);
5281 %}
5282 %}
5283
5284 operand indOffL1(iRegP reg, immLoffset1 off)
5285 %{
5286 constraint(ALLOC_IN_RC(ptr_reg));
5287 match(AddP reg off);
5288 op_cost(0);
5289 format %{ "[$reg, $off]" %}
5290 interface(MEMORY_INTER) %{
5291 base($reg);
5292 index(0xffffffff);
5293 scale(0x0);
5294 disp($off);
5295 %}
5296 %}
5297
5298 operand indOffL2(iRegP reg, immLoffset2 off)
5299 %{
5300 constraint(ALLOC_IN_RC(ptr_reg));
5301 match(AddP reg off);
5302 op_cost(0);
5303 format %{ "[$reg, $off]" %}
5304 interface(MEMORY_INTER) %{
5305 base($reg);
5306 index(0xffffffff);
5307 scale(0x0);
5308 disp($off);
5309 %}
5310 %}
5311
5312 operand indOffL4(iRegP reg, immLoffset4 off)
5313 %{
5314 constraint(ALLOC_IN_RC(ptr_reg));
5315 match(AddP reg off);
5316 op_cost(0);
5317 format %{ "[$reg, $off]" %}
5318 interface(MEMORY_INTER) %{
5319 base($reg);
5320 index(0xffffffff);
5321 scale(0x0);
5322 disp($off);
5323 %}
5324 %}
5325
5326 operand indOffL8(iRegP reg, immLoffset8 off)
5327 %{
5328 constraint(ALLOC_IN_RC(ptr_reg));
5329 match(AddP reg off);
5330 op_cost(0);
5331 format %{ "[$reg, $off]" %}
5332 interface(MEMORY_INTER) %{
5333 base($reg);
5334 index(0xffffffff);
5335 scale(0x0);
5336 disp($off);
5337 %}
5338 %}
5339
5340 operand indOffL16(iRegP reg, immLoffset16 off)
5341 %{
5342 constraint(ALLOC_IN_RC(ptr_reg));
5343 match(AddP reg off);
5344 op_cost(0);
5345 format %{ "[$reg, $off]" %}
5346 interface(MEMORY_INTER) %{
5347 base($reg);
5348 index(0xffffffff);
5349 scale(0x0);
5350 disp($off);
5351 %}
5352 %}
5353
5354 operand indirectX2P(iRegL reg)
5355 %{
5356 constraint(ALLOC_IN_RC(ptr_reg));
5357 match(CastX2P reg);
5358 op_cost(0);
5359 format %{ "[$reg]\t# long -> ptr" %}
5360 interface(MEMORY_INTER) %{
5361 base($reg);
5362 index(0xffffffff);
5363 scale(0x0);
5364 disp(0x0);
5365 %}
5366 %}
5367
5368 operand indOffX2P(iRegL reg, immLOffset off)
5369 %{
5370 constraint(ALLOC_IN_RC(ptr_reg));
5371 match(AddP (CastX2P reg) off);
5372 op_cost(0);
5373 format %{ "[$reg, $off]\t# long -> ptr" %}
5374 interface(MEMORY_INTER) %{
5375 base($reg);
5376 index(0xffffffff);
5377 scale(0x0);
5378 disp($off);
5379 %}
5380 %}
5381
5382 operand indirectN(iRegN reg)
5383 %{
5384 predicate(CompressedOops::shift() == 0);
5385 constraint(ALLOC_IN_RC(ptr_reg));
5386 match(DecodeN reg);
5387 op_cost(0);
5388 format %{ "[$reg]\t# narrow" %}
5389 interface(MEMORY_INTER) %{
5390 base($reg);
5391 index(0xffffffff);
5392 scale(0x0);
5393 disp(0x0);
5394 %}
5395 %}
5396
5397 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5398 %{
5399 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5400 constraint(ALLOC_IN_RC(ptr_reg));
5401 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5402 op_cost(0);
5403 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5404 interface(MEMORY_INTER) %{
5405 base($reg);
5406 index($ireg);
5407 scale($scale);
5408 disp(0x0);
5409 %}
5410 %}
5411
5412 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5413 %{
5414 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5415 constraint(ALLOC_IN_RC(ptr_reg));
5416 match(AddP (DecodeN reg) (LShiftL lreg scale));
5417 op_cost(0);
5418 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5419 interface(MEMORY_INTER) %{
5420 base($reg);
5421 index($lreg);
5422 scale($scale);
5423 disp(0x0);
5424 %}
5425 %}
5426
5427 operand indIndexI2LN(iRegN reg, iRegI ireg)
5428 %{
5429 predicate(CompressedOops::shift() == 0);
5430 constraint(ALLOC_IN_RC(ptr_reg));
5431 match(AddP (DecodeN reg) (ConvI2L ireg));
5432 op_cost(0);
5433 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5434 interface(MEMORY_INTER) %{
5435 base($reg);
5436 index($ireg);
5437 scale(0x0);
5438 disp(0x0);
5439 %}
5440 %}
5441
5442 operand indIndexN(iRegN reg, iRegL lreg)
5443 %{
5444 predicate(CompressedOops::shift() == 0);
5445 constraint(ALLOC_IN_RC(ptr_reg));
5446 match(AddP (DecodeN reg) lreg);
5447 op_cost(0);
5448 format %{ "$reg, $lreg\t# narrow" %}
5449 interface(MEMORY_INTER) %{
5450 base($reg);
5451 index($lreg);
5452 scale(0x0);
5453 disp(0x0);
5454 %}
5455 %}
5456
5457 operand indOffIN(iRegN reg, immIOffset off)
5458 %{
5459 predicate(CompressedOops::shift() == 0);
5460 constraint(ALLOC_IN_RC(ptr_reg));
5461 match(AddP (DecodeN reg) off);
5462 op_cost(0);
5463 format %{ "[$reg, $off]\t# narrow" %}
5464 interface(MEMORY_INTER) %{
5465 base($reg);
5466 index(0xffffffff);
5467 scale(0x0);
5468 disp($off);
5469 %}
5470 %}
5471
5472 operand indOffLN(iRegN reg, immLOffset off)
5473 %{
5474 predicate(CompressedOops::shift() == 0);
5475 constraint(ALLOC_IN_RC(ptr_reg));
5476 match(AddP (DecodeN reg) off);
5477 op_cost(0);
5478 format %{ "[$reg, $off]\t# narrow" %}
5479 interface(MEMORY_INTER) %{
5480 base($reg);
5481 index(0xffffffff);
5482 scale(0x0);
5483 disp($off);
5484 %}
5485 %}
5486
5487
5488 //----------Special Memory Operands--------------------------------------------
5489 // Stack Slot Operand - This operand is used for loading and storing temporary
5490 // values on the stack where a match requires a value to
5491 // flow through memory.
5492 operand stackSlotP(sRegP reg)
5493 %{
5494 constraint(ALLOC_IN_RC(stack_slots));
5495 op_cost(100);
5496 // No match rule because this operand is only generated in matching
5497 // match(RegP);
5498 format %{ "[$reg]" %}
5499 interface(MEMORY_INTER) %{
5500 base(0x1e); // RSP
5501 index(0x0); // No Index
5502 scale(0x0); // No Scale
5503 disp($reg); // Stack Offset
5504 %}
5505 %}
5506
5507 operand stackSlotI(sRegI reg)
5508 %{
5509 constraint(ALLOC_IN_RC(stack_slots));
5510 // No match rule because this operand is only generated in matching
5511 // match(RegI);
5512 format %{ "[$reg]" %}
5513 interface(MEMORY_INTER) %{
5514 base(0x1e); // RSP
5515 index(0x0); // No Index
5516 scale(0x0); // No Scale
5517 disp($reg); // Stack Offset
5518 %}
5519 %}
5520
5521 operand stackSlotF(sRegF reg)
5522 %{
5523 constraint(ALLOC_IN_RC(stack_slots));
5524 // No match rule because this operand is only generated in matching
5525 // match(RegF);
5526 format %{ "[$reg]" %}
5527 interface(MEMORY_INTER) %{
5528 base(0x1e); // RSP
5529 index(0x0); // No Index
5530 scale(0x0); // No Scale
5531 disp($reg); // Stack Offset
5532 %}
5533 %}
5534
5535 operand stackSlotD(sRegD reg)
5536 %{
5537 constraint(ALLOC_IN_RC(stack_slots));
5538 // No match rule because this operand is only generated in matching
5539 // match(RegD);
5540 format %{ "[$reg]" %}
5541 interface(MEMORY_INTER) %{
5542 base(0x1e); // RSP
5543 index(0x0); // No Index
5544 scale(0x0); // No Scale
5545 disp($reg); // Stack Offset
5546 %}
5547 %}
5548
5549 operand stackSlotL(sRegL reg)
5550 %{
5551 constraint(ALLOC_IN_RC(stack_slots));
5552 // No match rule because this operand is only generated in matching
5553 // match(RegL);
5554 format %{ "[$reg]" %}
5555 interface(MEMORY_INTER) %{
5556 base(0x1e); // RSP
5557 index(0x0); // No Index
5558 scale(0x0); // No Scale
5559 disp($reg); // Stack Offset
5560 %}
5561 %}
5562
5563 // Operands for expressing Control Flow
5564 // NOTE: Label is a predefined operand which should not be redefined in
5565 // the AD file. It is generically handled within the ADLC.
5566
5567 //----------Conditional Branch Operands----------------------------------------
5568 // Comparison Op - This is the operation of the comparison, and is limited to
5569 // the following set of codes:
5570 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5571 //
5572 // Other attributes of the comparison, such as unsignedness, are specified
5573 // by the comparison instruction that sets a condition code flags register.
5574 // That result is represented by a flags operand whose subtype is appropriate
5575 // to the unsignedness (etc.) of the comparison.
5576 //
5577 // Later, the instruction which matches both the Comparison Op (a Bool) and
5578 // the flags (produced by the Cmp) specifies the coding of the comparison op
5579 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5580
5581 // used for signed integral comparisons and fp comparisons
5582
5583 operand cmpOp()
5584 %{
5585 match(Bool);
5586
5587 format %{ "" %}
5588 interface(COND_INTER) %{
5589 equal(0x0, "eq");
5590 not_equal(0x1, "ne");
5591 less(0xb, "lt");
5592 greater_equal(0xa, "ge");
5593 less_equal(0xd, "le");
5594 greater(0xc, "gt");
5595 overflow(0x6, "vs");
5596 no_overflow(0x7, "vc");
5597 %}
5598 %}
5599
5600 // used for unsigned integral comparisons
5601
5602 operand cmpOpU()
5603 %{
5604 match(Bool);
5605
5606 format %{ "" %}
5607 interface(COND_INTER) %{
5608 equal(0x0, "eq");
5609 not_equal(0x1, "ne");
5610 less(0x3, "lo");
5611 greater_equal(0x2, "hs");
5612 less_equal(0x9, "ls");
5613 greater(0x8, "hi");
5614 overflow(0x6, "vs");
5615 no_overflow(0x7, "vc");
5616 %}
5617 %}
5618
5619 // used for certain integral comparisons which can be
5620 // converted to cbxx or tbxx instructions
5621
5622 operand cmpOpEqNe()
5623 %{
5624 match(Bool);
5625 op_cost(0);
5626 predicate(n->as_Bool()->_test._test == BoolTest::ne
5627 || n->as_Bool()->_test._test == BoolTest::eq);
5628
5629 format %{ "" %}
5630 interface(COND_INTER) %{
5631 equal(0x0, "eq");
5632 not_equal(0x1, "ne");
5633 less(0xb, "lt");
5634 greater_equal(0xa, "ge");
5635 less_equal(0xd, "le");
5636 greater(0xc, "gt");
5637 overflow(0x6, "vs");
5638 no_overflow(0x7, "vc");
5639 %}
5640 %}
5641
5642 // used for certain integral comparisons which can be
5643 // converted to cbxx or tbxx instructions
5644
5645 operand cmpOpLtGe()
5646 %{
5647 match(Bool);
5648 op_cost(0);
5649
5650 predicate(n->as_Bool()->_test._test == BoolTest::lt
5651 || n->as_Bool()->_test._test == BoolTest::ge);
5652
5653 format %{ "" %}
5654 interface(COND_INTER) %{
5655 equal(0x0, "eq");
5656 not_equal(0x1, "ne");
5657 less(0xb, "lt");
5658 greater_equal(0xa, "ge");
5659 less_equal(0xd, "le");
5660 greater(0xc, "gt");
5661 overflow(0x6, "vs");
5662 no_overflow(0x7, "vc");
5663 %}
5664 %}
5665
5666 // used for certain unsigned integral comparisons which can be
5667 // converted to cbxx or tbxx instructions
5668
5669 operand cmpOpUEqNeLeGt()
5670 %{
5671 match(Bool);
5672 op_cost(0);
5673
5674 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5675 n->as_Bool()->_test._test == BoolTest::ne ||
5676 n->as_Bool()->_test._test == BoolTest::le ||
5677 n->as_Bool()->_test._test == BoolTest::gt);
5678
5679 format %{ "" %}
5680 interface(COND_INTER) %{
5681 equal(0x0, "eq");
5682 not_equal(0x1, "ne");
5683 less(0x3, "lo");
5684 greater_equal(0x2, "hs");
5685 less_equal(0x9, "ls");
5686 greater(0x8, "hi");
5687 overflow(0x6, "vs");
5688 no_overflow(0x7, "vc");
5689 %}
5690 %}
5691
5692 // Special operand allowing long args to int ops to be truncated for free
5693
5694 operand iRegL2I(iRegL reg) %{
5695
5696 op_cost(0);
5697
5698 match(ConvL2I reg);
5699
5700 format %{ "l2i($reg)" %}
5701
5702 interface(REG_INTER)
5703 %}
5704
5705 operand iRegL2P(iRegL reg) %{
5706
5707 op_cost(0);
5708
5709 match(CastX2P reg);
5710
5711 format %{ "l2p($reg)" %}
5712
5713 interface(REG_INTER)
5714 %}
5715
5716 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5717 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5718 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5719 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5720
5721 //----------OPERAND CLASSES----------------------------------------------------
5722 // Operand Classes are groups of operands that are used as to simplify
5723 // instruction definitions by not requiring the AD writer to specify
5724 // separate instructions for every form of operand when the
5725 // instruction accepts multiple operand types with the same basic
5726 // encoding and format. The classic case of this is memory operands.
5727
5728 // memory is used to define read/write location for load/store
5729 // instruction defs. we can turn a memory op into an Address
5730
5731 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5732 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5733
5734 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5735 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5736
5737 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5738 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5739
5740 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5741 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5742
5743 // All of the memory operands. For the pipeline description.
5744 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5745 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5746 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5747
5748
5749 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5750 // operations. it allows the src to be either an iRegI or a (ConvL2I
5751 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5752 // can be elided because the 32-bit instruction will just employ the
5753 // lower 32 bits anyway.
5754 //
5755 // n.b. this does not elide all L2I conversions. if the truncated
5756 // value is consumed by more than one operation then the ConvL2I
5757 // cannot be bundled into the consuming nodes so an l2i gets planted
5758 // (actually a movw $dst $src) and the downstream instructions consume
5759 // the result of the l2i as an iRegI input. That's a shame since the
5760 // movw is actually redundant but its not too costly.
5761
5762 opclass iRegIorL2I(iRegI, iRegL2I);
5763 opclass iRegPorL2P(iRegP, iRegL2P);
5764
5765 //----------PIPELINE-----------------------------------------------------------
5766 // Rules which define the behavior of the target architectures pipeline.
5767
5768 // For specific pipelines, eg A53, define the stages of that pipeline
5769 //pipe_desc(ISS, EX1, EX2, WR);
5770 #define ISS S0
5771 #define EX1 S1
5772 #define EX2 S2
5773 #define WR S3
5774
5775 // Integer ALU reg operation
5776 pipeline %{
5777
5778 attributes %{
5779 // ARM instructions are of fixed length
5780 fixed_size_instructions; // Fixed size instructions TODO does
5781 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5782 // ARM instructions come in 32-bit word units
5783 instruction_unit_size = 4; // An instruction is 4 bytes long
5784 instruction_fetch_unit_size = 64; // The processor fetches one line
5785 instruction_fetch_units = 1; // of 64 bytes
5786
5787 // List of nop instructions
5788 nops( MachNop );
5789 %}
5790
5791 // We don't use an actual pipeline model so don't care about resources
5792 // or description. we do use pipeline classes to introduce fixed
5793 // latencies
5794
5795 //----------RESOURCES----------------------------------------------------------
5796 // Resources are the functional units available to the machine
5797
5798 resources( INS0, INS1, INS01 = INS0 | INS1,
5799 ALU0, ALU1, ALU = ALU0 | ALU1,
5800 MAC,
5801 DIV,
5802 BRANCH,
5803 LDST,
5804 NEON_FP);
5805
5806 //----------PIPELINE DESCRIPTION-----------------------------------------------
5807 // Pipeline Description specifies the stages in the machine's pipeline
5808
5809 // Define the pipeline as a generic 6 stage pipeline
5810 pipe_desc(S0, S1, S2, S3, S4, S5);
5811
5812 //----------PIPELINE CLASSES---------------------------------------------------
5813 // Pipeline Classes describe the stages in which input and output are
5814 // referenced by the hardware pipeline.
5815
5816 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5817 %{
5818 single_instruction;
5819 src1 : S1(read);
5820 src2 : S2(read);
5821 dst : S5(write);
5822 INS01 : ISS;
5823 NEON_FP : S5;
5824 %}
5825
5826 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5827 %{
5828 single_instruction;
5829 src1 : S1(read);
5830 src2 : S2(read);
5831 dst : S5(write);
5832 INS01 : ISS;
5833 NEON_FP : S5;
5834 %}
5835
5836 pipe_class fp_uop_s(vRegF dst, vRegF src)
5837 %{
5838 single_instruction;
5839 src : S1(read);
5840 dst : S5(write);
5841 INS01 : ISS;
5842 NEON_FP : S5;
5843 %}
5844
5845 pipe_class fp_uop_d(vRegD dst, vRegD src)
5846 %{
5847 single_instruction;
5848 src : S1(read);
5849 dst : S5(write);
5850 INS01 : ISS;
5851 NEON_FP : S5;
5852 %}
5853
5854 pipe_class fp_d2f(vRegF dst, vRegD src)
5855 %{
5856 single_instruction;
5857 src : S1(read);
5858 dst : S5(write);
5859 INS01 : ISS;
5860 NEON_FP : S5;
5861 %}
5862
5863 pipe_class fp_f2d(vRegD dst, vRegF src)
5864 %{
5865 single_instruction;
5866 src : S1(read);
5867 dst : S5(write);
5868 INS01 : ISS;
5869 NEON_FP : S5;
5870 %}
5871
5872 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5873 %{
5874 single_instruction;
5875 src : S1(read);
5876 dst : S5(write);
5877 INS01 : ISS;
5878 NEON_FP : S5;
5879 %}
5880
5881 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5882 %{
5883 single_instruction;
5884 src : S1(read);
5885 dst : S5(write);
5886 INS01 : ISS;
5887 NEON_FP : S5;
5888 %}
5889
5890 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5891 %{
5892 single_instruction;
5893 src : S1(read);
5894 dst : S5(write);
5895 INS01 : ISS;
5896 NEON_FP : S5;
5897 %}
5898
5899 pipe_class fp_l2f(vRegF dst, iRegL src)
5900 %{
5901 single_instruction;
5902 src : S1(read);
5903 dst : S5(write);
5904 INS01 : ISS;
5905 NEON_FP : S5;
5906 %}
5907
5908 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5909 %{
5910 single_instruction;
5911 src : S1(read);
5912 dst : S5(write);
5913 INS01 : ISS;
5914 NEON_FP : S5;
5915 %}
5916
5917 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5918 %{
5919 single_instruction;
5920 src : S1(read);
5921 dst : S5(write);
5922 INS01 : ISS;
5923 NEON_FP : S5;
5924 %}
5925
5926 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5927 %{
5928 single_instruction;
5929 src : S1(read);
5930 dst : S5(write);
5931 INS01 : ISS;
5932 NEON_FP : S5;
5933 %}
5934
5935 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5936 %{
5937 single_instruction;
5938 src : S1(read);
5939 dst : S5(write);
5940 INS01 : ISS;
5941 NEON_FP : S5;
5942 %}
5943
5944 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5945 %{
5946 single_instruction;
5947 src1 : S1(read);
5948 src2 : S2(read);
5949 dst : S5(write);
5950 INS0 : ISS;
5951 NEON_FP : S5;
5952 %}
5953
5954 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5955 %{
5956 single_instruction;
5957 src1 : S1(read);
5958 src2 : S2(read);
5959 dst : S5(write);
5960 INS0 : ISS;
5961 NEON_FP : S5;
5962 %}
5963
5964 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5965 %{
5966 single_instruction;
5967 cr : S1(read);
5968 src1 : S1(read);
5969 src2 : S1(read);
5970 dst : S3(write);
5971 INS01 : ISS;
5972 NEON_FP : S3;
5973 %}
5974
5975 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5976 %{
5977 single_instruction;
5978 cr : S1(read);
5979 src1 : S1(read);
5980 src2 : S1(read);
5981 dst : S3(write);
5982 INS01 : ISS;
5983 NEON_FP : S3;
5984 %}
5985
5986 pipe_class fp_imm_s(vRegF dst)
5987 %{
5988 single_instruction;
5989 dst : S3(write);
5990 INS01 : ISS;
5991 NEON_FP : S3;
5992 %}
5993
5994 pipe_class fp_imm_d(vRegD dst)
5995 %{
5996 single_instruction;
5997 dst : S3(write);
5998 INS01 : ISS;
5999 NEON_FP : S3;
6000 %}
6001
6002 pipe_class fp_load_constant_s(vRegF dst)
6003 %{
6004 single_instruction;
6005 dst : S4(write);
6006 INS01 : ISS;
6007 NEON_FP : S4;
6008 %}
6009
6010 pipe_class fp_load_constant_d(vRegD dst)
6011 %{
6012 single_instruction;
6013 dst : S4(write);
6014 INS01 : ISS;
6015 NEON_FP : S4;
6016 %}
6017
6018 //------- Integer ALU operations --------------------------
6019
6020 // Integer ALU reg-reg operation
6021 // Operands needed in EX1, result generated in EX2
6022 // Eg. ADD x0, x1, x2
6023 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6024 %{
6025 single_instruction;
6026 dst : EX2(write);
6027 src1 : EX1(read);
6028 src2 : EX1(read);
6029 INS01 : ISS; // Dual issue as instruction 0 or 1
6030 ALU : EX2;
6031 %}
6032
6033 // Integer ALU reg-reg operation with constant shift
6034 // Shifted register must be available in LATE_ISS instead of EX1
6035 // Eg. ADD x0, x1, x2, LSL #2
6036 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6037 %{
6038 single_instruction;
6039 dst : EX2(write);
6040 src1 : EX1(read);
6041 src2 : ISS(read);
6042 INS01 : ISS;
6043 ALU : EX2;
6044 %}
6045
6046 // Integer ALU reg operation with constant shift
6047 // Eg. LSL x0, x1, #shift
6048 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6049 %{
6050 single_instruction;
6051 dst : EX2(write);
6052 src1 : ISS(read);
6053 INS01 : ISS;
6054 ALU : EX2;
6055 %}
6056
6057 // Integer ALU reg-reg operation with variable shift
6058 // Both operands must be available in LATE_ISS instead of EX1
6059 // Result is available in EX1 instead of EX2
6060 // Eg. LSLV x0, x1, x2
6061 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6062 %{
6063 single_instruction;
6064 dst : EX1(write);
6065 src1 : ISS(read);
6066 src2 : ISS(read);
6067 INS01 : ISS;
6068 ALU : EX1;
6069 %}
6070
6071 // Integer ALU reg-reg operation with extract
6072 // As for _vshift above, but result generated in EX2
6073 // Eg. EXTR x0, x1, x2, #N
6074 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6075 %{
6076 single_instruction;
6077 dst : EX2(write);
6078 src1 : ISS(read);
6079 src2 : ISS(read);
6080 INS1 : ISS; // Can only dual issue as Instruction 1
6081 ALU : EX1;
6082 %}
6083
6084 // Integer ALU reg operation
6085 // Eg. NEG x0, x1
6086 pipe_class ialu_reg(iRegI dst, iRegI src)
6087 %{
6088 single_instruction;
6089 dst : EX2(write);
6090 src : EX1(read);
6091 INS01 : ISS;
6092 ALU : EX2;
6093 %}
6094
6095 // Integer ALU reg mmediate operation
6096 // Eg. ADD x0, x1, #N
6097 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6098 %{
6099 single_instruction;
6100 dst : EX2(write);
6101 src1 : EX1(read);
6102 INS01 : ISS;
6103 ALU : EX2;
6104 %}
6105
6106 // Integer ALU immediate operation (no source operands)
6107 // Eg. MOV x0, #N
6108 pipe_class ialu_imm(iRegI dst)
6109 %{
6110 single_instruction;
6111 dst : EX1(write);
6112 INS01 : ISS;
6113 ALU : EX1;
6114 %}
6115
6116 //------- Compare operation -------------------------------
6117
6118 // Compare reg-reg
6119 // Eg. CMP x0, x1
6120 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6121 %{
6122 single_instruction;
6123 // fixed_latency(16);
6124 cr : EX2(write);
6125 op1 : EX1(read);
6126 op2 : EX1(read);
6127 INS01 : ISS;
6128 ALU : EX2;
6129 %}
6130
6131 // Compare reg-reg
6132 // Eg. CMP x0, #N
6133 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6134 %{
6135 single_instruction;
6136 // fixed_latency(16);
6137 cr : EX2(write);
6138 op1 : EX1(read);
6139 INS01 : ISS;
6140 ALU : EX2;
6141 %}
6142
6143 //------- Conditional instructions ------------------------
6144
6145 // Conditional no operands
6146 // Eg. CSINC x0, zr, zr, <cond>
6147 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6148 %{
6149 single_instruction;
6150 cr : EX1(read);
6151 dst : EX2(write);
6152 INS01 : ISS;
6153 ALU : EX2;
6154 %}
6155
6156 // Conditional 2 operand
6157 // EG. CSEL X0, X1, X2, <cond>
6158 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6159 %{
6160 single_instruction;
6161 cr : EX1(read);
6162 src1 : EX1(read);
6163 src2 : EX1(read);
6164 dst : EX2(write);
6165 INS01 : ISS;
6166 ALU : EX2;
6167 %}
6168
6169 // Conditional 2 operand
6170 // EG. CSEL X0, X1, X2, <cond>
6171 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6172 %{
6173 single_instruction;
6174 cr : EX1(read);
6175 src : EX1(read);
6176 dst : EX2(write);
6177 INS01 : ISS;
6178 ALU : EX2;
6179 %}
6180
6181 //------- Multiply pipeline operations --------------------
6182
6183 // Multiply reg-reg
6184 // Eg. MUL w0, w1, w2
6185 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6186 %{
6187 single_instruction;
6188 dst : WR(write);
6189 src1 : ISS(read);
6190 src2 : ISS(read);
6191 INS01 : ISS;
6192 MAC : WR;
6193 %}
6194
6195 // Multiply accumulate
6196 // Eg. MADD w0, w1, w2, w3
6197 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6198 %{
6199 single_instruction;
6200 dst : WR(write);
6201 src1 : ISS(read);
6202 src2 : ISS(read);
6203 src3 : ISS(read);
6204 INS01 : ISS;
6205 MAC : WR;
6206 %}
6207
6208 // Eg. MUL w0, w1, w2
6209 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6210 %{
6211 single_instruction;
6212 fixed_latency(3); // Maximum latency for 64 bit mul
6213 dst : WR(write);
6214 src1 : ISS(read);
6215 src2 : ISS(read);
6216 INS01 : ISS;
6217 MAC : WR;
6218 %}
6219
6220 // Multiply accumulate
6221 // Eg. MADD w0, w1, w2, w3
6222 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6223 %{
6224 single_instruction;
6225 fixed_latency(3); // Maximum latency for 64 bit mul
6226 dst : WR(write);
6227 src1 : ISS(read);
6228 src2 : ISS(read);
6229 src3 : ISS(read);
6230 INS01 : ISS;
6231 MAC : WR;
6232 %}
6233
6234 //------- Divide pipeline operations --------------------
6235
6236 // Eg. SDIV w0, w1, w2
6237 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6238 %{
6239 single_instruction;
6240 fixed_latency(8); // Maximum latency for 32 bit divide
6241 dst : WR(write);
6242 src1 : ISS(read);
6243 src2 : ISS(read);
6244 INS0 : ISS; // Can only dual issue as instruction 0
6245 DIV : WR;
6246 %}
6247
6248 // Eg. SDIV x0, x1, x2
6249 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6250 %{
6251 single_instruction;
6252 fixed_latency(16); // Maximum latency for 64 bit divide
6253 dst : WR(write);
6254 src1 : ISS(read);
6255 src2 : ISS(read);
6256 INS0 : ISS; // Can only dual issue as instruction 0
6257 DIV : WR;
6258 %}
6259
6260 //------- Load pipeline operations ------------------------
6261
6262 // Load - prefetch
6263 // Eg. PFRM <mem>
6264 pipe_class iload_prefetch(memory mem)
6265 %{
6266 single_instruction;
6267 mem : ISS(read);
6268 INS01 : ISS;
6269 LDST : WR;
6270 %}
6271
6272 // Load - reg, mem
6273 // Eg. LDR x0, <mem>
6274 pipe_class iload_reg_mem(iRegI dst, memory mem)
6275 %{
6276 single_instruction;
6277 dst : WR(write);
6278 mem : ISS(read);
6279 INS01 : ISS;
6280 LDST : WR;
6281 %}
6282
6283 // Load - reg, reg
6284 // Eg. LDR x0, [sp, x1]
6285 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6286 %{
6287 single_instruction;
6288 dst : WR(write);
6289 src : ISS(read);
6290 INS01 : ISS;
6291 LDST : WR;
6292 %}
6293
6294 //------- Store pipeline operations -----------------------
6295
6296 // Store - zr, mem
6297 // Eg. STR zr, <mem>
6298 pipe_class istore_mem(memory mem)
6299 %{
6300 single_instruction;
6301 mem : ISS(read);
6302 INS01 : ISS;
6303 LDST : WR;
6304 %}
6305
6306 // Store - reg, mem
6307 // Eg. STR x0, <mem>
6308 pipe_class istore_reg_mem(iRegI src, memory mem)
6309 %{
6310 single_instruction;
6311 mem : ISS(read);
6312 src : EX2(read);
6313 INS01 : ISS;
6314 LDST : WR;
6315 %}
6316
6317 // Store - reg, reg
6318 // Eg. STR x0, [sp, x1]
6319 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6320 %{
6321 single_instruction;
6322 dst : ISS(read);
6323 src : EX2(read);
6324 INS01 : ISS;
6325 LDST : WR;
6326 %}
6327
6328 //------- Store pipeline operations -----------------------
6329
6330 // Branch
6331 pipe_class pipe_branch()
6332 %{
6333 single_instruction;
6334 INS01 : ISS;
6335 BRANCH : EX1;
6336 %}
6337
6338 // Conditional branch
6339 pipe_class pipe_branch_cond(rFlagsReg cr)
6340 %{
6341 single_instruction;
6342 cr : EX1(read);
6343 INS01 : ISS;
6344 BRANCH : EX1;
6345 %}
6346
6347 // Compare & Branch
6348 // EG. CBZ/CBNZ
6349 pipe_class pipe_cmp_branch(iRegI op1)
6350 %{
6351 single_instruction;
6352 op1 : EX1(read);
6353 INS01 : ISS;
6354 BRANCH : EX1;
6355 %}
6356
6357 //------- Synchronisation operations ----------------------
6358
6359 // Any operation requiring serialization.
6360 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6361 pipe_class pipe_serial()
6362 %{
6363 single_instruction;
6364 force_serialization;
6365 fixed_latency(16);
6366 INS01 : ISS(2); // Cannot dual issue with any other instruction
6367 LDST : WR;
6368 %}
6369
6370 // Generic big/slow expanded idiom - also serialized
6371 pipe_class pipe_slow()
6372 %{
6373 instruction_count(10);
6374 multiple_bundles;
6375 force_serialization;
6376 fixed_latency(16);
6377 INS01 : ISS(2); // Cannot dual issue with any other instruction
6378 LDST : WR;
6379 %}
6380
6381 // Empty pipeline class
6382 pipe_class pipe_class_empty()
6383 %{
6384 single_instruction;
6385 fixed_latency(0);
6386 %}
6387
6388 // Default pipeline class.
6389 pipe_class pipe_class_default()
6390 %{
6391 single_instruction;
6392 fixed_latency(2);
6393 %}
6394
6395 // Pipeline class for compares.
6396 pipe_class pipe_class_compare()
6397 %{
6398 single_instruction;
6399 fixed_latency(16);
6400 %}
6401
6402 // Pipeline class for memory operations.
6403 pipe_class pipe_class_memory()
6404 %{
6405 single_instruction;
6406 fixed_latency(16);
6407 %}
6408
6409 // Pipeline class for call.
6410 pipe_class pipe_class_call()
6411 %{
6412 single_instruction;
6413 fixed_latency(100);
6414 %}
6415
6416 // Define the class for the Nop node.
6417 define %{
6418 MachNop = pipe_class_empty;
6419 %}
6420
6421 %}
6422 //----------INSTRUCTIONS-------------------------------------------------------
6423 //
6424 // match -- States which machine-independent subtree may be replaced
6425 // by this instruction.
6426 // ins_cost -- The estimated cost of this instruction is used by instruction
6427 // selection to identify a minimum cost tree of machine
6428 // instructions that matches a tree of machine-independent
6429 // instructions.
6430 // format -- A string providing the disassembly for this instruction.
6431 // The value of an instruction's operand may be inserted
6432 // by referring to it with a '$' prefix.
6433 // opcode -- Three instruction opcodes may be provided. These are referred
6434 // to within an encode class as $primary, $secondary, and $tertiary
6435 // rrspectively. The primary opcode is commonly used to
6436 // indicate the type of machine instruction, while secondary
6437 // and tertiary are often used for prefix options or addressing
6438 // modes.
6439 // ins_encode -- A list of encode classes with parameters. The encode class
6440 // name must have been defined in an 'enc_class' specification
6441 // in the encode section of the architecture description.
6442
6443 // ============================================================================
6444 // Memory (Load/Store) Instructions
6445
6446 // Load Instructions
6447
6448 // Load Byte (8 bit signed)
6449 instruct loadB(iRegINoSp dst, memory1 mem)
6450 %{
6451 match(Set dst (LoadB mem));
6452 predicate(!needs_acquiring_load(n));
6453
6454 ins_cost(4 * INSN_COST);
6455 format %{ "ldrsbw $dst, $mem\t# byte" %}
6456
6457 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6458
6459 ins_pipe(iload_reg_mem);
6460 %}
6461
6462 // Load Byte (8 bit signed) into long
6463 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6464 %{
6465 match(Set dst (ConvI2L (LoadB mem)));
6466 predicate(!needs_acquiring_load(n->in(1)));
6467
6468 ins_cost(4 * INSN_COST);
6469 format %{ "ldrsb $dst, $mem\t# byte" %}
6470
6471 ins_encode(aarch64_enc_ldrsb(dst, mem));
6472
6473 ins_pipe(iload_reg_mem);
6474 %}
6475
6476 // Load Byte (8 bit unsigned)
6477 instruct loadUB(iRegINoSp dst, memory1 mem)
6478 %{
6479 match(Set dst (LoadUB mem));
6480 predicate(!needs_acquiring_load(n));
6481
6482 ins_cost(4 * INSN_COST);
6483 format %{ "ldrbw $dst, $mem\t# byte" %}
6484
6485 ins_encode(aarch64_enc_ldrb(dst, mem));
6486
6487 ins_pipe(iload_reg_mem);
6488 %}
6489
6490 // Load Byte (8 bit unsigned) into long
6491 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6492 %{
6493 match(Set dst (ConvI2L (LoadUB mem)));
6494 predicate(!needs_acquiring_load(n->in(1)));
6495
6496 ins_cost(4 * INSN_COST);
6497 format %{ "ldrb $dst, $mem\t# byte" %}
6498
6499 ins_encode(aarch64_enc_ldrb(dst, mem));
6500
6501 ins_pipe(iload_reg_mem);
6502 %}
6503
6504 // Load Short (16 bit signed)
6505 instruct loadS(iRegINoSp dst, memory2 mem)
6506 %{
6507 match(Set dst (LoadS mem));
6508 predicate(!needs_acquiring_load(n));
6509
6510 ins_cost(4 * INSN_COST);
6511 format %{ "ldrshw $dst, $mem\t# short" %}
6512
6513 ins_encode(aarch64_enc_ldrshw(dst, mem));
6514
6515 ins_pipe(iload_reg_mem);
6516 %}
6517
6518 // Load Short (16 bit signed) into long
6519 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6520 %{
6521 match(Set dst (ConvI2L (LoadS mem)));
6522 predicate(!needs_acquiring_load(n->in(1)));
6523
6524 ins_cost(4 * INSN_COST);
6525 format %{ "ldrsh $dst, $mem\t# short" %}
6526
6527 ins_encode(aarch64_enc_ldrsh(dst, mem));
6528
6529 ins_pipe(iload_reg_mem);
6530 %}
6531
6532 // Load Char (16 bit unsigned)
6533 instruct loadUS(iRegINoSp dst, memory2 mem)
6534 %{
6535 match(Set dst (LoadUS mem));
6536 predicate(!needs_acquiring_load(n));
6537
6538 ins_cost(4 * INSN_COST);
6539 format %{ "ldrh $dst, $mem\t# short" %}
6540
6541 ins_encode(aarch64_enc_ldrh(dst, mem));
6542
6543 ins_pipe(iload_reg_mem);
6544 %}
6545
6546 // Load Short/Char (16 bit unsigned) into long
6547 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6548 %{
6549 match(Set dst (ConvI2L (LoadUS mem)));
6550 predicate(!needs_acquiring_load(n->in(1)));
6551
6552 ins_cost(4 * INSN_COST);
6553 format %{ "ldrh $dst, $mem\t# short" %}
6554
6555 ins_encode(aarch64_enc_ldrh(dst, mem));
6556
6557 ins_pipe(iload_reg_mem);
6558 %}
6559
6560 // Load Integer (32 bit signed)
6561 instruct loadI(iRegINoSp dst, memory4 mem)
6562 %{
6563 match(Set dst (LoadI mem));
6564 predicate(!needs_acquiring_load(n));
6565
6566 ins_cost(4 * INSN_COST);
6567 format %{ "ldrw $dst, $mem\t# int" %}
6568
6569 ins_encode(aarch64_enc_ldrw(dst, mem));
6570
6571 ins_pipe(iload_reg_mem);
6572 %}
6573
6574 // Load Integer (32 bit signed) into long
6575 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6576 %{
6577 match(Set dst (ConvI2L (LoadI mem)));
6578 predicate(!needs_acquiring_load(n->in(1)));
6579
6580 ins_cost(4 * INSN_COST);
6581 format %{ "ldrsw $dst, $mem\t# int" %}
6582
6583 ins_encode(aarch64_enc_ldrsw(dst, mem));
6584
6585 ins_pipe(iload_reg_mem);
6586 %}
6587
6588 // Load Integer (32 bit unsigned) into long
6589 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6590 %{
6591 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6592 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6593
6594 ins_cost(4 * INSN_COST);
6595 format %{ "ldrw $dst, $mem\t# int" %}
6596
6597 ins_encode(aarch64_enc_ldrw(dst, mem));
6598
6599 ins_pipe(iload_reg_mem);
6600 %}
6601
6602 // Load Long (64 bit signed)
6603 instruct loadL(iRegLNoSp dst, memory8 mem)
6604 %{
6605 match(Set dst (LoadL mem));
6606 predicate(!needs_acquiring_load(n));
6607
6608 ins_cost(4 * INSN_COST);
6609 format %{ "ldr $dst, $mem\t# int" %}
6610
6611 ins_encode(aarch64_enc_ldr(dst, mem));
6612
6613 ins_pipe(iload_reg_mem);
6614 %}
6615
6616 // Load Range
6617 instruct loadRange(iRegINoSp dst, memory4 mem)
6618 %{
6619 match(Set dst (LoadRange mem));
6620
6621 ins_cost(4 * INSN_COST);
6622 format %{ "ldrw $dst, $mem\t# range" %}
6623
6624 ins_encode(aarch64_enc_ldrw(dst, mem));
6625
6626 ins_pipe(iload_reg_mem);
6627 %}
6628
6629 // Load Pointer
6630 instruct loadP(iRegPNoSp dst, memory8 mem)
6631 %{
6632 match(Set dst (LoadP mem));
6633 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6634
6635 ins_cost(4 * INSN_COST);
6636 format %{ "ldr $dst, $mem\t# ptr" %}
6637
6638 ins_encode(aarch64_enc_ldr(dst, mem));
6639
6640 ins_pipe(iload_reg_mem);
6641 %}
6642
6643 // Load Compressed Pointer
6644 instruct loadN(iRegNNoSp dst, memory4 mem)
6645 %{
6646 match(Set dst (LoadN mem));
6647 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6648
6649 ins_cost(4 * INSN_COST);
6650 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6651
6652 ins_encode(aarch64_enc_ldrw(dst, mem));
6653
6654 ins_pipe(iload_reg_mem);
6655 %}
6656
6657 // Load Klass Pointer
6658 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6659 %{
6660 match(Set dst (LoadKlass mem));
6661 predicate(!needs_acquiring_load(n));
6662
6663 ins_cost(4 * INSN_COST);
6664 format %{ "ldr $dst, $mem\t# class" %}
6665
6666 ins_encode(aarch64_enc_ldr(dst, mem));
6667
6668 ins_pipe(iload_reg_mem);
6669 %}
6670
6671 // Load Narrow Klass Pointer
6672 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6673 %{
6674 match(Set dst (LoadNKlass mem));
6675 predicate(!needs_acquiring_load(n));
6676
6677 ins_cost(4 * INSN_COST);
6678 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6679
6680 ins_encode(aarch64_enc_ldrw(dst, mem));
6681
6682 ins_pipe(iload_reg_mem);
6683 %}
6684
6685 // Load Float
6686 instruct loadF(vRegF dst, memory4 mem)
6687 %{
6688 match(Set dst (LoadF mem));
6689 predicate(!needs_acquiring_load(n));
6690
6691 ins_cost(4 * INSN_COST);
6692 format %{ "ldrs $dst, $mem\t# float" %}
6693
6694 ins_encode( aarch64_enc_ldrs(dst, mem) );
6695
6696 ins_pipe(pipe_class_memory);
6697 %}
6698
6699 // Load Double
6700 instruct loadD(vRegD dst, memory8 mem)
6701 %{
6702 match(Set dst (LoadD mem));
6703 predicate(!needs_acquiring_load(n));
6704
6705 ins_cost(4 * INSN_COST);
6706 format %{ "ldrd $dst, $mem\t# double" %}
6707
6708 ins_encode( aarch64_enc_ldrd(dst, mem) );
6709
6710 ins_pipe(pipe_class_memory);
6711 %}
6712
6713
6714 // Load Int Constant
6715 instruct loadConI(iRegINoSp dst, immI src)
6716 %{
6717 match(Set dst src);
6718
6719 ins_cost(INSN_COST);
6720 format %{ "mov $dst, $src\t# int" %}
6721
6722 ins_encode( aarch64_enc_movw_imm(dst, src) );
6723
6724 ins_pipe(ialu_imm);
6725 %}
6726
6727 // Load Long Constant
6728 instruct loadConL(iRegLNoSp dst, immL src)
6729 %{
6730 match(Set dst src);
6731
6732 ins_cost(INSN_COST);
6733 format %{ "mov $dst, $src\t# long" %}
6734
6735 ins_encode( aarch64_enc_mov_imm(dst, src) );
6736
6737 ins_pipe(ialu_imm);
6738 %}
6739
6740 // Load Pointer Constant
6741
6742 instruct loadConP(iRegPNoSp dst, immP con)
6743 %{
6744 match(Set dst con);
6745
6746 ins_cost(INSN_COST * 4);
6747 format %{
6748 "mov $dst, $con\t# ptr\n\t"
6749 %}
6750
6751 ins_encode(aarch64_enc_mov_p(dst, con));
6752
6753 ins_pipe(ialu_imm);
6754 %}
6755
6756 // Load Null Pointer Constant
6757
6758 instruct loadConP0(iRegPNoSp dst, immP0 con)
6759 %{
6760 match(Set dst con);
6761
6762 ins_cost(INSN_COST);
6763 format %{ "mov $dst, $con\t# nullptr ptr" %}
6764
6765 ins_encode(aarch64_enc_mov_p0(dst, con));
6766
6767 ins_pipe(ialu_imm);
6768 %}
6769
6770 // Load Pointer Constant One
6771
6772 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6773 %{
6774 match(Set dst con);
6775
6776 ins_cost(INSN_COST);
6777 format %{ "mov $dst, $con\t# nullptr ptr" %}
6778
6779 ins_encode(aarch64_enc_mov_p1(dst, con));
6780
6781 ins_pipe(ialu_imm);
6782 %}
6783
6784 // Load Byte Map Base Constant
6785
6786 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6787 %{
6788 match(Set dst con);
6789
6790 ins_cost(INSN_COST);
6791 format %{ "adr $dst, $con\t# Byte Map Base" %}
6792
6793 ins_encode %{
6794 __ load_byte_map_base($dst$$Register);
6795 %}
6796
6797 ins_pipe(ialu_imm);
6798 %}
6799
6800 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6801 %{
6802 match(Set dst con);
6803
6804 ins_cost(INSN_COST);
6805 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6806
6807 ins_encode %{
6808 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6809 %}
6810
6811 ins_pipe(ialu_imm);
6812 %}
6813
6814 // Load Narrow Pointer Constant
6815
6816 instruct loadConN(iRegNNoSp dst, immN con)
6817 %{
6818 match(Set dst con);
6819
6820 ins_cost(INSN_COST * 4);
6821 format %{ "mov $dst, $con\t# compressed ptr" %}
6822
6823 ins_encode(aarch64_enc_mov_n(dst, con));
6824
6825 ins_pipe(ialu_imm);
6826 %}
6827
6828 // Load Narrow Null Pointer Constant
6829
6830 instruct loadConN0(iRegNNoSp dst, immN0 con)
6831 %{
6832 match(Set dst con);
6833
6834 ins_cost(INSN_COST);
6835 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6836
6837 ins_encode(aarch64_enc_mov_n0(dst, con));
6838
6839 ins_pipe(ialu_imm);
6840 %}
6841
6842 // Load Narrow Klass Constant
6843
6844 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6845 %{
6846 match(Set dst con);
6847
6848 ins_cost(INSN_COST);
6849 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6850
6851 ins_encode(aarch64_enc_mov_nk(dst, con));
6852
6853 ins_pipe(ialu_imm);
6854 %}
6855
6856 // Load Packed Float Constant
6857
6858 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6859 match(Set dst con);
6860 ins_cost(INSN_COST * 4);
6861 format %{ "fmovs $dst, $con"%}
6862 ins_encode %{
6863 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6864 %}
6865
6866 ins_pipe(fp_imm_s);
6867 %}
6868
6869 // Load Float Constant
6870
6871 instruct loadConF(vRegF dst, immF con) %{
6872 match(Set dst con);
6873
6874 ins_cost(INSN_COST * 4);
6875
6876 format %{
6877 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6878 %}
6879
6880 ins_encode %{
6881 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6882 %}
6883
6884 ins_pipe(fp_load_constant_s);
6885 %}
6886
6887 // Load Packed Double Constant
6888
6889 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6890 match(Set dst con);
6891 ins_cost(INSN_COST);
6892 format %{ "fmovd $dst, $con"%}
6893 ins_encode %{
6894 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6895 %}
6896
6897 ins_pipe(fp_imm_d);
6898 %}
6899
6900 // Load Double Constant
6901
6902 instruct loadConD(vRegD dst, immD con) %{
6903 match(Set dst con);
6904
6905 ins_cost(INSN_COST * 5);
6906 format %{
6907 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6908 %}
6909
6910 ins_encode %{
6911 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6912 %}
6913
6914 ins_pipe(fp_load_constant_d);
6915 %}
6916
6917 // Store Instructions
6918
6919 // Store CMS card-mark Immediate
6920 instruct storeimmCM0(immI0 zero, memory1 mem)
6921 %{
6922 match(Set mem (StoreCM mem zero));
6923
6924 ins_cost(INSN_COST);
6925 format %{ "storestore (elided)\n\t"
6926 "strb zr, $mem\t# byte" %}
6927
6928 ins_encode(aarch64_enc_strb0(mem));
6929
6930 ins_pipe(istore_mem);
6931 %}
6932
6933 // Store CMS card-mark Immediate with intervening StoreStore
6934 // needed when using CMS with no conditional card marking
6935 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
6936 %{
6937 match(Set mem (StoreCM mem zero));
6938
6939 ins_cost(INSN_COST * 2);
6940 format %{ "storestore\n\t"
6941 "dmb ishst"
6942 "\n\tstrb zr, $mem\t# byte" %}
6943
6944 ins_encode(aarch64_enc_strb0_ordered(mem));
6945
6946 ins_pipe(istore_mem);
6947 %}
6948
6949 // Store Byte
6950 instruct storeB(iRegIorL2I src, memory1 mem)
6951 %{
6952 match(Set mem (StoreB mem src));
6953 predicate(!needs_releasing_store(n));
6954
6955 ins_cost(INSN_COST);
6956 format %{ "strb $src, $mem\t# byte" %}
6957
6958 ins_encode(aarch64_enc_strb(src, mem));
6959
6960 ins_pipe(istore_reg_mem);
6961 %}
6962
6963
6964 instruct storeimmB0(immI0 zero, memory1 mem)
6965 %{
6966 match(Set mem (StoreB mem zero));
6967 predicate(!needs_releasing_store(n));
6968
6969 ins_cost(INSN_COST);
6970 format %{ "strb rscractch2, $mem\t# byte" %}
6971
6972 ins_encode(aarch64_enc_strb0(mem));
6973
6974 ins_pipe(istore_mem);
6975 %}
6976
6977 // Store Char/Short
6978 instruct storeC(iRegIorL2I src, memory2 mem)
6979 %{
6980 match(Set mem (StoreC mem src));
6981 predicate(!needs_releasing_store(n));
6982
6983 ins_cost(INSN_COST);
6984 format %{ "strh $src, $mem\t# short" %}
6985
6986 ins_encode(aarch64_enc_strh(src, mem));
6987
6988 ins_pipe(istore_reg_mem);
6989 %}
6990
6991 instruct storeimmC0(immI0 zero, memory2 mem)
6992 %{
6993 match(Set mem (StoreC mem zero));
6994 predicate(!needs_releasing_store(n));
6995
6996 ins_cost(INSN_COST);
6997 format %{ "strh zr, $mem\t# short" %}
6998
6999 ins_encode(aarch64_enc_strh0(mem));
7000
7001 ins_pipe(istore_mem);
7002 %}
7003
7004 // Store Integer
7005
7006 instruct storeI(iRegIorL2I src, memory4 mem)
7007 %{
7008 match(Set mem(StoreI mem src));
7009 predicate(!needs_releasing_store(n));
7010
7011 ins_cost(INSN_COST);
7012 format %{ "strw $src, $mem\t# int" %}
7013
7014 ins_encode(aarch64_enc_strw(src, mem));
7015
7016 ins_pipe(istore_reg_mem);
7017 %}
7018
7019 instruct storeimmI0(immI0 zero, memory4 mem)
7020 %{
7021 match(Set mem(StoreI mem zero));
7022 predicate(!needs_releasing_store(n));
7023
7024 ins_cost(INSN_COST);
7025 format %{ "strw zr, $mem\t# int" %}
7026
7027 ins_encode(aarch64_enc_strw0(mem));
7028
7029 ins_pipe(istore_mem);
7030 %}
7031
7032 // Store Long (64 bit signed)
7033 instruct storeL(iRegL src, memory8 mem)
7034 %{
7035 match(Set mem (StoreL mem src));
7036 predicate(!needs_releasing_store(n));
7037
7038 ins_cost(INSN_COST);
7039 format %{ "str $src, $mem\t# int" %}
7040
7041 ins_encode(aarch64_enc_str(src, mem));
7042
7043 ins_pipe(istore_reg_mem);
7044 %}
7045
7046 // Store Long (64 bit signed)
7047 instruct storeimmL0(immL0 zero, memory8 mem)
7048 %{
7049 match(Set mem (StoreL mem zero));
7050 predicate(!needs_releasing_store(n));
7051
7052 ins_cost(INSN_COST);
7053 format %{ "str zr, $mem\t# int" %}
7054
7055 ins_encode(aarch64_enc_str0(mem));
7056
7057 ins_pipe(istore_mem);
7058 %}
7059
7060 // Store Pointer
7061 instruct storeP(iRegP src, memory8 mem)
7062 %{
7063 match(Set mem (StoreP mem src));
7064 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7065
7066 ins_cost(INSN_COST);
7067 format %{ "str $src, $mem\t# ptr" %}
7068
7069 ins_encode(aarch64_enc_str(src, mem));
7070
7071 ins_pipe(istore_reg_mem);
7072 %}
7073
7074 // Store Pointer
7075 instruct storeimmP0(immP0 zero, memory8 mem)
7076 %{
7077 match(Set mem (StoreP mem zero));
7078 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7079
7080 ins_cost(INSN_COST);
7081 format %{ "str zr, $mem\t# ptr" %}
7082
7083 ins_encode(aarch64_enc_str0(mem));
7084
7085 ins_pipe(istore_mem);
7086 %}
7087
7088 // Store Compressed Pointer
7089 instruct storeN(iRegN src, memory4 mem)
7090 %{
7091 match(Set mem (StoreN mem src));
7092 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7093
7094 ins_cost(INSN_COST);
7095 format %{ "strw $src, $mem\t# compressed ptr" %}
7096
7097 ins_encode(aarch64_enc_strw(src, mem));
7098
7099 ins_pipe(istore_reg_mem);
7100 %}
7101
7102 instruct storeImmN0(immN0 zero, memory4 mem)
7103 %{
7104 match(Set mem (StoreN mem zero));
7105 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7106
7107 ins_cost(INSN_COST);
7108 format %{ "strw zr, $mem\t# compressed ptr" %}
7109
7110 ins_encode(aarch64_enc_strw0(mem));
7111
7112 ins_pipe(istore_mem);
7113 %}
7114
7115 // Store Float
7116 instruct storeF(vRegF src, memory4 mem)
7117 %{
7118 match(Set mem (StoreF mem src));
7119 predicate(!needs_releasing_store(n));
7120
7121 ins_cost(INSN_COST);
7122 format %{ "strs $src, $mem\t# float" %}
7123
7124 ins_encode( aarch64_enc_strs(src, mem) );
7125
7126 ins_pipe(pipe_class_memory);
7127 %}
7128
7129 // TODO
7130 // implement storeImmF0 and storeFImmPacked
7131
7132 // Store Double
7133 instruct storeD(vRegD src, memory8 mem)
7134 %{
7135 match(Set mem (StoreD mem src));
7136 predicate(!needs_releasing_store(n));
7137
7138 ins_cost(INSN_COST);
7139 format %{ "strd $src, $mem\t# double" %}
7140
7141 ins_encode( aarch64_enc_strd(src, mem) );
7142
7143 ins_pipe(pipe_class_memory);
7144 %}
7145
7146 // Store Compressed Klass Pointer
7147 instruct storeNKlass(iRegN src, memory4 mem)
7148 %{
7149 predicate(!needs_releasing_store(n));
7150 match(Set mem (StoreNKlass mem src));
7151
7152 ins_cost(INSN_COST);
7153 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7154
7155 ins_encode(aarch64_enc_strw(src, mem));
7156
7157 ins_pipe(istore_reg_mem);
7158 %}
7159
7160 // TODO
7161 // implement storeImmD0 and storeDImmPacked
7162
7163 // prefetch instructions
7164 // Must be safe to execute with invalid address (cannot fault).
7165
7166 instruct prefetchalloc( memory8 mem ) %{
7167 match(PrefetchAllocation mem);
7168
7169 ins_cost(INSN_COST);
7170 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7171
7172 ins_encode( aarch64_enc_prefetchw(mem) );
7173
7174 ins_pipe(iload_prefetch);
7175 %}
7176
7177 // ---------------- volatile loads and stores ----------------
7178
7179 // Load Byte (8 bit signed)
7180 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7181 %{
7182 match(Set dst (LoadB mem));
7183
7184 ins_cost(VOLATILE_REF_COST);
7185 format %{ "ldarsb $dst, $mem\t# byte" %}
7186
7187 ins_encode(aarch64_enc_ldarsb(dst, mem));
7188
7189 ins_pipe(pipe_serial);
7190 %}
7191
7192 // Load Byte (8 bit signed) into long
7193 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7194 %{
7195 match(Set dst (ConvI2L (LoadB mem)));
7196
7197 ins_cost(VOLATILE_REF_COST);
7198 format %{ "ldarsb $dst, $mem\t# byte" %}
7199
7200 ins_encode(aarch64_enc_ldarsb(dst, mem));
7201
7202 ins_pipe(pipe_serial);
7203 %}
7204
7205 // Load Byte (8 bit unsigned)
7206 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7207 %{
7208 match(Set dst (LoadUB mem));
7209
7210 ins_cost(VOLATILE_REF_COST);
7211 format %{ "ldarb $dst, $mem\t# byte" %}
7212
7213 ins_encode(aarch64_enc_ldarb(dst, mem));
7214
7215 ins_pipe(pipe_serial);
7216 %}
7217
7218 // Load Byte (8 bit unsigned) into long
7219 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7220 %{
7221 match(Set dst (ConvI2L (LoadUB mem)));
7222
7223 ins_cost(VOLATILE_REF_COST);
7224 format %{ "ldarb $dst, $mem\t# byte" %}
7225
7226 ins_encode(aarch64_enc_ldarb(dst, mem));
7227
7228 ins_pipe(pipe_serial);
7229 %}
7230
7231 // Load Short (16 bit signed)
7232 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7233 %{
7234 match(Set dst (LoadS mem));
7235
7236 ins_cost(VOLATILE_REF_COST);
7237 format %{ "ldarshw $dst, $mem\t# short" %}
7238
7239 ins_encode(aarch64_enc_ldarshw(dst, mem));
7240
7241 ins_pipe(pipe_serial);
7242 %}
7243
7244 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7245 %{
7246 match(Set dst (LoadUS mem));
7247
7248 ins_cost(VOLATILE_REF_COST);
7249 format %{ "ldarhw $dst, $mem\t# short" %}
7250
7251 ins_encode(aarch64_enc_ldarhw(dst, mem));
7252
7253 ins_pipe(pipe_serial);
7254 %}
7255
7256 // Load Short/Char (16 bit unsigned) into long
7257 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7258 %{
7259 match(Set dst (ConvI2L (LoadUS mem)));
7260
7261 ins_cost(VOLATILE_REF_COST);
7262 format %{ "ldarh $dst, $mem\t# short" %}
7263
7264 ins_encode(aarch64_enc_ldarh(dst, mem));
7265
7266 ins_pipe(pipe_serial);
7267 %}
7268
7269 // Load Short/Char (16 bit signed) into long
7270 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (ConvI2L (LoadS mem)));
7273
7274 ins_cost(VOLATILE_REF_COST);
7275 format %{ "ldarh $dst, $mem\t# short" %}
7276
7277 ins_encode(aarch64_enc_ldarsh(dst, mem));
7278
7279 ins_pipe(pipe_serial);
7280 %}
7281
7282 // Load Integer (32 bit signed)
7283 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7284 %{
7285 match(Set dst (LoadI mem));
7286
7287 ins_cost(VOLATILE_REF_COST);
7288 format %{ "ldarw $dst, $mem\t# int" %}
7289
7290 ins_encode(aarch64_enc_ldarw(dst, mem));
7291
7292 ins_pipe(pipe_serial);
7293 %}
7294
7295 // Load Integer (32 bit unsigned) into long
7296 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7297 %{
7298 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7299
7300 ins_cost(VOLATILE_REF_COST);
7301 format %{ "ldarw $dst, $mem\t# int" %}
7302
7303 ins_encode(aarch64_enc_ldarw(dst, mem));
7304
7305 ins_pipe(pipe_serial);
7306 %}
7307
7308 // Load Long (64 bit signed)
7309 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7310 %{
7311 match(Set dst (LoadL mem));
7312
7313 ins_cost(VOLATILE_REF_COST);
7314 format %{ "ldar $dst, $mem\t# int" %}
7315
7316 ins_encode(aarch64_enc_ldar(dst, mem));
7317
7318 ins_pipe(pipe_serial);
7319 %}
7320
7321 // Load Pointer
7322 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7323 %{
7324 match(Set dst (LoadP mem));
7325 predicate(n->as_Load()->barrier_data() == 0);
7326
7327 ins_cost(VOLATILE_REF_COST);
7328 format %{ "ldar $dst, $mem\t# ptr" %}
7329
7330 ins_encode(aarch64_enc_ldar(dst, mem));
7331
7332 ins_pipe(pipe_serial);
7333 %}
7334
7335 // Load Compressed Pointer
7336 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7337 %{
7338 match(Set dst (LoadN mem));
7339 predicate(n->as_Load()->barrier_data() == 0);
7340
7341 ins_cost(VOLATILE_REF_COST);
7342 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7343
7344 ins_encode(aarch64_enc_ldarw(dst, mem));
7345
7346 ins_pipe(pipe_serial);
7347 %}
7348
7349 // Load Float
7350 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7351 %{
7352 match(Set dst (LoadF mem));
7353
7354 ins_cost(VOLATILE_REF_COST);
7355 format %{ "ldars $dst, $mem\t# float" %}
7356
7357 ins_encode( aarch64_enc_fldars(dst, mem) );
7358
7359 ins_pipe(pipe_serial);
7360 %}
7361
7362 // Load Double
7363 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7364 %{
7365 match(Set dst (LoadD mem));
7366
7367 ins_cost(VOLATILE_REF_COST);
7368 format %{ "ldard $dst, $mem\t# double" %}
7369
7370 ins_encode( aarch64_enc_fldard(dst, mem) );
7371
7372 ins_pipe(pipe_serial);
7373 %}
7374
7375 // Store Byte
7376 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7377 %{
7378 match(Set mem (StoreB mem src));
7379
7380 ins_cost(VOLATILE_REF_COST);
7381 format %{ "stlrb $src, $mem\t# byte" %}
7382
7383 ins_encode(aarch64_enc_stlrb(src, mem));
7384
7385 ins_pipe(pipe_class_memory);
7386 %}
7387
7388 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7389 %{
7390 match(Set mem (StoreB mem zero));
7391
7392 ins_cost(VOLATILE_REF_COST);
7393 format %{ "stlrb zr, $mem\t# byte" %}
7394
7395 ins_encode(aarch64_enc_stlrb0(mem));
7396
7397 ins_pipe(pipe_class_memory);
7398 %}
7399
7400 // Store Char/Short
7401 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7402 %{
7403 match(Set mem (StoreC mem src));
7404
7405 ins_cost(VOLATILE_REF_COST);
7406 format %{ "stlrh $src, $mem\t# short" %}
7407
7408 ins_encode(aarch64_enc_stlrh(src, mem));
7409
7410 ins_pipe(pipe_class_memory);
7411 %}
7412
7413 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7414 %{
7415 match(Set mem (StoreC mem zero));
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "stlrh zr, $mem\t# short" %}
7419
7420 ins_encode(aarch64_enc_stlrh0(mem));
7421
7422 ins_pipe(pipe_class_memory);
7423 %}
7424
7425 // Store Integer
7426
7427 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7428 %{
7429 match(Set mem(StoreI mem src));
7430
7431 ins_cost(VOLATILE_REF_COST);
7432 format %{ "stlrw $src, $mem\t# int" %}
7433
7434 ins_encode(aarch64_enc_stlrw(src, mem));
7435
7436 ins_pipe(pipe_class_memory);
7437 %}
7438
7439 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set mem(StoreI mem zero));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "stlrw zr, $mem\t# int" %}
7445
7446 ins_encode(aarch64_enc_stlrw0(mem));
7447
7448 ins_pipe(pipe_class_memory);
7449 %}
7450
7451 // Store Long (64 bit signed)
7452 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set mem (StoreL mem src));
7455
7456 ins_cost(VOLATILE_REF_COST);
7457 format %{ "stlr $src, $mem\t# int" %}
7458
7459 ins_encode(aarch64_enc_stlr(src, mem));
7460
7461 ins_pipe(pipe_class_memory);
7462 %}
7463
7464 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set mem (StoreL mem zero));
7467
7468 ins_cost(VOLATILE_REF_COST);
7469 format %{ "stlr zr, $mem\t# int" %}
7470
7471 ins_encode(aarch64_enc_stlr0(mem));
7472
7473 ins_pipe(pipe_class_memory);
7474 %}
7475
7476 // Store Pointer
7477 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreP mem src));
7480 predicate(n->as_Store()->barrier_data() == 0);
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlr $src, $mem\t# ptr" %}
7484
7485 ins_encode(aarch64_enc_stlr(src, mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7491 %{
7492 match(Set mem (StoreP mem zero));
7493 predicate(n->as_Store()->barrier_data() == 0);
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "stlr zr, $mem\t# ptr" %}
7497
7498 ins_encode(aarch64_enc_stlr0(mem));
7499
7500 ins_pipe(pipe_class_memory);
7501 %}
7502
7503 // Store Compressed Pointer
7504 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem (StoreN mem src));
7507 predicate(n->as_Store()->barrier_data() == 0);
7508
7509 ins_cost(VOLATILE_REF_COST);
7510 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7511
7512 ins_encode(aarch64_enc_stlrw(src, mem));
7513
7514 ins_pipe(pipe_class_memory);
7515 %}
7516
7517 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7518 %{
7519 match(Set mem (StoreN mem zero));
7520 predicate(n->as_Store()->barrier_data() == 0);
7521
7522 ins_cost(VOLATILE_REF_COST);
7523 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7524
7525 ins_encode(aarch64_enc_stlrw0(mem));
7526
7527 ins_pipe(pipe_class_memory);
7528 %}
7529
7530 // Store Float
7531 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreF mem src));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlrs $src, $mem\t# float" %}
7537
7538 ins_encode( aarch64_enc_fstlrs(src, mem) );
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 // TODO
7544 // implement storeImmF0 and storeFImmPacked
7545
7546 // Store Double
7547 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7548 %{
7549 match(Set mem (StoreD mem src));
7550
7551 ins_cost(VOLATILE_REF_COST);
7552 format %{ "stlrd $src, $mem\t# double" %}
7553
7554 ins_encode( aarch64_enc_fstlrd(src, mem) );
7555
7556 ins_pipe(pipe_class_memory);
7557 %}
7558
7559 // ---------------- end of volatile loads and stores ----------------
7560
7561 instruct cacheWB(indirect addr)
7562 %{
7563 predicate(VM_Version::supports_data_cache_line_flush());
7564 match(CacheWB addr);
7565
7566 ins_cost(100);
7567 format %{"cache wb $addr" %}
7568 ins_encode %{
7569 assert($addr->index_position() < 0, "should be");
7570 assert($addr$$disp == 0, "should be");
7571 __ cache_wb(Address($addr$$base$$Register, 0));
7572 %}
7573 ins_pipe(pipe_slow); // XXX
7574 %}
7575
7576 instruct cacheWBPreSync()
7577 %{
7578 predicate(VM_Version::supports_data_cache_line_flush());
7579 match(CacheWBPreSync);
7580
7581 ins_cost(100);
7582 format %{"cache wb presync" %}
7583 ins_encode %{
7584 __ cache_wbsync(true);
7585 %}
7586 ins_pipe(pipe_slow); // XXX
7587 %}
7588
7589 instruct cacheWBPostSync()
7590 %{
7591 predicate(VM_Version::supports_data_cache_line_flush());
7592 match(CacheWBPostSync);
7593
7594 ins_cost(100);
7595 format %{"cache wb postsync" %}
7596 ins_encode %{
7597 __ cache_wbsync(false);
7598 %}
7599 ins_pipe(pipe_slow); // XXX
7600 %}
7601
7602 // ============================================================================
7603 // BSWAP Instructions
7604
7605 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7606 match(Set dst (ReverseBytesI src));
7607
7608 ins_cost(INSN_COST);
7609 format %{ "revw $dst, $src" %}
7610
7611 ins_encode %{
7612 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7613 %}
7614
7615 ins_pipe(ialu_reg);
7616 %}
7617
7618 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7619 match(Set dst (ReverseBytesL src));
7620
7621 ins_cost(INSN_COST);
7622 format %{ "rev $dst, $src" %}
7623
7624 ins_encode %{
7625 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7626 %}
7627
7628 ins_pipe(ialu_reg);
7629 %}
7630
7631 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7632 match(Set dst (ReverseBytesUS src));
7633
7634 ins_cost(INSN_COST);
7635 format %{ "rev16w $dst, $src" %}
7636
7637 ins_encode %{
7638 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7639 %}
7640
7641 ins_pipe(ialu_reg);
7642 %}
7643
7644 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7645 match(Set dst (ReverseBytesS src));
7646
7647 ins_cost(INSN_COST);
7648 format %{ "rev16w $dst, $src\n\t"
7649 "sbfmw $dst, $dst, #0, #15" %}
7650
7651 ins_encode %{
7652 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7653 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7654 %}
7655
7656 ins_pipe(ialu_reg);
7657 %}
7658
7659 // ============================================================================
7660 // Zero Count Instructions
7661
7662 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7663 match(Set dst (CountLeadingZerosI src));
7664
7665 ins_cost(INSN_COST);
7666 format %{ "clzw $dst, $src" %}
7667 ins_encode %{
7668 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7669 %}
7670
7671 ins_pipe(ialu_reg);
7672 %}
7673
7674 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7675 match(Set dst (CountLeadingZerosL src));
7676
7677 ins_cost(INSN_COST);
7678 format %{ "clz $dst, $src" %}
7679 ins_encode %{
7680 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7681 %}
7682
7683 ins_pipe(ialu_reg);
7684 %}
7685
7686 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7687 match(Set dst (CountTrailingZerosI src));
7688
7689 ins_cost(INSN_COST * 2);
7690 format %{ "rbitw $dst, $src\n\t"
7691 "clzw $dst, $dst" %}
7692 ins_encode %{
7693 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7694 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7695 %}
7696
7697 ins_pipe(ialu_reg);
7698 %}
7699
7700 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7701 match(Set dst (CountTrailingZerosL src));
7702
7703 ins_cost(INSN_COST * 2);
7704 format %{ "rbit $dst, $src\n\t"
7705 "clz $dst, $dst" %}
7706 ins_encode %{
7707 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7708 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7709 %}
7710
7711 ins_pipe(ialu_reg);
7712 %}
7713
7714 //---------- Population Count Instructions -------------------------------------
7715 //
7716
7717 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7718 match(Set dst (PopCountI src));
7719 effect(TEMP tmp);
7720 ins_cost(INSN_COST * 13);
7721
7722 format %{ "movw $src, $src\n\t"
7723 "mov $tmp, $src\t# vector (1D)\n\t"
7724 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7725 "addv $tmp, $tmp\t# vector (8B)\n\t"
7726 "mov $dst, $tmp\t# vector (1D)" %}
7727 ins_encode %{
7728 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7729 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7730 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7731 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7732 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7733 %}
7734
7735 ins_pipe(pipe_class_default);
7736 %}
7737
7738 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7739 match(Set dst (PopCountI (LoadI mem)));
7740 effect(TEMP tmp);
7741 ins_cost(INSN_COST * 13);
7742
7743 format %{ "ldrs $tmp, $mem\n\t"
7744 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7745 "addv $tmp, $tmp\t# vector (8B)\n\t"
7746 "mov $dst, $tmp\t# vector (1D)" %}
7747 ins_encode %{
7748 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7749 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7750 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7751 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7752 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7753 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7754 %}
7755
7756 ins_pipe(pipe_class_default);
7757 %}
7758
7759 // Note: Long.bitCount(long) returns an int.
7760 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7761 match(Set dst (PopCountL src));
7762 effect(TEMP tmp);
7763 ins_cost(INSN_COST * 13);
7764
7765 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7766 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7767 "addv $tmp, $tmp\t# vector (8B)\n\t"
7768 "mov $dst, $tmp\t# vector (1D)" %}
7769 ins_encode %{
7770 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7771 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7772 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7773 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7774 %}
7775
7776 ins_pipe(pipe_class_default);
7777 %}
7778
7779 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7780 match(Set dst (PopCountL (LoadL mem)));
7781 effect(TEMP tmp);
7782 ins_cost(INSN_COST * 13);
7783
7784 format %{ "ldrd $tmp, $mem\n\t"
7785 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7786 "addv $tmp, $tmp\t# vector (8B)\n\t"
7787 "mov $dst, $tmp\t# vector (1D)" %}
7788 ins_encode %{
7789 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7790 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7791 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7792 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7793 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7794 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7795 %}
7796
7797 ins_pipe(pipe_class_default);
7798 %}
7799
7800 // ============================================================================
7801 // VerifyVectorAlignment Instruction
7802
7803 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7804 match(Set addr (VerifyVectorAlignment addr mask));
7805 effect(KILL cr);
7806 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7807 ins_encode %{
7808 Label Lskip;
7809 // check if masked bits of addr are zero
7810 __ tst($addr$$Register, $mask$$constant);
7811 __ br(Assembler::EQ, Lskip);
7812 __ stop("verify_vector_alignment found a misaligned vector memory access");
7813 __ bind(Lskip);
7814 %}
7815 ins_pipe(pipe_slow);
7816 %}
7817
7818 // ============================================================================
7819 // MemBar Instruction
7820
7821 instruct load_fence() %{
7822 match(LoadFence);
7823 ins_cost(VOLATILE_REF_COST);
7824
7825 format %{ "load_fence" %}
7826
7827 ins_encode %{
7828 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7829 %}
7830 ins_pipe(pipe_serial);
7831 %}
7832
7833 instruct unnecessary_membar_acquire() %{
7834 predicate(unnecessary_acquire(n));
7835 match(MemBarAcquire);
7836 ins_cost(0);
7837
7838 format %{ "membar_acquire (elided)" %}
7839
7840 ins_encode %{
7841 __ block_comment("membar_acquire (elided)");
7842 %}
7843
7844 ins_pipe(pipe_class_empty);
7845 %}
7846
7847 instruct membar_acquire() %{
7848 match(MemBarAcquire);
7849 ins_cost(VOLATILE_REF_COST);
7850
7851 format %{ "membar_acquire\n\t"
7852 "dmb ishld" %}
7853
7854 ins_encode %{
7855 __ block_comment("membar_acquire");
7856 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7857 %}
7858
7859 ins_pipe(pipe_serial);
7860 %}
7861
7862
7863 instruct membar_acquire_lock() %{
7864 match(MemBarAcquireLock);
7865 ins_cost(VOLATILE_REF_COST);
7866
7867 format %{ "membar_acquire_lock (elided)" %}
7868
7869 ins_encode %{
7870 __ block_comment("membar_acquire_lock (elided)");
7871 %}
7872
7873 ins_pipe(pipe_serial);
7874 %}
7875
7876 instruct store_fence() %{
7877 match(StoreFence);
7878 ins_cost(VOLATILE_REF_COST);
7879
7880 format %{ "store_fence" %}
7881
7882 ins_encode %{
7883 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7884 %}
7885 ins_pipe(pipe_serial);
7886 %}
7887
7888 instruct unnecessary_membar_release() %{
7889 predicate(unnecessary_release(n));
7890 match(MemBarRelease);
7891 ins_cost(0);
7892
7893 format %{ "membar_release (elided)" %}
7894
7895 ins_encode %{
7896 __ block_comment("membar_release (elided)");
7897 %}
7898 ins_pipe(pipe_serial);
7899 %}
7900
7901 instruct membar_release() %{
7902 match(MemBarRelease);
7903 ins_cost(VOLATILE_REF_COST);
7904
7905 format %{ "membar_release\n\t"
7906 "dmb ishst\n\tdmb ishld" %}
7907
7908 ins_encode %{
7909 __ block_comment("membar_release");
7910 // These will be merged if AlwaysMergeDMB is enabled.
7911 __ membar(Assembler::StoreStore);
7912 __ membar(Assembler::LoadStore);
7913 %}
7914 ins_pipe(pipe_serial);
7915 %}
7916
7917 instruct membar_storestore() %{
7918 match(MemBarStoreStore);
7919 match(StoreStoreFence);
7920 ins_cost(VOLATILE_REF_COST);
7921
7922 format %{ "MEMBAR-store-store" %}
7923
7924 ins_encode %{
7925 __ membar(Assembler::StoreStore);
7926 %}
7927 ins_pipe(pipe_serial);
7928 %}
7929
7930 instruct membar_release_lock() %{
7931 match(MemBarReleaseLock);
7932 ins_cost(VOLATILE_REF_COST);
7933
7934 format %{ "membar_release_lock (elided)" %}
7935
7936 ins_encode %{
7937 __ block_comment("membar_release_lock (elided)");
7938 %}
7939
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 instruct unnecessary_membar_volatile() %{
7944 predicate(unnecessary_volatile(n));
7945 match(MemBarVolatile);
7946 ins_cost(0);
7947
7948 format %{ "membar_volatile (elided)" %}
7949
7950 ins_encode %{
7951 __ block_comment("membar_volatile (elided)");
7952 %}
7953
7954 ins_pipe(pipe_serial);
7955 %}
7956
7957 instruct membar_volatile() %{
7958 match(MemBarVolatile);
7959 ins_cost(VOLATILE_REF_COST*100);
7960
7961 format %{ "membar_volatile\n\t"
7962 "dmb ish"%}
7963
7964 ins_encode %{
7965 __ block_comment("membar_volatile");
7966 __ membar(Assembler::StoreLoad);
7967 %}
7968
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 // ============================================================================
7973 // Cast/Convert Instructions
7974
7975 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7976 match(Set dst (CastX2P src));
7977
7978 ins_cost(INSN_COST);
7979 format %{ "mov $dst, $src\t# long -> ptr" %}
7980
7981 ins_encode %{
7982 if ($dst$$reg != $src$$reg) {
7983 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7984 }
7985 %}
7986
7987 ins_pipe(ialu_reg);
7988 %}
7989
7990 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7991 match(Set dst (CastP2X src));
7992
7993 ins_cost(INSN_COST);
7994 format %{ "mov $dst, $src\t# ptr -> long" %}
7995
7996 ins_encode %{
7997 if ($dst$$reg != $src$$reg) {
7998 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7999 }
8000 %}
8001
8002 ins_pipe(ialu_reg);
8003 %}
8004
8005 // Convert oop into int for vectors alignment masking
8006 instruct convP2I(iRegINoSp dst, iRegP src) %{
8007 match(Set dst (ConvL2I (CastP2X src)));
8008
8009 ins_cost(INSN_COST);
8010 format %{ "movw $dst, $src\t# ptr -> int" %}
8011 ins_encode %{
8012 __ movw($dst$$Register, $src$$Register);
8013 %}
8014
8015 ins_pipe(ialu_reg);
8016 %}
8017
8018 // Convert compressed oop into int for vectors alignment masking
8019 // in case of 32bit oops (heap < 4Gb).
8020 instruct convN2I(iRegINoSp dst, iRegN src)
8021 %{
8022 predicate(CompressedOops::shift() == 0);
8023 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8024
8025 ins_cost(INSN_COST);
8026 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8027 ins_encode %{
8028 __ movw($dst$$Register, $src$$Register);
8029 %}
8030
8031 ins_pipe(ialu_reg);
8032 %}
8033
8034
8035 // Convert oop pointer into compressed form
8036 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8037 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8038 match(Set dst (EncodeP src));
8039 effect(KILL cr);
8040 ins_cost(INSN_COST * 3);
8041 format %{ "encode_heap_oop $dst, $src" %}
8042 ins_encode %{
8043 Register s = $src$$Register;
8044 Register d = $dst$$Register;
8045 __ encode_heap_oop(d, s);
8046 %}
8047 ins_pipe(ialu_reg);
8048 %}
8049
8050 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8051 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8052 match(Set dst (EncodeP src));
8053 ins_cost(INSN_COST * 3);
8054 format %{ "encode_heap_oop_not_null $dst, $src" %}
8055 ins_encode %{
8056 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8057 %}
8058 ins_pipe(ialu_reg);
8059 %}
8060
8061 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8062 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8063 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8064 match(Set dst (DecodeN src));
8065 ins_cost(INSN_COST * 3);
8066 format %{ "decode_heap_oop $dst, $src" %}
8067 ins_encode %{
8068 Register s = $src$$Register;
8069 Register d = $dst$$Register;
8070 __ decode_heap_oop(d, s);
8071 %}
8072 ins_pipe(ialu_reg);
8073 %}
8074
8075 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8076 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8077 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8078 match(Set dst (DecodeN src));
8079 ins_cost(INSN_COST * 3);
8080 format %{ "decode_heap_oop_not_null $dst, $src" %}
8081 ins_encode %{
8082 Register s = $src$$Register;
8083 Register d = $dst$$Register;
8084 __ decode_heap_oop_not_null(d, s);
8085 %}
8086 ins_pipe(ialu_reg);
8087 %}
8088
8089 // n.b. AArch64 implementations of encode_klass_not_null and
8090 // decode_klass_not_null do not modify the flags register so, unlike
8091 // Intel, we don't kill CR as a side effect here
8092
8093 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8094 match(Set dst (EncodePKlass src));
8095
8096 ins_cost(INSN_COST * 3);
8097 format %{ "encode_klass_not_null $dst,$src" %}
8098
8099 ins_encode %{
8100 Register src_reg = as_Register($src$$reg);
8101 Register dst_reg = as_Register($dst$$reg);
8102 __ encode_klass_not_null(dst_reg, src_reg);
8103 %}
8104
8105 ins_pipe(ialu_reg);
8106 %}
8107
8108 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8109 match(Set dst (DecodeNKlass src));
8110
8111 ins_cost(INSN_COST * 3);
8112 format %{ "decode_klass_not_null $dst,$src" %}
8113
8114 ins_encode %{
8115 Register src_reg = as_Register($src$$reg);
8116 Register dst_reg = as_Register($dst$$reg);
8117 if (dst_reg != src_reg) {
8118 __ decode_klass_not_null(dst_reg, src_reg);
8119 } else {
8120 __ decode_klass_not_null(dst_reg);
8121 }
8122 %}
8123
8124 ins_pipe(ialu_reg);
8125 %}
8126
8127 instruct checkCastPP(iRegPNoSp dst)
8128 %{
8129 match(Set dst (CheckCastPP dst));
8130
8131 size(0);
8132 format %{ "# checkcastPP of $dst" %}
8133 ins_encode(/* empty encoding */);
8134 ins_pipe(pipe_class_empty);
8135 %}
8136
8137 instruct castPP(iRegPNoSp dst)
8138 %{
8139 match(Set dst (CastPP dst));
8140
8141 size(0);
8142 format %{ "# castPP of $dst" %}
8143 ins_encode(/* empty encoding */);
8144 ins_pipe(pipe_class_empty);
8145 %}
8146
8147 instruct castII(iRegI dst)
8148 %{
8149 match(Set dst (CastII dst));
8150
8151 size(0);
8152 format %{ "# castII of $dst" %}
8153 ins_encode(/* empty encoding */);
8154 ins_cost(0);
8155 ins_pipe(pipe_class_empty);
8156 %}
8157
8158 instruct castLL(iRegL dst)
8159 %{
8160 match(Set dst (CastLL dst));
8161
8162 size(0);
8163 format %{ "# castLL of $dst" %}
8164 ins_encode(/* empty encoding */);
8165 ins_cost(0);
8166 ins_pipe(pipe_class_empty);
8167 %}
8168
8169 instruct castFF(vRegF dst)
8170 %{
8171 match(Set dst (CastFF dst));
8172
8173 size(0);
8174 format %{ "# castFF of $dst" %}
8175 ins_encode(/* empty encoding */);
8176 ins_cost(0);
8177 ins_pipe(pipe_class_empty);
8178 %}
8179
8180 instruct castDD(vRegD dst)
8181 %{
8182 match(Set dst (CastDD dst));
8183
8184 size(0);
8185 format %{ "# castDD of $dst" %}
8186 ins_encode(/* empty encoding */);
8187 ins_cost(0);
8188 ins_pipe(pipe_class_empty);
8189 %}
8190
8191 instruct castVV(vReg dst)
8192 %{
8193 match(Set dst (CastVV dst));
8194
8195 size(0);
8196 format %{ "# castVV of $dst" %}
8197 ins_encode(/* empty encoding */);
8198 ins_cost(0);
8199 ins_pipe(pipe_class_empty);
8200 %}
8201
8202 instruct castVVMask(pRegGov dst)
8203 %{
8204 match(Set dst (CastVV dst));
8205
8206 size(0);
8207 format %{ "# castVV of $dst" %}
8208 ins_encode(/* empty encoding */);
8209 ins_cost(0);
8210 ins_pipe(pipe_class_empty);
8211 %}
8212
8213 // ============================================================================
8214 // Atomic operation instructions
8215 //
8216
8217 // standard CompareAndSwapX when we are using barriers
8218 // these have higher priority than the rules selected by a predicate
8219
8220 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8221 // can't match them
8222
8223 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8224
8225 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8226 ins_cost(2 * VOLATILE_REF_COST);
8227
8228 effect(KILL cr);
8229
8230 format %{
8231 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8232 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8233 %}
8234
8235 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8236 aarch64_enc_cset_eq(res));
8237
8238 ins_pipe(pipe_slow);
8239 %}
8240
8241 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8242
8243 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8244 ins_cost(2 * VOLATILE_REF_COST);
8245
8246 effect(KILL cr);
8247
8248 format %{
8249 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8250 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8251 %}
8252
8253 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8254 aarch64_enc_cset_eq(res));
8255
8256 ins_pipe(pipe_slow);
8257 %}
8258
8259 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8260
8261 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8262 ins_cost(2 * VOLATILE_REF_COST);
8263
8264 effect(KILL cr);
8265
8266 format %{
8267 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8268 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8269 %}
8270
8271 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8272 aarch64_enc_cset_eq(res));
8273
8274 ins_pipe(pipe_slow);
8275 %}
8276
8277 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8278
8279 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8280 ins_cost(2 * VOLATILE_REF_COST);
8281
8282 effect(KILL cr);
8283
8284 format %{
8285 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8286 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8287 %}
8288
8289 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8290 aarch64_enc_cset_eq(res));
8291
8292 ins_pipe(pipe_slow);
8293 %}
8294
8295 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8296
8297 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8298 predicate(n->as_LoadStore()->barrier_data() == 0);
8299 ins_cost(2 * VOLATILE_REF_COST);
8300
8301 effect(KILL cr);
8302
8303 format %{
8304 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8305 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8306 %}
8307
8308 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8309 aarch64_enc_cset_eq(res));
8310
8311 ins_pipe(pipe_slow);
8312 %}
8313
8314 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8315
8316 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8317 predicate(n->as_LoadStore()->barrier_data() == 0);
8318 ins_cost(2 * VOLATILE_REF_COST);
8319
8320 effect(KILL cr);
8321
8322 format %{
8323 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8324 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8325 %}
8326
8327 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8328 aarch64_enc_cset_eq(res));
8329
8330 ins_pipe(pipe_slow);
8331 %}
8332
8333 // alternative CompareAndSwapX when we are eliding barriers
8334
8335 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8336
8337 predicate(needs_acquiring_load_exclusive(n));
8338 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8339 ins_cost(VOLATILE_REF_COST);
8340
8341 effect(KILL cr);
8342
8343 format %{
8344 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8345 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8346 %}
8347
8348 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8349 aarch64_enc_cset_eq(res));
8350
8351 ins_pipe(pipe_slow);
8352 %}
8353
8354 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8355
8356 predicate(needs_acquiring_load_exclusive(n));
8357 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8358 ins_cost(VOLATILE_REF_COST);
8359
8360 effect(KILL cr);
8361
8362 format %{
8363 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8364 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8365 %}
8366
8367 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8368 aarch64_enc_cset_eq(res));
8369
8370 ins_pipe(pipe_slow);
8371 %}
8372
8373 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8374
8375 predicate(needs_acquiring_load_exclusive(n));
8376 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8377 ins_cost(VOLATILE_REF_COST);
8378
8379 effect(KILL cr);
8380
8381 format %{
8382 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8383 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8384 %}
8385
8386 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8387 aarch64_enc_cset_eq(res));
8388
8389 ins_pipe(pipe_slow);
8390 %}
8391
8392 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8393
8394 predicate(needs_acquiring_load_exclusive(n));
8395 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8396 ins_cost(VOLATILE_REF_COST);
8397
8398 effect(KILL cr);
8399
8400 format %{
8401 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8402 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8403 %}
8404
8405 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8406 aarch64_enc_cset_eq(res));
8407
8408 ins_pipe(pipe_slow);
8409 %}
8410
8411 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8412
8413 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8414 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8415 ins_cost(VOLATILE_REF_COST);
8416
8417 effect(KILL cr);
8418
8419 format %{
8420 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8421 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8422 %}
8423
8424 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8425 aarch64_enc_cset_eq(res));
8426
8427 ins_pipe(pipe_slow);
8428 %}
8429
8430 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8431
8432 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8433 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8434 ins_cost(VOLATILE_REF_COST);
8435
8436 effect(KILL cr);
8437
8438 format %{
8439 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8440 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8441 %}
8442
8443 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8444 aarch64_enc_cset_eq(res));
8445
8446 ins_pipe(pipe_slow);
8447 %}
8448
8449
8450 // ---------------------------------------------------------------------
8451
8452 // BEGIN This section of the file is automatically generated. Do not edit --------------
8453
8454 // Sundry CAS operations. Note that release is always true,
8455 // regardless of the memory ordering of the CAS. This is because we
8456 // need the volatile case to be sequentially consistent but there is
8457 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8458 // can't check the type of memory ordering here, so we always emit a
8459 // STLXR.
8460
8461 // This section is generated from cas.m4
8462
8463
8464 // This pattern is generated automatically from cas.m4.
8465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8466 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8467 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8468 ins_cost(2 * VOLATILE_REF_COST);
8469 effect(TEMP_DEF res, KILL cr);
8470 format %{
8471 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8472 %}
8473 ins_encode %{
8474 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8475 Assembler::byte, /*acquire*/ false, /*release*/ true,
8476 /*weak*/ false, $res$$Register);
8477 __ sxtbw($res$$Register, $res$$Register);
8478 %}
8479 ins_pipe(pipe_slow);
8480 %}
8481
8482 // This pattern is generated automatically from cas.m4.
8483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8484 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8485 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8486 ins_cost(2 * VOLATILE_REF_COST);
8487 effect(TEMP_DEF res, KILL cr);
8488 format %{
8489 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8490 %}
8491 ins_encode %{
8492 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8493 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8494 /*weak*/ false, $res$$Register);
8495 __ sxthw($res$$Register, $res$$Register);
8496 %}
8497 ins_pipe(pipe_slow);
8498 %}
8499
8500 // This pattern is generated automatically from cas.m4.
8501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8502 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8503 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8504 ins_cost(2 * VOLATILE_REF_COST);
8505 effect(TEMP_DEF res, KILL cr);
8506 format %{
8507 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8508 %}
8509 ins_encode %{
8510 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8511 Assembler::word, /*acquire*/ false, /*release*/ true,
8512 /*weak*/ false, $res$$Register);
8513 %}
8514 ins_pipe(pipe_slow);
8515 %}
8516
8517 // This pattern is generated automatically from cas.m4.
8518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8519 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8520 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8521 ins_cost(2 * VOLATILE_REF_COST);
8522 effect(TEMP_DEF res, KILL cr);
8523 format %{
8524 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8525 %}
8526 ins_encode %{
8527 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8528 Assembler::xword, /*acquire*/ false, /*release*/ true,
8529 /*weak*/ false, $res$$Register);
8530 %}
8531 ins_pipe(pipe_slow);
8532 %}
8533
8534 // This pattern is generated automatically from cas.m4.
8535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8536 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8537 predicate(n->as_LoadStore()->barrier_data() == 0);
8538 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8539 ins_cost(2 * VOLATILE_REF_COST);
8540 effect(TEMP_DEF res, KILL cr);
8541 format %{
8542 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8543 %}
8544 ins_encode %{
8545 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8546 Assembler::word, /*acquire*/ false, /*release*/ true,
8547 /*weak*/ false, $res$$Register);
8548 %}
8549 ins_pipe(pipe_slow);
8550 %}
8551
8552 // This pattern is generated automatically from cas.m4.
8553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8554 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8555 predicate(n->as_LoadStore()->barrier_data() == 0);
8556 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8557 ins_cost(2 * VOLATILE_REF_COST);
8558 effect(TEMP_DEF res, KILL cr);
8559 format %{
8560 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8561 %}
8562 ins_encode %{
8563 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8564 Assembler::xword, /*acquire*/ false, /*release*/ true,
8565 /*weak*/ false, $res$$Register);
8566 %}
8567 ins_pipe(pipe_slow);
8568 %}
8569
8570 // This pattern is generated automatically from cas.m4.
8571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8572 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8573 predicate(needs_acquiring_load_exclusive(n));
8574 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8575 ins_cost(VOLATILE_REF_COST);
8576 effect(TEMP_DEF res, KILL cr);
8577 format %{
8578 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8579 %}
8580 ins_encode %{
8581 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8582 Assembler::byte, /*acquire*/ true, /*release*/ true,
8583 /*weak*/ false, $res$$Register);
8584 __ sxtbw($res$$Register, $res$$Register);
8585 %}
8586 ins_pipe(pipe_slow);
8587 %}
8588
8589 // This pattern is generated automatically from cas.m4.
8590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8591 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8592 predicate(needs_acquiring_load_exclusive(n));
8593 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8594 ins_cost(VOLATILE_REF_COST);
8595 effect(TEMP_DEF res, KILL cr);
8596 format %{
8597 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8598 %}
8599 ins_encode %{
8600 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8601 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8602 /*weak*/ false, $res$$Register);
8603 __ sxthw($res$$Register, $res$$Register);
8604 %}
8605 ins_pipe(pipe_slow);
8606 %}
8607
8608 // This pattern is generated automatically from cas.m4.
8609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8610 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8611 predicate(needs_acquiring_load_exclusive(n));
8612 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8613 ins_cost(VOLATILE_REF_COST);
8614 effect(TEMP_DEF res, KILL cr);
8615 format %{
8616 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8617 %}
8618 ins_encode %{
8619 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8620 Assembler::word, /*acquire*/ true, /*release*/ true,
8621 /*weak*/ false, $res$$Register);
8622 %}
8623 ins_pipe(pipe_slow);
8624 %}
8625
8626 // This pattern is generated automatically from cas.m4.
8627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8628 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8629 predicate(needs_acquiring_load_exclusive(n));
8630 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8631 ins_cost(VOLATILE_REF_COST);
8632 effect(TEMP_DEF res, KILL cr);
8633 format %{
8634 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8635 %}
8636 ins_encode %{
8637 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8638 Assembler::xword, /*acquire*/ true, /*release*/ true,
8639 /*weak*/ false, $res$$Register);
8640 %}
8641 ins_pipe(pipe_slow);
8642 %}
8643
8644 // This pattern is generated automatically from cas.m4.
8645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8646 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8647 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8648 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8649 ins_cost(VOLATILE_REF_COST);
8650 effect(TEMP_DEF res, KILL cr);
8651 format %{
8652 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8653 %}
8654 ins_encode %{
8655 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8656 Assembler::word, /*acquire*/ true, /*release*/ true,
8657 /*weak*/ false, $res$$Register);
8658 %}
8659 ins_pipe(pipe_slow);
8660 %}
8661
8662 // This pattern is generated automatically from cas.m4.
8663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8664 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8665 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8666 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8667 ins_cost(VOLATILE_REF_COST);
8668 effect(TEMP_DEF res, KILL cr);
8669 format %{
8670 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8671 %}
8672 ins_encode %{
8673 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8674 Assembler::xword, /*acquire*/ true, /*release*/ true,
8675 /*weak*/ false, $res$$Register);
8676 %}
8677 ins_pipe(pipe_slow);
8678 %}
8679
8680 // This pattern is generated automatically from cas.m4.
8681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8682 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8683 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8684 ins_cost(2 * VOLATILE_REF_COST);
8685 effect(KILL cr);
8686 format %{
8687 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8688 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8689 %}
8690 ins_encode %{
8691 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8692 Assembler::byte, /*acquire*/ false, /*release*/ true,
8693 /*weak*/ true, noreg);
8694 __ csetw($res$$Register, Assembler::EQ);
8695 %}
8696 ins_pipe(pipe_slow);
8697 %}
8698
8699 // This pattern is generated automatically from cas.m4.
8700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8701 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8702 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8703 ins_cost(2 * VOLATILE_REF_COST);
8704 effect(KILL cr);
8705 format %{
8706 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8707 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8708 %}
8709 ins_encode %{
8710 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8711 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8712 /*weak*/ true, noreg);
8713 __ csetw($res$$Register, Assembler::EQ);
8714 %}
8715 ins_pipe(pipe_slow);
8716 %}
8717
8718 // This pattern is generated automatically from cas.m4.
8719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8720 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8721 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8722 ins_cost(2 * VOLATILE_REF_COST);
8723 effect(KILL cr);
8724 format %{
8725 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8726 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8727 %}
8728 ins_encode %{
8729 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8730 Assembler::word, /*acquire*/ false, /*release*/ true,
8731 /*weak*/ true, noreg);
8732 __ csetw($res$$Register, Assembler::EQ);
8733 %}
8734 ins_pipe(pipe_slow);
8735 %}
8736
8737 // This pattern is generated automatically from cas.m4.
8738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8739 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8740 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8741 ins_cost(2 * VOLATILE_REF_COST);
8742 effect(KILL cr);
8743 format %{
8744 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8745 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8746 %}
8747 ins_encode %{
8748 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8749 Assembler::xword, /*acquire*/ false, /*release*/ true,
8750 /*weak*/ true, noreg);
8751 __ csetw($res$$Register, Assembler::EQ);
8752 %}
8753 ins_pipe(pipe_slow);
8754 %}
8755
8756 // This pattern is generated automatically from cas.m4.
8757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8758 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8759 predicate(n->as_LoadStore()->barrier_data() == 0);
8760 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8761 ins_cost(2 * VOLATILE_REF_COST);
8762 effect(KILL cr);
8763 format %{
8764 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8765 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8766 %}
8767 ins_encode %{
8768 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8769 Assembler::word, /*acquire*/ false, /*release*/ true,
8770 /*weak*/ true, noreg);
8771 __ csetw($res$$Register, Assembler::EQ);
8772 %}
8773 ins_pipe(pipe_slow);
8774 %}
8775
8776 // This pattern is generated automatically from cas.m4.
8777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8778 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8779 predicate(n->as_LoadStore()->barrier_data() == 0);
8780 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8781 ins_cost(2 * VOLATILE_REF_COST);
8782 effect(KILL cr);
8783 format %{
8784 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8785 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8786 %}
8787 ins_encode %{
8788 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8789 Assembler::xword, /*acquire*/ false, /*release*/ true,
8790 /*weak*/ true, noreg);
8791 __ csetw($res$$Register, Assembler::EQ);
8792 %}
8793 ins_pipe(pipe_slow);
8794 %}
8795
8796 // This pattern is generated automatically from cas.m4.
8797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8798 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8799 predicate(needs_acquiring_load_exclusive(n));
8800 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8801 ins_cost(VOLATILE_REF_COST);
8802 effect(KILL cr);
8803 format %{
8804 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8805 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8806 %}
8807 ins_encode %{
8808 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8809 Assembler::byte, /*acquire*/ true, /*release*/ true,
8810 /*weak*/ true, noreg);
8811 __ csetw($res$$Register, Assembler::EQ);
8812 %}
8813 ins_pipe(pipe_slow);
8814 %}
8815
8816 // This pattern is generated automatically from cas.m4.
8817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8818 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8819 predicate(needs_acquiring_load_exclusive(n));
8820 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8821 ins_cost(VOLATILE_REF_COST);
8822 effect(KILL cr);
8823 format %{
8824 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8825 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8826 %}
8827 ins_encode %{
8828 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8829 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8830 /*weak*/ true, noreg);
8831 __ csetw($res$$Register, Assembler::EQ);
8832 %}
8833 ins_pipe(pipe_slow);
8834 %}
8835
8836 // This pattern is generated automatically from cas.m4.
8837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8838 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8839 predicate(needs_acquiring_load_exclusive(n));
8840 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8841 ins_cost(VOLATILE_REF_COST);
8842 effect(KILL cr);
8843 format %{
8844 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8845 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8846 %}
8847 ins_encode %{
8848 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8849 Assembler::word, /*acquire*/ true, /*release*/ true,
8850 /*weak*/ true, noreg);
8851 __ csetw($res$$Register, Assembler::EQ);
8852 %}
8853 ins_pipe(pipe_slow);
8854 %}
8855
8856 // This pattern is generated automatically from cas.m4.
8857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8858 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8859 predicate(needs_acquiring_load_exclusive(n));
8860 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8861 ins_cost(VOLATILE_REF_COST);
8862 effect(KILL cr);
8863 format %{
8864 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8865 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8866 %}
8867 ins_encode %{
8868 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8869 Assembler::xword, /*acquire*/ true, /*release*/ true,
8870 /*weak*/ true, noreg);
8871 __ csetw($res$$Register, Assembler::EQ);
8872 %}
8873 ins_pipe(pipe_slow);
8874 %}
8875
8876 // This pattern is generated automatically from cas.m4.
8877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8878 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8879 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8880 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8881 ins_cost(VOLATILE_REF_COST);
8882 effect(KILL cr);
8883 format %{
8884 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8885 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8886 %}
8887 ins_encode %{
8888 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8889 Assembler::word, /*acquire*/ true, /*release*/ true,
8890 /*weak*/ true, noreg);
8891 __ csetw($res$$Register, Assembler::EQ);
8892 %}
8893 ins_pipe(pipe_slow);
8894 %}
8895
8896 // This pattern is generated automatically from cas.m4.
8897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8898 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8899 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8900 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8901 ins_cost(VOLATILE_REF_COST);
8902 effect(KILL cr);
8903 format %{
8904 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8905 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8906 %}
8907 ins_encode %{
8908 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8909 Assembler::xword, /*acquire*/ true, /*release*/ true,
8910 /*weak*/ true, noreg);
8911 __ csetw($res$$Register, Assembler::EQ);
8912 %}
8913 ins_pipe(pipe_slow);
8914 %}
8915
8916 // END This section of the file is automatically generated. Do not edit --------------
8917 // ---------------------------------------------------------------------
8918
8919 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8920 match(Set prev (GetAndSetI mem newv));
8921 ins_cost(2 * VOLATILE_REF_COST);
8922 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8923 ins_encode %{
8924 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8925 %}
8926 ins_pipe(pipe_serial);
8927 %}
8928
8929 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8930 match(Set prev (GetAndSetL mem newv));
8931 ins_cost(2 * VOLATILE_REF_COST);
8932 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8933 ins_encode %{
8934 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8935 %}
8936 ins_pipe(pipe_serial);
8937 %}
8938
8939 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8940 predicate(n->as_LoadStore()->barrier_data() == 0);
8941 match(Set prev (GetAndSetN mem newv));
8942 ins_cost(2 * VOLATILE_REF_COST);
8943 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8944 ins_encode %{
8945 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8946 %}
8947 ins_pipe(pipe_serial);
8948 %}
8949
8950 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8951 predicate(n->as_LoadStore()->barrier_data() == 0);
8952 match(Set prev (GetAndSetP mem newv));
8953 ins_cost(2 * VOLATILE_REF_COST);
8954 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8955 ins_encode %{
8956 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8957 %}
8958 ins_pipe(pipe_serial);
8959 %}
8960
8961 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8962 predicate(needs_acquiring_load_exclusive(n));
8963 match(Set prev (GetAndSetI mem newv));
8964 ins_cost(VOLATILE_REF_COST);
8965 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8966 ins_encode %{
8967 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8968 %}
8969 ins_pipe(pipe_serial);
8970 %}
8971
8972 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8973 predicate(needs_acquiring_load_exclusive(n));
8974 match(Set prev (GetAndSetL mem newv));
8975 ins_cost(VOLATILE_REF_COST);
8976 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8977 ins_encode %{
8978 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8979 %}
8980 ins_pipe(pipe_serial);
8981 %}
8982
8983 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8984 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8985 match(Set prev (GetAndSetN mem newv));
8986 ins_cost(VOLATILE_REF_COST);
8987 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8988 ins_encode %{
8989 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8990 %}
8991 ins_pipe(pipe_serial);
8992 %}
8993
8994 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8995 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8996 match(Set prev (GetAndSetP mem newv));
8997 ins_cost(VOLATILE_REF_COST);
8998 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8999 ins_encode %{
9000 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9001 %}
9002 ins_pipe(pipe_serial);
9003 %}
9004
9005
9006 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9007 match(Set newval (GetAndAddL mem incr));
9008 ins_cost(2 * VOLATILE_REF_COST + 1);
9009 format %{ "get_and_addL $newval, [$mem], $incr" %}
9010 ins_encode %{
9011 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9012 %}
9013 ins_pipe(pipe_serial);
9014 %}
9015
9016 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9017 predicate(n->as_LoadStore()->result_not_used());
9018 match(Set dummy (GetAndAddL mem incr));
9019 ins_cost(2 * VOLATILE_REF_COST);
9020 format %{ "get_and_addL [$mem], $incr" %}
9021 ins_encode %{
9022 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9023 %}
9024 ins_pipe(pipe_serial);
9025 %}
9026
9027 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9028 match(Set newval (GetAndAddL mem incr));
9029 ins_cost(2 * VOLATILE_REF_COST + 1);
9030 format %{ "get_and_addL $newval, [$mem], $incr" %}
9031 ins_encode %{
9032 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9033 %}
9034 ins_pipe(pipe_serial);
9035 %}
9036
9037 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9038 predicate(n->as_LoadStore()->result_not_used());
9039 match(Set dummy (GetAndAddL mem incr));
9040 ins_cost(2 * VOLATILE_REF_COST);
9041 format %{ "get_and_addL [$mem], $incr" %}
9042 ins_encode %{
9043 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9044 %}
9045 ins_pipe(pipe_serial);
9046 %}
9047
9048 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9049 match(Set newval (GetAndAddI mem incr));
9050 ins_cost(2 * VOLATILE_REF_COST + 1);
9051 format %{ "get_and_addI $newval, [$mem], $incr" %}
9052 ins_encode %{
9053 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9054 %}
9055 ins_pipe(pipe_serial);
9056 %}
9057
9058 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9059 predicate(n->as_LoadStore()->result_not_used());
9060 match(Set dummy (GetAndAddI mem incr));
9061 ins_cost(2 * VOLATILE_REF_COST);
9062 format %{ "get_and_addI [$mem], $incr" %}
9063 ins_encode %{
9064 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9065 %}
9066 ins_pipe(pipe_serial);
9067 %}
9068
9069 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9070 match(Set newval (GetAndAddI mem incr));
9071 ins_cost(2 * VOLATILE_REF_COST + 1);
9072 format %{ "get_and_addI $newval, [$mem], $incr" %}
9073 ins_encode %{
9074 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9075 %}
9076 ins_pipe(pipe_serial);
9077 %}
9078
9079 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9080 predicate(n->as_LoadStore()->result_not_used());
9081 match(Set dummy (GetAndAddI mem incr));
9082 ins_cost(2 * VOLATILE_REF_COST);
9083 format %{ "get_and_addI [$mem], $incr" %}
9084 ins_encode %{
9085 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9086 %}
9087 ins_pipe(pipe_serial);
9088 %}
9089
9090 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9091 predicate(needs_acquiring_load_exclusive(n));
9092 match(Set newval (GetAndAddL mem incr));
9093 ins_cost(VOLATILE_REF_COST + 1);
9094 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9095 ins_encode %{
9096 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9097 %}
9098 ins_pipe(pipe_serial);
9099 %}
9100
9101 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9102 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9103 match(Set dummy (GetAndAddL mem incr));
9104 ins_cost(VOLATILE_REF_COST);
9105 format %{ "get_and_addL_acq [$mem], $incr" %}
9106 ins_encode %{
9107 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9108 %}
9109 ins_pipe(pipe_serial);
9110 %}
9111
9112 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9113 predicate(needs_acquiring_load_exclusive(n));
9114 match(Set newval (GetAndAddL mem incr));
9115 ins_cost(VOLATILE_REF_COST + 1);
9116 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9117 ins_encode %{
9118 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9119 %}
9120 ins_pipe(pipe_serial);
9121 %}
9122
9123 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9124 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9125 match(Set dummy (GetAndAddL mem incr));
9126 ins_cost(VOLATILE_REF_COST);
9127 format %{ "get_and_addL_acq [$mem], $incr" %}
9128 ins_encode %{
9129 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9130 %}
9131 ins_pipe(pipe_serial);
9132 %}
9133
9134 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9135 predicate(needs_acquiring_load_exclusive(n));
9136 match(Set newval (GetAndAddI mem incr));
9137 ins_cost(VOLATILE_REF_COST + 1);
9138 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9139 ins_encode %{
9140 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9141 %}
9142 ins_pipe(pipe_serial);
9143 %}
9144
9145 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9146 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9147 match(Set dummy (GetAndAddI mem incr));
9148 ins_cost(VOLATILE_REF_COST);
9149 format %{ "get_and_addI_acq [$mem], $incr" %}
9150 ins_encode %{
9151 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9152 %}
9153 ins_pipe(pipe_serial);
9154 %}
9155
9156 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9157 predicate(needs_acquiring_load_exclusive(n));
9158 match(Set newval (GetAndAddI mem incr));
9159 ins_cost(VOLATILE_REF_COST + 1);
9160 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9161 ins_encode %{
9162 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9163 %}
9164 ins_pipe(pipe_serial);
9165 %}
9166
9167 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9168 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9169 match(Set dummy (GetAndAddI mem incr));
9170 ins_cost(VOLATILE_REF_COST);
9171 format %{ "get_and_addI_acq [$mem], $incr" %}
9172 ins_encode %{
9173 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9174 %}
9175 ins_pipe(pipe_serial);
9176 %}
9177
9178 // Manifest a CmpU result in an integer register.
9179 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9180 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9181 %{
9182 match(Set dst (CmpU3 src1 src2));
9183 effect(KILL flags);
9184
9185 ins_cost(INSN_COST * 3);
9186 format %{
9187 "cmpw $src1, $src2\n\t"
9188 "csetw $dst, ne\n\t"
9189 "cnegw $dst, lo\t# CmpU3(reg)"
9190 %}
9191 ins_encode %{
9192 __ cmpw($src1$$Register, $src2$$Register);
9193 __ csetw($dst$$Register, Assembler::NE);
9194 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9195 %}
9196
9197 ins_pipe(pipe_class_default);
9198 %}
9199
9200 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9201 %{
9202 match(Set dst (CmpU3 src1 src2));
9203 effect(KILL flags);
9204
9205 ins_cost(INSN_COST * 3);
9206 format %{
9207 "subsw zr, $src1, $src2\n\t"
9208 "csetw $dst, ne\n\t"
9209 "cnegw $dst, lo\t# CmpU3(imm)"
9210 %}
9211 ins_encode %{
9212 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9213 __ csetw($dst$$Register, Assembler::NE);
9214 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9215 %}
9216
9217 ins_pipe(pipe_class_default);
9218 %}
9219
9220 // Manifest a CmpUL result in an integer register.
9221 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9222 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9223 %{
9224 match(Set dst (CmpUL3 src1 src2));
9225 effect(KILL flags);
9226
9227 ins_cost(INSN_COST * 3);
9228 format %{
9229 "cmp $src1, $src2\n\t"
9230 "csetw $dst, ne\n\t"
9231 "cnegw $dst, lo\t# CmpUL3(reg)"
9232 %}
9233 ins_encode %{
9234 __ cmp($src1$$Register, $src2$$Register);
9235 __ csetw($dst$$Register, Assembler::NE);
9236 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9237 %}
9238
9239 ins_pipe(pipe_class_default);
9240 %}
9241
9242 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9243 %{
9244 match(Set dst (CmpUL3 src1 src2));
9245 effect(KILL flags);
9246
9247 ins_cost(INSN_COST * 3);
9248 format %{
9249 "subs zr, $src1, $src2\n\t"
9250 "csetw $dst, ne\n\t"
9251 "cnegw $dst, lo\t# CmpUL3(imm)"
9252 %}
9253 ins_encode %{
9254 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9255 __ csetw($dst$$Register, Assembler::NE);
9256 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9257 %}
9258
9259 ins_pipe(pipe_class_default);
9260 %}
9261
9262 // Manifest a CmpL result in an integer register.
9263 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9264 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9265 %{
9266 match(Set dst (CmpL3 src1 src2));
9267 effect(KILL flags);
9268
9269 ins_cost(INSN_COST * 3);
9270 format %{
9271 "cmp $src1, $src2\n\t"
9272 "csetw $dst, ne\n\t"
9273 "cnegw $dst, lt\t# CmpL3(reg)"
9274 %}
9275 ins_encode %{
9276 __ cmp($src1$$Register, $src2$$Register);
9277 __ csetw($dst$$Register, Assembler::NE);
9278 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9279 %}
9280
9281 ins_pipe(pipe_class_default);
9282 %}
9283
9284 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9285 %{
9286 match(Set dst (CmpL3 src1 src2));
9287 effect(KILL flags);
9288
9289 ins_cost(INSN_COST * 3);
9290 format %{
9291 "subs zr, $src1, $src2\n\t"
9292 "csetw $dst, ne\n\t"
9293 "cnegw $dst, lt\t# CmpL3(imm)"
9294 %}
9295 ins_encode %{
9296 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9297 __ csetw($dst$$Register, Assembler::NE);
9298 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9299 %}
9300
9301 ins_pipe(pipe_class_default);
9302 %}
9303
9304 // ============================================================================
9305 // Conditional Move Instructions
9306
9307 // n.b. we have identical rules for both a signed compare op (cmpOp)
9308 // and an unsigned compare op (cmpOpU). it would be nice if we could
9309 // define an op class which merged both inputs and use it to type the
9310 // argument to a single rule. unfortunatelyt his fails because the
9311 // opclass does not live up to the COND_INTER interface of its
9312 // component operands. When the generic code tries to negate the
9313 // operand it ends up running the generci Machoper::negate method
9314 // which throws a ShouldNotHappen. So, we have to provide two flavours
9315 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9316
9317 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9318 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9319
9320 ins_cost(INSN_COST * 2);
9321 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9322
9323 ins_encode %{
9324 __ cselw(as_Register($dst$$reg),
9325 as_Register($src2$$reg),
9326 as_Register($src1$$reg),
9327 (Assembler::Condition)$cmp$$cmpcode);
9328 %}
9329
9330 ins_pipe(icond_reg_reg);
9331 %}
9332
9333 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9334 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9335
9336 ins_cost(INSN_COST * 2);
9337 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9338
9339 ins_encode %{
9340 __ cselw(as_Register($dst$$reg),
9341 as_Register($src2$$reg),
9342 as_Register($src1$$reg),
9343 (Assembler::Condition)$cmp$$cmpcode);
9344 %}
9345
9346 ins_pipe(icond_reg_reg);
9347 %}
9348
9349 // special cases where one arg is zero
9350
9351 // n.b. this is selected in preference to the rule above because it
9352 // avoids loading constant 0 into a source register
9353
9354 // TODO
9355 // we ought only to be able to cull one of these variants as the ideal
9356 // transforms ought always to order the zero consistently (to left/right?)
9357
9358 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9359 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9360
9361 ins_cost(INSN_COST * 2);
9362 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9363
9364 ins_encode %{
9365 __ cselw(as_Register($dst$$reg),
9366 as_Register($src$$reg),
9367 zr,
9368 (Assembler::Condition)$cmp$$cmpcode);
9369 %}
9370
9371 ins_pipe(icond_reg);
9372 %}
9373
9374 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9375 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9376
9377 ins_cost(INSN_COST * 2);
9378 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9379
9380 ins_encode %{
9381 __ cselw(as_Register($dst$$reg),
9382 as_Register($src$$reg),
9383 zr,
9384 (Assembler::Condition)$cmp$$cmpcode);
9385 %}
9386
9387 ins_pipe(icond_reg);
9388 %}
9389
9390 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9391 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9392
9393 ins_cost(INSN_COST * 2);
9394 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9395
9396 ins_encode %{
9397 __ cselw(as_Register($dst$$reg),
9398 zr,
9399 as_Register($src$$reg),
9400 (Assembler::Condition)$cmp$$cmpcode);
9401 %}
9402
9403 ins_pipe(icond_reg);
9404 %}
9405
9406 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9407 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9408
9409 ins_cost(INSN_COST * 2);
9410 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9411
9412 ins_encode %{
9413 __ cselw(as_Register($dst$$reg),
9414 zr,
9415 as_Register($src$$reg),
9416 (Assembler::Condition)$cmp$$cmpcode);
9417 %}
9418
9419 ins_pipe(icond_reg);
9420 %}
9421
9422 // special case for creating a boolean 0 or 1
9423
9424 // n.b. this is selected in preference to the rule above because it
9425 // avoids loading constants 0 and 1 into a source register
9426
9427 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9428 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9429
9430 ins_cost(INSN_COST * 2);
9431 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9432
9433 ins_encode %{
9434 // equivalently
9435 // cset(as_Register($dst$$reg),
9436 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9437 __ csincw(as_Register($dst$$reg),
9438 zr,
9439 zr,
9440 (Assembler::Condition)$cmp$$cmpcode);
9441 %}
9442
9443 ins_pipe(icond_none);
9444 %}
9445
9446 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9447 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9448
9449 ins_cost(INSN_COST * 2);
9450 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9451
9452 ins_encode %{
9453 // equivalently
9454 // cset(as_Register($dst$$reg),
9455 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9456 __ csincw(as_Register($dst$$reg),
9457 zr,
9458 zr,
9459 (Assembler::Condition)$cmp$$cmpcode);
9460 %}
9461
9462 ins_pipe(icond_none);
9463 %}
9464
9465 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9466 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9467
9468 ins_cost(INSN_COST * 2);
9469 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9470
9471 ins_encode %{
9472 __ csel(as_Register($dst$$reg),
9473 as_Register($src2$$reg),
9474 as_Register($src1$$reg),
9475 (Assembler::Condition)$cmp$$cmpcode);
9476 %}
9477
9478 ins_pipe(icond_reg_reg);
9479 %}
9480
9481 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9482 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9483
9484 ins_cost(INSN_COST * 2);
9485 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9486
9487 ins_encode %{
9488 __ csel(as_Register($dst$$reg),
9489 as_Register($src2$$reg),
9490 as_Register($src1$$reg),
9491 (Assembler::Condition)$cmp$$cmpcode);
9492 %}
9493
9494 ins_pipe(icond_reg_reg);
9495 %}
9496
9497 // special cases where one arg is zero
9498
9499 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9500 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9501
9502 ins_cost(INSN_COST * 2);
9503 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9504
9505 ins_encode %{
9506 __ csel(as_Register($dst$$reg),
9507 zr,
9508 as_Register($src$$reg),
9509 (Assembler::Condition)$cmp$$cmpcode);
9510 %}
9511
9512 ins_pipe(icond_reg);
9513 %}
9514
9515 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9516 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9517
9518 ins_cost(INSN_COST * 2);
9519 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9520
9521 ins_encode %{
9522 __ csel(as_Register($dst$$reg),
9523 zr,
9524 as_Register($src$$reg),
9525 (Assembler::Condition)$cmp$$cmpcode);
9526 %}
9527
9528 ins_pipe(icond_reg);
9529 %}
9530
9531 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9532 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9533
9534 ins_cost(INSN_COST * 2);
9535 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9536
9537 ins_encode %{
9538 __ csel(as_Register($dst$$reg),
9539 as_Register($src$$reg),
9540 zr,
9541 (Assembler::Condition)$cmp$$cmpcode);
9542 %}
9543
9544 ins_pipe(icond_reg);
9545 %}
9546
9547 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9548 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9549
9550 ins_cost(INSN_COST * 2);
9551 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9552
9553 ins_encode %{
9554 __ csel(as_Register($dst$$reg),
9555 as_Register($src$$reg),
9556 zr,
9557 (Assembler::Condition)$cmp$$cmpcode);
9558 %}
9559
9560 ins_pipe(icond_reg);
9561 %}
9562
9563 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9564 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9565
9566 ins_cost(INSN_COST * 2);
9567 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9568
9569 ins_encode %{
9570 __ csel(as_Register($dst$$reg),
9571 as_Register($src2$$reg),
9572 as_Register($src1$$reg),
9573 (Assembler::Condition)$cmp$$cmpcode);
9574 %}
9575
9576 ins_pipe(icond_reg_reg);
9577 %}
9578
9579 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9580 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9581
9582 ins_cost(INSN_COST * 2);
9583 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9584
9585 ins_encode %{
9586 __ csel(as_Register($dst$$reg),
9587 as_Register($src2$$reg),
9588 as_Register($src1$$reg),
9589 (Assembler::Condition)$cmp$$cmpcode);
9590 %}
9591
9592 ins_pipe(icond_reg_reg);
9593 %}
9594
9595 // special cases where one arg is zero
9596
9597 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9598 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9599
9600 ins_cost(INSN_COST * 2);
9601 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9602
9603 ins_encode %{
9604 __ csel(as_Register($dst$$reg),
9605 zr,
9606 as_Register($src$$reg),
9607 (Assembler::Condition)$cmp$$cmpcode);
9608 %}
9609
9610 ins_pipe(icond_reg);
9611 %}
9612
9613 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9614 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9615
9616 ins_cost(INSN_COST * 2);
9617 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9618
9619 ins_encode %{
9620 __ csel(as_Register($dst$$reg),
9621 zr,
9622 as_Register($src$$reg),
9623 (Assembler::Condition)$cmp$$cmpcode);
9624 %}
9625
9626 ins_pipe(icond_reg);
9627 %}
9628
9629 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9630 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9631
9632 ins_cost(INSN_COST * 2);
9633 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9634
9635 ins_encode %{
9636 __ csel(as_Register($dst$$reg),
9637 as_Register($src$$reg),
9638 zr,
9639 (Assembler::Condition)$cmp$$cmpcode);
9640 %}
9641
9642 ins_pipe(icond_reg);
9643 %}
9644
9645 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9646 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9647
9648 ins_cost(INSN_COST * 2);
9649 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9650
9651 ins_encode %{
9652 __ csel(as_Register($dst$$reg),
9653 as_Register($src$$reg),
9654 zr,
9655 (Assembler::Condition)$cmp$$cmpcode);
9656 %}
9657
9658 ins_pipe(icond_reg);
9659 %}
9660
9661 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9662 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9663
9664 ins_cost(INSN_COST * 2);
9665 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9666
9667 ins_encode %{
9668 __ cselw(as_Register($dst$$reg),
9669 as_Register($src2$$reg),
9670 as_Register($src1$$reg),
9671 (Assembler::Condition)$cmp$$cmpcode);
9672 %}
9673
9674 ins_pipe(icond_reg_reg);
9675 %}
9676
9677 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9678 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9679
9680 ins_cost(INSN_COST * 2);
9681 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9682
9683 ins_encode %{
9684 __ cselw(as_Register($dst$$reg),
9685 as_Register($src2$$reg),
9686 as_Register($src1$$reg),
9687 (Assembler::Condition)$cmp$$cmpcode);
9688 %}
9689
9690 ins_pipe(icond_reg_reg);
9691 %}
9692
9693 // special cases where one arg is zero
9694
9695 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9696 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9697
9698 ins_cost(INSN_COST * 2);
9699 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9700
9701 ins_encode %{
9702 __ cselw(as_Register($dst$$reg),
9703 zr,
9704 as_Register($src$$reg),
9705 (Assembler::Condition)$cmp$$cmpcode);
9706 %}
9707
9708 ins_pipe(icond_reg);
9709 %}
9710
9711 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9712 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9713
9714 ins_cost(INSN_COST * 2);
9715 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9716
9717 ins_encode %{
9718 __ cselw(as_Register($dst$$reg),
9719 zr,
9720 as_Register($src$$reg),
9721 (Assembler::Condition)$cmp$$cmpcode);
9722 %}
9723
9724 ins_pipe(icond_reg);
9725 %}
9726
9727 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9728 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9729
9730 ins_cost(INSN_COST * 2);
9731 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9732
9733 ins_encode %{
9734 __ cselw(as_Register($dst$$reg),
9735 as_Register($src$$reg),
9736 zr,
9737 (Assembler::Condition)$cmp$$cmpcode);
9738 %}
9739
9740 ins_pipe(icond_reg);
9741 %}
9742
9743 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9744 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9745
9746 ins_cost(INSN_COST * 2);
9747 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9748
9749 ins_encode %{
9750 __ cselw(as_Register($dst$$reg),
9751 as_Register($src$$reg),
9752 zr,
9753 (Assembler::Condition)$cmp$$cmpcode);
9754 %}
9755
9756 ins_pipe(icond_reg);
9757 %}
9758
9759 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9760 %{
9761 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9762
9763 ins_cost(INSN_COST * 3);
9764
9765 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9766 ins_encode %{
9767 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9768 __ fcsels(as_FloatRegister($dst$$reg),
9769 as_FloatRegister($src2$$reg),
9770 as_FloatRegister($src1$$reg),
9771 cond);
9772 %}
9773
9774 ins_pipe(fp_cond_reg_reg_s);
9775 %}
9776
9777 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9778 %{
9779 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9780
9781 ins_cost(INSN_COST * 3);
9782
9783 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9784 ins_encode %{
9785 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9786 __ fcsels(as_FloatRegister($dst$$reg),
9787 as_FloatRegister($src2$$reg),
9788 as_FloatRegister($src1$$reg),
9789 cond);
9790 %}
9791
9792 ins_pipe(fp_cond_reg_reg_s);
9793 %}
9794
9795 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9796 %{
9797 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9798
9799 ins_cost(INSN_COST * 3);
9800
9801 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9802 ins_encode %{
9803 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9804 __ fcseld(as_FloatRegister($dst$$reg),
9805 as_FloatRegister($src2$$reg),
9806 as_FloatRegister($src1$$reg),
9807 cond);
9808 %}
9809
9810 ins_pipe(fp_cond_reg_reg_d);
9811 %}
9812
9813 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9814 %{
9815 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9816
9817 ins_cost(INSN_COST * 3);
9818
9819 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9820 ins_encode %{
9821 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9822 __ fcseld(as_FloatRegister($dst$$reg),
9823 as_FloatRegister($src2$$reg),
9824 as_FloatRegister($src1$$reg),
9825 cond);
9826 %}
9827
9828 ins_pipe(fp_cond_reg_reg_d);
9829 %}
9830
9831 // ============================================================================
9832 // Arithmetic Instructions
9833 //
9834
9835 // Integer Addition
9836
9837 // TODO
9838 // these currently employ operations which do not set CR and hence are
9839 // not flagged as killing CR but we would like to isolate the cases
9840 // where we want to set flags from those where we don't. need to work
9841 // out how to do that.
9842
9843 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9844 match(Set dst (AddI src1 src2));
9845
9846 ins_cost(INSN_COST);
9847 format %{ "addw $dst, $src1, $src2" %}
9848
9849 ins_encode %{
9850 __ addw(as_Register($dst$$reg),
9851 as_Register($src1$$reg),
9852 as_Register($src2$$reg));
9853 %}
9854
9855 ins_pipe(ialu_reg_reg);
9856 %}
9857
9858 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9859 match(Set dst (AddI src1 src2));
9860
9861 ins_cost(INSN_COST);
9862 format %{ "addw $dst, $src1, $src2" %}
9863
9864 // use opcode to indicate that this is an add not a sub
9865 opcode(0x0);
9866
9867 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9868
9869 ins_pipe(ialu_reg_imm);
9870 %}
9871
9872 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9873 match(Set dst (AddI (ConvL2I src1) src2));
9874
9875 ins_cost(INSN_COST);
9876 format %{ "addw $dst, $src1, $src2" %}
9877
9878 // use opcode to indicate that this is an add not a sub
9879 opcode(0x0);
9880
9881 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9882
9883 ins_pipe(ialu_reg_imm);
9884 %}
9885
9886 // Pointer Addition
9887 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9888 match(Set dst (AddP src1 src2));
9889
9890 ins_cost(INSN_COST);
9891 format %{ "add $dst, $src1, $src2\t# ptr" %}
9892
9893 ins_encode %{
9894 __ add(as_Register($dst$$reg),
9895 as_Register($src1$$reg),
9896 as_Register($src2$$reg));
9897 %}
9898
9899 ins_pipe(ialu_reg_reg);
9900 %}
9901
9902 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9903 match(Set dst (AddP src1 (ConvI2L src2)));
9904
9905 ins_cost(1.9 * INSN_COST);
9906 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9907
9908 ins_encode %{
9909 __ add(as_Register($dst$$reg),
9910 as_Register($src1$$reg),
9911 as_Register($src2$$reg), ext::sxtw);
9912 %}
9913
9914 ins_pipe(ialu_reg_reg);
9915 %}
9916
9917 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9918 match(Set dst (AddP src1 (LShiftL src2 scale)));
9919
9920 ins_cost(1.9 * INSN_COST);
9921 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9922
9923 ins_encode %{
9924 __ lea(as_Register($dst$$reg),
9925 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9926 Address::lsl($scale$$constant)));
9927 %}
9928
9929 ins_pipe(ialu_reg_reg_shift);
9930 %}
9931
9932 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9933 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9934
9935 ins_cost(1.9 * INSN_COST);
9936 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9937
9938 ins_encode %{
9939 __ lea(as_Register($dst$$reg),
9940 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9941 Address::sxtw($scale$$constant)));
9942 %}
9943
9944 ins_pipe(ialu_reg_reg_shift);
9945 %}
9946
9947 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9948 match(Set dst (LShiftL (ConvI2L src) scale));
9949
9950 ins_cost(INSN_COST);
9951 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9952
9953 ins_encode %{
9954 __ sbfiz(as_Register($dst$$reg),
9955 as_Register($src$$reg),
9956 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9957 %}
9958
9959 ins_pipe(ialu_reg_shift);
9960 %}
9961
9962 // Pointer Immediate Addition
9963 // n.b. this needs to be more expensive than using an indirect memory
9964 // operand
9965 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9966 match(Set dst (AddP src1 src2));
9967
9968 ins_cost(INSN_COST);
9969 format %{ "add $dst, $src1, $src2\t# ptr" %}
9970
9971 // use opcode to indicate that this is an add not a sub
9972 opcode(0x0);
9973
9974 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9975
9976 ins_pipe(ialu_reg_imm);
9977 %}
9978
9979 // Long Addition
9980 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9981
9982 match(Set dst (AddL src1 src2));
9983
9984 ins_cost(INSN_COST);
9985 format %{ "add $dst, $src1, $src2" %}
9986
9987 ins_encode %{
9988 __ add(as_Register($dst$$reg),
9989 as_Register($src1$$reg),
9990 as_Register($src2$$reg));
9991 %}
9992
9993 ins_pipe(ialu_reg_reg);
9994 %}
9995
9996 // No constant pool entries requiredLong Immediate Addition.
9997 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9998 match(Set dst (AddL src1 src2));
9999
10000 ins_cost(INSN_COST);
10001 format %{ "add $dst, $src1, $src2" %}
10002
10003 // use opcode to indicate that this is an add not a sub
10004 opcode(0x0);
10005
10006 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10007
10008 ins_pipe(ialu_reg_imm);
10009 %}
10010
10011 // Integer Subtraction
10012 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10013 match(Set dst (SubI src1 src2));
10014
10015 ins_cost(INSN_COST);
10016 format %{ "subw $dst, $src1, $src2" %}
10017
10018 ins_encode %{
10019 __ subw(as_Register($dst$$reg),
10020 as_Register($src1$$reg),
10021 as_Register($src2$$reg));
10022 %}
10023
10024 ins_pipe(ialu_reg_reg);
10025 %}
10026
10027 // Immediate Subtraction
10028 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10029 match(Set dst (SubI src1 src2));
10030
10031 ins_cost(INSN_COST);
10032 format %{ "subw $dst, $src1, $src2" %}
10033
10034 // use opcode to indicate that this is a sub not an add
10035 opcode(0x1);
10036
10037 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10038
10039 ins_pipe(ialu_reg_imm);
10040 %}
10041
10042 // Long Subtraction
10043 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10044
10045 match(Set dst (SubL src1 src2));
10046
10047 ins_cost(INSN_COST);
10048 format %{ "sub $dst, $src1, $src2" %}
10049
10050 ins_encode %{
10051 __ sub(as_Register($dst$$reg),
10052 as_Register($src1$$reg),
10053 as_Register($src2$$reg));
10054 %}
10055
10056 ins_pipe(ialu_reg_reg);
10057 %}
10058
10059 // No constant pool entries requiredLong Immediate Subtraction.
10060 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10061 match(Set dst (SubL src1 src2));
10062
10063 ins_cost(INSN_COST);
10064 format %{ "sub$dst, $src1, $src2" %}
10065
10066 // use opcode to indicate that this is a sub not an add
10067 opcode(0x1);
10068
10069 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10070
10071 ins_pipe(ialu_reg_imm);
10072 %}
10073
10074 // Integer Negation (special case for sub)
10075
10076 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10077 match(Set dst (SubI zero src));
10078
10079 ins_cost(INSN_COST);
10080 format %{ "negw $dst, $src\t# int" %}
10081
10082 ins_encode %{
10083 __ negw(as_Register($dst$$reg),
10084 as_Register($src$$reg));
10085 %}
10086
10087 ins_pipe(ialu_reg);
10088 %}
10089
10090 // Long Negation
10091
10092 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10093 match(Set dst (SubL zero src));
10094
10095 ins_cost(INSN_COST);
10096 format %{ "neg $dst, $src\t# long" %}
10097
10098 ins_encode %{
10099 __ neg(as_Register($dst$$reg),
10100 as_Register($src$$reg));
10101 %}
10102
10103 ins_pipe(ialu_reg);
10104 %}
10105
10106 // Integer Multiply
10107
10108 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10109 match(Set dst (MulI src1 src2));
10110
10111 ins_cost(INSN_COST * 3);
10112 format %{ "mulw $dst, $src1, $src2" %}
10113
10114 ins_encode %{
10115 __ mulw(as_Register($dst$$reg),
10116 as_Register($src1$$reg),
10117 as_Register($src2$$reg));
10118 %}
10119
10120 ins_pipe(imul_reg_reg);
10121 %}
10122
10123 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10124 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10125
10126 ins_cost(INSN_COST * 3);
10127 format %{ "smull $dst, $src1, $src2" %}
10128
10129 ins_encode %{
10130 __ smull(as_Register($dst$$reg),
10131 as_Register($src1$$reg),
10132 as_Register($src2$$reg));
10133 %}
10134
10135 ins_pipe(imul_reg_reg);
10136 %}
10137
10138 // Long Multiply
10139
10140 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10141 match(Set dst (MulL src1 src2));
10142
10143 ins_cost(INSN_COST * 5);
10144 format %{ "mul $dst, $src1, $src2" %}
10145
10146 ins_encode %{
10147 __ mul(as_Register($dst$$reg),
10148 as_Register($src1$$reg),
10149 as_Register($src2$$reg));
10150 %}
10151
10152 ins_pipe(lmul_reg_reg);
10153 %}
10154
10155 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10156 %{
10157 match(Set dst (MulHiL src1 src2));
10158
10159 ins_cost(INSN_COST * 7);
10160 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10161
10162 ins_encode %{
10163 __ smulh(as_Register($dst$$reg),
10164 as_Register($src1$$reg),
10165 as_Register($src2$$reg));
10166 %}
10167
10168 ins_pipe(lmul_reg_reg);
10169 %}
10170
10171 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10172 %{
10173 match(Set dst (UMulHiL src1 src2));
10174
10175 ins_cost(INSN_COST * 7);
10176 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10177
10178 ins_encode %{
10179 __ umulh(as_Register($dst$$reg),
10180 as_Register($src1$$reg),
10181 as_Register($src2$$reg));
10182 %}
10183
10184 ins_pipe(lmul_reg_reg);
10185 %}
10186
10187 // Combined Integer Multiply & Add/Sub
10188
10189 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10190 match(Set dst (AddI src3 (MulI src1 src2)));
10191
10192 ins_cost(INSN_COST * 3);
10193 format %{ "madd $dst, $src1, $src2, $src3" %}
10194
10195 ins_encode %{
10196 __ maddw(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg),
10199 as_Register($src3$$reg));
10200 %}
10201
10202 ins_pipe(imac_reg_reg);
10203 %}
10204
10205 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10206 match(Set dst (SubI src3 (MulI src1 src2)));
10207
10208 ins_cost(INSN_COST * 3);
10209 format %{ "msub $dst, $src1, $src2, $src3" %}
10210
10211 ins_encode %{
10212 __ msubw(as_Register($dst$$reg),
10213 as_Register($src1$$reg),
10214 as_Register($src2$$reg),
10215 as_Register($src3$$reg));
10216 %}
10217
10218 ins_pipe(imac_reg_reg);
10219 %}
10220
10221 // Combined Integer Multiply & Neg
10222
10223 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10224 match(Set dst (MulI (SubI zero src1) src2));
10225
10226 ins_cost(INSN_COST * 3);
10227 format %{ "mneg $dst, $src1, $src2" %}
10228
10229 ins_encode %{
10230 __ mnegw(as_Register($dst$$reg),
10231 as_Register($src1$$reg),
10232 as_Register($src2$$reg));
10233 %}
10234
10235 ins_pipe(imac_reg_reg);
10236 %}
10237
10238 // Combined Long Multiply & Add/Sub
10239
10240 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10241 match(Set dst (AddL src3 (MulL src1 src2)));
10242
10243 ins_cost(INSN_COST * 5);
10244 format %{ "madd $dst, $src1, $src2, $src3" %}
10245
10246 ins_encode %{
10247 __ madd(as_Register($dst$$reg),
10248 as_Register($src1$$reg),
10249 as_Register($src2$$reg),
10250 as_Register($src3$$reg));
10251 %}
10252
10253 ins_pipe(lmac_reg_reg);
10254 %}
10255
10256 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10257 match(Set dst (SubL src3 (MulL src1 src2)));
10258
10259 ins_cost(INSN_COST * 5);
10260 format %{ "msub $dst, $src1, $src2, $src3" %}
10261
10262 ins_encode %{
10263 __ msub(as_Register($dst$$reg),
10264 as_Register($src1$$reg),
10265 as_Register($src2$$reg),
10266 as_Register($src3$$reg));
10267 %}
10268
10269 ins_pipe(lmac_reg_reg);
10270 %}
10271
10272 // Combined Long Multiply & Neg
10273
10274 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10275 match(Set dst (MulL (SubL zero src1) src2));
10276
10277 ins_cost(INSN_COST * 5);
10278 format %{ "mneg $dst, $src1, $src2" %}
10279
10280 ins_encode %{
10281 __ mneg(as_Register($dst$$reg),
10282 as_Register($src1$$reg),
10283 as_Register($src2$$reg));
10284 %}
10285
10286 ins_pipe(lmac_reg_reg);
10287 %}
10288
10289 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10290
10291 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10292 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10293
10294 ins_cost(INSN_COST * 3);
10295 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10296
10297 ins_encode %{
10298 __ smaddl(as_Register($dst$$reg),
10299 as_Register($src1$$reg),
10300 as_Register($src2$$reg),
10301 as_Register($src3$$reg));
10302 %}
10303
10304 ins_pipe(imac_reg_reg);
10305 %}
10306
10307 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10308 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10309
10310 ins_cost(INSN_COST * 3);
10311 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10312
10313 ins_encode %{
10314 __ smsubl(as_Register($dst$$reg),
10315 as_Register($src1$$reg),
10316 as_Register($src2$$reg),
10317 as_Register($src3$$reg));
10318 %}
10319
10320 ins_pipe(imac_reg_reg);
10321 %}
10322
10323 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10324 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10325
10326 ins_cost(INSN_COST * 3);
10327 format %{ "smnegl $dst, $src1, $src2" %}
10328
10329 ins_encode %{
10330 __ smnegl(as_Register($dst$$reg),
10331 as_Register($src1$$reg),
10332 as_Register($src2$$reg));
10333 %}
10334
10335 ins_pipe(imac_reg_reg);
10336 %}
10337
10338 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10339
10340 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10341 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10342
10343 ins_cost(INSN_COST * 5);
10344 format %{ "mulw rscratch1, $src1, $src2\n\t"
10345 "maddw $dst, $src3, $src4, rscratch1" %}
10346
10347 ins_encode %{
10348 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10349 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10350
10351 ins_pipe(imac_reg_reg);
10352 %}
10353
10354 // Integer Divide
10355
10356 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10357 match(Set dst (DivI src1 src2));
10358
10359 ins_cost(INSN_COST * 19);
10360 format %{ "sdivw $dst, $src1, $src2" %}
10361
10362 ins_encode(aarch64_enc_divw(dst, src1, src2));
10363 ins_pipe(idiv_reg_reg);
10364 %}
10365
10366 // Long Divide
10367
10368 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10369 match(Set dst (DivL src1 src2));
10370
10371 ins_cost(INSN_COST * 35);
10372 format %{ "sdiv $dst, $src1, $src2" %}
10373
10374 ins_encode(aarch64_enc_div(dst, src1, src2));
10375 ins_pipe(ldiv_reg_reg);
10376 %}
10377
10378 // Integer Remainder
10379
10380 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10381 match(Set dst (ModI src1 src2));
10382
10383 ins_cost(INSN_COST * 22);
10384 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10385 "msubw $dst, rscratch1, $src2, $src1" %}
10386
10387 ins_encode(aarch64_enc_modw(dst, src1, src2));
10388 ins_pipe(idiv_reg_reg);
10389 %}
10390
10391 // Long Remainder
10392
10393 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10394 match(Set dst (ModL src1 src2));
10395
10396 ins_cost(INSN_COST * 38);
10397 format %{ "sdiv rscratch1, $src1, $src2\n"
10398 "msub $dst, rscratch1, $src2, $src1" %}
10399
10400 ins_encode(aarch64_enc_mod(dst, src1, src2));
10401 ins_pipe(ldiv_reg_reg);
10402 %}
10403
10404 // Unsigned Integer Divide
10405
10406 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10407 match(Set dst (UDivI src1 src2));
10408
10409 ins_cost(INSN_COST * 19);
10410 format %{ "udivw $dst, $src1, $src2" %}
10411
10412 ins_encode %{
10413 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10414 %}
10415
10416 ins_pipe(idiv_reg_reg);
10417 %}
10418
10419 // Unsigned Long Divide
10420
10421 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10422 match(Set dst (UDivL src1 src2));
10423
10424 ins_cost(INSN_COST * 35);
10425 format %{ "udiv $dst, $src1, $src2" %}
10426
10427 ins_encode %{
10428 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10429 %}
10430
10431 ins_pipe(ldiv_reg_reg);
10432 %}
10433
10434 // Unsigned Integer Remainder
10435
10436 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10437 match(Set dst (UModI src1 src2));
10438
10439 ins_cost(INSN_COST * 22);
10440 format %{ "udivw rscratch1, $src1, $src2\n\t"
10441 "msubw $dst, rscratch1, $src2, $src1" %}
10442
10443 ins_encode %{
10444 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10445 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10446 %}
10447
10448 ins_pipe(idiv_reg_reg);
10449 %}
10450
10451 // Unsigned Long Remainder
10452
10453 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10454 match(Set dst (UModL src1 src2));
10455
10456 ins_cost(INSN_COST * 38);
10457 format %{ "udiv rscratch1, $src1, $src2\n"
10458 "msub $dst, rscratch1, $src2, $src1" %}
10459
10460 ins_encode %{
10461 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10462 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10463 %}
10464
10465 ins_pipe(ldiv_reg_reg);
10466 %}
10467
10468 // Integer Shifts
10469
10470 // Shift Left Register
10471 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10472 match(Set dst (LShiftI src1 src2));
10473
10474 ins_cost(INSN_COST * 2);
10475 format %{ "lslvw $dst, $src1, $src2" %}
10476
10477 ins_encode %{
10478 __ lslvw(as_Register($dst$$reg),
10479 as_Register($src1$$reg),
10480 as_Register($src2$$reg));
10481 %}
10482
10483 ins_pipe(ialu_reg_reg_vshift);
10484 %}
10485
10486 // Shift Left Immediate
10487 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10488 match(Set dst (LShiftI src1 src2));
10489
10490 ins_cost(INSN_COST);
10491 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10492
10493 ins_encode %{
10494 __ lslw(as_Register($dst$$reg),
10495 as_Register($src1$$reg),
10496 $src2$$constant & 0x1f);
10497 %}
10498
10499 ins_pipe(ialu_reg_shift);
10500 %}
10501
10502 // Shift Right Logical Register
10503 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10504 match(Set dst (URShiftI src1 src2));
10505
10506 ins_cost(INSN_COST * 2);
10507 format %{ "lsrvw $dst, $src1, $src2" %}
10508
10509 ins_encode %{
10510 __ lsrvw(as_Register($dst$$reg),
10511 as_Register($src1$$reg),
10512 as_Register($src2$$reg));
10513 %}
10514
10515 ins_pipe(ialu_reg_reg_vshift);
10516 %}
10517
10518 // Shift Right Logical Immediate
10519 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10520 match(Set dst (URShiftI src1 src2));
10521
10522 ins_cost(INSN_COST);
10523 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10524
10525 ins_encode %{
10526 __ lsrw(as_Register($dst$$reg),
10527 as_Register($src1$$reg),
10528 $src2$$constant & 0x1f);
10529 %}
10530
10531 ins_pipe(ialu_reg_shift);
10532 %}
10533
10534 // Shift Right Arithmetic Register
10535 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10536 match(Set dst (RShiftI src1 src2));
10537
10538 ins_cost(INSN_COST * 2);
10539 format %{ "asrvw $dst, $src1, $src2" %}
10540
10541 ins_encode %{
10542 __ asrvw(as_Register($dst$$reg),
10543 as_Register($src1$$reg),
10544 as_Register($src2$$reg));
10545 %}
10546
10547 ins_pipe(ialu_reg_reg_vshift);
10548 %}
10549
10550 // Shift Right Arithmetic Immediate
10551 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10552 match(Set dst (RShiftI src1 src2));
10553
10554 ins_cost(INSN_COST);
10555 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10556
10557 ins_encode %{
10558 __ asrw(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 $src2$$constant & 0x1f);
10561 %}
10562
10563 ins_pipe(ialu_reg_shift);
10564 %}
10565
10566 // Combined Int Mask and Right Shift (using UBFM)
10567 // TODO
10568
10569 // Long Shifts
10570
10571 // Shift Left Register
10572 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10573 match(Set dst (LShiftL src1 src2));
10574
10575 ins_cost(INSN_COST * 2);
10576 format %{ "lslv $dst, $src1, $src2" %}
10577
10578 ins_encode %{
10579 __ lslv(as_Register($dst$$reg),
10580 as_Register($src1$$reg),
10581 as_Register($src2$$reg));
10582 %}
10583
10584 ins_pipe(ialu_reg_reg_vshift);
10585 %}
10586
10587 // Shift Left Immediate
10588 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10589 match(Set dst (LShiftL src1 src2));
10590
10591 ins_cost(INSN_COST);
10592 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10593
10594 ins_encode %{
10595 __ lsl(as_Register($dst$$reg),
10596 as_Register($src1$$reg),
10597 $src2$$constant & 0x3f);
10598 %}
10599
10600 ins_pipe(ialu_reg_shift);
10601 %}
10602
10603 // Shift Right Logical Register
10604 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10605 match(Set dst (URShiftL src1 src2));
10606
10607 ins_cost(INSN_COST * 2);
10608 format %{ "lsrv $dst, $src1, $src2" %}
10609
10610 ins_encode %{
10611 __ lsrv(as_Register($dst$$reg),
10612 as_Register($src1$$reg),
10613 as_Register($src2$$reg));
10614 %}
10615
10616 ins_pipe(ialu_reg_reg_vshift);
10617 %}
10618
10619 // Shift Right Logical Immediate
10620 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10621 match(Set dst (URShiftL src1 src2));
10622
10623 ins_cost(INSN_COST);
10624 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10625
10626 ins_encode %{
10627 __ lsr(as_Register($dst$$reg),
10628 as_Register($src1$$reg),
10629 $src2$$constant & 0x3f);
10630 %}
10631
10632 ins_pipe(ialu_reg_shift);
10633 %}
10634
10635 // A special-case pattern for card table stores.
10636 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10637 match(Set dst (URShiftL (CastP2X src1) src2));
10638
10639 ins_cost(INSN_COST);
10640 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10641
10642 ins_encode %{
10643 __ lsr(as_Register($dst$$reg),
10644 as_Register($src1$$reg),
10645 $src2$$constant & 0x3f);
10646 %}
10647
10648 ins_pipe(ialu_reg_shift);
10649 %}
10650
10651 // Shift Right Arithmetic Register
10652 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10653 match(Set dst (RShiftL src1 src2));
10654
10655 ins_cost(INSN_COST * 2);
10656 format %{ "asrv $dst, $src1, $src2" %}
10657
10658 ins_encode %{
10659 __ asrv(as_Register($dst$$reg),
10660 as_Register($src1$$reg),
10661 as_Register($src2$$reg));
10662 %}
10663
10664 ins_pipe(ialu_reg_reg_vshift);
10665 %}
10666
10667 // Shift Right Arithmetic Immediate
10668 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10669 match(Set dst (RShiftL src1 src2));
10670
10671 ins_cost(INSN_COST);
10672 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10673
10674 ins_encode %{
10675 __ asr(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 $src2$$constant & 0x3f);
10678 %}
10679
10680 ins_pipe(ialu_reg_shift);
10681 %}
10682
10683 // BEGIN This section of the file is automatically generated. Do not edit --------------
10684 // This section is generated from aarch64_ad.m4
10685
10686 // This pattern is automatically generated from aarch64_ad.m4
10687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10688 instruct regL_not_reg(iRegLNoSp dst,
10689 iRegL src1, immL_M1 m1,
10690 rFlagsReg cr) %{
10691 match(Set dst (XorL src1 m1));
10692 ins_cost(INSN_COST);
10693 format %{ "eon $dst, $src1, zr" %}
10694
10695 ins_encode %{
10696 __ eon(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 zr,
10699 Assembler::LSL, 0);
10700 %}
10701
10702 ins_pipe(ialu_reg);
10703 %}
10704
10705 // This pattern is automatically generated from aarch64_ad.m4
10706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10707 instruct regI_not_reg(iRegINoSp dst,
10708 iRegIorL2I src1, immI_M1 m1,
10709 rFlagsReg cr) %{
10710 match(Set dst (XorI src1 m1));
10711 ins_cost(INSN_COST);
10712 format %{ "eonw $dst, $src1, zr" %}
10713
10714 ins_encode %{
10715 __ eonw(as_Register($dst$$reg),
10716 as_Register($src1$$reg),
10717 zr,
10718 Assembler::LSL, 0);
10719 %}
10720
10721 ins_pipe(ialu_reg);
10722 %}
10723
10724 // This pattern is automatically generated from aarch64_ad.m4
10725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10726 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10727 immI0 zero, iRegIorL2I src1, immI src2) %{
10728 match(Set dst (SubI zero (URShiftI src1 src2)));
10729
10730 ins_cost(1.9 * INSN_COST);
10731 format %{ "negw $dst, $src1, LSR $src2" %}
10732
10733 ins_encode %{
10734 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10735 Assembler::LSR, $src2$$constant & 0x1f);
10736 %}
10737
10738 ins_pipe(ialu_reg_shift);
10739 %}
10740
10741 // This pattern is automatically generated from aarch64_ad.m4
10742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10743 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10744 immI0 zero, iRegIorL2I src1, immI src2) %{
10745 match(Set dst (SubI zero (RShiftI src1 src2)));
10746
10747 ins_cost(1.9 * INSN_COST);
10748 format %{ "negw $dst, $src1, ASR $src2" %}
10749
10750 ins_encode %{
10751 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10752 Assembler::ASR, $src2$$constant & 0x1f);
10753 %}
10754
10755 ins_pipe(ialu_reg_shift);
10756 %}
10757
10758 // This pattern is automatically generated from aarch64_ad.m4
10759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10760 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10761 immI0 zero, iRegIorL2I src1, immI src2) %{
10762 match(Set dst (SubI zero (LShiftI src1 src2)));
10763
10764 ins_cost(1.9 * INSN_COST);
10765 format %{ "negw $dst, $src1, LSL $src2" %}
10766
10767 ins_encode %{
10768 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10769 Assembler::LSL, $src2$$constant & 0x1f);
10770 %}
10771
10772 ins_pipe(ialu_reg_shift);
10773 %}
10774
10775 // This pattern is automatically generated from aarch64_ad.m4
10776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10777 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10778 immL0 zero, iRegL src1, immI src2) %{
10779 match(Set dst (SubL zero (URShiftL src1 src2)));
10780
10781 ins_cost(1.9 * INSN_COST);
10782 format %{ "neg $dst, $src1, LSR $src2" %}
10783
10784 ins_encode %{
10785 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10786 Assembler::LSR, $src2$$constant & 0x3f);
10787 %}
10788
10789 ins_pipe(ialu_reg_shift);
10790 %}
10791
10792 // This pattern is automatically generated from aarch64_ad.m4
10793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10794 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10795 immL0 zero, iRegL src1, immI src2) %{
10796 match(Set dst (SubL zero (RShiftL src1 src2)));
10797
10798 ins_cost(1.9 * INSN_COST);
10799 format %{ "neg $dst, $src1, ASR $src2" %}
10800
10801 ins_encode %{
10802 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10803 Assembler::ASR, $src2$$constant & 0x3f);
10804 %}
10805
10806 ins_pipe(ialu_reg_shift);
10807 %}
10808
10809 // This pattern is automatically generated from aarch64_ad.m4
10810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10811 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10812 immL0 zero, iRegL src1, immI src2) %{
10813 match(Set dst (SubL zero (LShiftL src1 src2)));
10814
10815 ins_cost(1.9 * INSN_COST);
10816 format %{ "neg $dst, $src1, LSL $src2" %}
10817
10818 ins_encode %{
10819 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10820 Assembler::LSL, $src2$$constant & 0x3f);
10821 %}
10822
10823 ins_pipe(ialu_reg_shift);
10824 %}
10825
10826 // This pattern is automatically generated from aarch64_ad.m4
10827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10828 instruct AndI_reg_not_reg(iRegINoSp dst,
10829 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10830 match(Set dst (AndI src1 (XorI src2 m1)));
10831 ins_cost(INSN_COST);
10832 format %{ "bicw $dst, $src1, $src2" %}
10833
10834 ins_encode %{
10835 __ bicw(as_Register($dst$$reg),
10836 as_Register($src1$$reg),
10837 as_Register($src2$$reg),
10838 Assembler::LSL, 0);
10839 %}
10840
10841 ins_pipe(ialu_reg_reg);
10842 %}
10843
10844 // This pattern is automatically generated from aarch64_ad.m4
10845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10846 instruct AndL_reg_not_reg(iRegLNoSp dst,
10847 iRegL src1, iRegL src2, immL_M1 m1) %{
10848 match(Set dst (AndL src1 (XorL src2 m1)));
10849 ins_cost(INSN_COST);
10850 format %{ "bic $dst, $src1, $src2" %}
10851
10852 ins_encode %{
10853 __ bic(as_Register($dst$$reg),
10854 as_Register($src1$$reg),
10855 as_Register($src2$$reg),
10856 Assembler::LSL, 0);
10857 %}
10858
10859 ins_pipe(ialu_reg_reg);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct OrI_reg_not_reg(iRegINoSp dst,
10865 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10866 match(Set dst (OrI src1 (XorI src2 m1)));
10867 ins_cost(INSN_COST);
10868 format %{ "ornw $dst, $src1, $src2" %}
10869
10870 ins_encode %{
10871 __ ornw(as_Register($dst$$reg),
10872 as_Register($src1$$reg),
10873 as_Register($src2$$reg),
10874 Assembler::LSL, 0);
10875 %}
10876
10877 ins_pipe(ialu_reg_reg);
10878 %}
10879
10880 // This pattern is automatically generated from aarch64_ad.m4
10881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10882 instruct OrL_reg_not_reg(iRegLNoSp dst,
10883 iRegL src1, iRegL src2, immL_M1 m1) %{
10884 match(Set dst (OrL src1 (XorL src2 m1)));
10885 ins_cost(INSN_COST);
10886 format %{ "orn $dst, $src1, $src2" %}
10887
10888 ins_encode %{
10889 __ orn(as_Register($dst$$reg),
10890 as_Register($src1$$reg),
10891 as_Register($src2$$reg),
10892 Assembler::LSL, 0);
10893 %}
10894
10895 ins_pipe(ialu_reg_reg);
10896 %}
10897
10898 // This pattern is automatically generated from aarch64_ad.m4
10899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10900 instruct XorI_reg_not_reg(iRegINoSp dst,
10901 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10902 match(Set dst (XorI m1 (XorI src2 src1)));
10903 ins_cost(INSN_COST);
10904 format %{ "eonw $dst, $src1, $src2" %}
10905
10906 ins_encode %{
10907 __ eonw(as_Register($dst$$reg),
10908 as_Register($src1$$reg),
10909 as_Register($src2$$reg),
10910 Assembler::LSL, 0);
10911 %}
10912
10913 ins_pipe(ialu_reg_reg);
10914 %}
10915
10916 // This pattern is automatically generated from aarch64_ad.m4
10917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10918 instruct XorL_reg_not_reg(iRegLNoSp dst,
10919 iRegL src1, iRegL src2, immL_M1 m1) %{
10920 match(Set dst (XorL m1 (XorL src2 src1)));
10921 ins_cost(INSN_COST);
10922 format %{ "eon $dst, $src1, $src2" %}
10923
10924 ins_encode %{
10925 __ eon(as_Register($dst$$reg),
10926 as_Register($src1$$reg),
10927 as_Register($src2$$reg),
10928 Assembler::LSL, 0);
10929 %}
10930
10931 ins_pipe(ialu_reg_reg);
10932 %}
10933
10934 // This pattern is automatically generated from aarch64_ad.m4
10935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10936 // val & (-1 ^ (val >>> shift)) ==> bicw
10937 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10938 iRegIorL2I src1, iRegIorL2I src2,
10939 immI src3, immI_M1 src4) %{
10940 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10941 ins_cost(1.9 * INSN_COST);
10942 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10943
10944 ins_encode %{
10945 __ bicw(as_Register($dst$$reg),
10946 as_Register($src1$$reg),
10947 as_Register($src2$$reg),
10948 Assembler::LSR,
10949 $src3$$constant & 0x1f);
10950 %}
10951
10952 ins_pipe(ialu_reg_reg_shift);
10953 %}
10954
10955 // This pattern is automatically generated from aarch64_ad.m4
10956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10957 // val & (-1 ^ (val >>> shift)) ==> bic
10958 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10959 iRegL src1, iRegL src2,
10960 immI src3, immL_M1 src4) %{
10961 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10962 ins_cost(1.9 * INSN_COST);
10963 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10964
10965 ins_encode %{
10966 __ bic(as_Register($dst$$reg),
10967 as_Register($src1$$reg),
10968 as_Register($src2$$reg),
10969 Assembler::LSR,
10970 $src3$$constant & 0x3f);
10971 %}
10972
10973 ins_pipe(ialu_reg_reg_shift);
10974 %}
10975
10976 // This pattern is automatically generated from aarch64_ad.m4
10977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10978 // val & (-1 ^ (val >> shift)) ==> bicw
10979 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10980 iRegIorL2I src1, iRegIorL2I src2,
10981 immI src3, immI_M1 src4) %{
10982 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10983 ins_cost(1.9 * INSN_COST);
10984 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10985
10986 ins_encode %{
10987 __ bicw(as_Register($dst$$reg),
10988 as_Register($src1$$reg),
10989 as_Register($src2$$reg),
10990 Assembler::ASR,
10991 $src3$$constant & 0x1f);
10992 %}
10993
10994 ins_pipe(ialu_reg_reg_shift);
10995 %}
10996
10997 // This pattern is automatically generated from aarch64_ad.m4
10998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10999 // val & (-1 ^ (val >> shift)) ==> bic
11000 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11001 iRegL src1, iRegL src2,
11002 immI src3, immL_M1 src4) %{
11003 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11004 ins_cost(1.9 * INSN_COST);
11005 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11006
11007 ins_encode %{
11008 __ bic(as_Register($dst$$reg),
11009 as_Register($src1$$reg),
11010 as_Register($src2$$reg),
11011 Assembler::ASR,
11012 $src3$$constant & 0x3f);
11013 %}
11014
11015 ins_pipe(ialu_reg_reg_shift);
11016 %}
11017
11018 // This pattern is automatically generated from aarch64_ad.m4
11019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11020 // val & (-1 ^ (val ror shift)) ==> bicw
11021 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11022 iRegIorL2I src1, iRegIorL2I src2,
11023 immI src3, immI_M1 src4) %{
11024 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11025 ins_cost(1.9 * INSN_COST);
11026 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11027
11028 ins_encode %{
11029 __ bicw(as_Register($dst$$reg),
11030 as_Register($src1$$reg),
11031 as_Register($src2$$reg),
11032 Assembler::ROR,
11033 $src3$$constant & 0x1f);
11034 %}
11035
11036 ins_pipe(ialu_reg_reg_shift);
11037 %}
11038
11039 // This pattern is automatically generated from aarch64_ad.m4
11040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11041 // val & (-1 ^ (val ror shift)) ==> bic
11042 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11043 iRegL src1, iRegL src2,
11044 immI src3, immL_M1 src4) %{
11045 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11046 ins_cost(1.9 * INSN_COST);
11047 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11048
11049 ins_encode %{
11050 __ bic(as_Register($dst$$reg),
11051 as_Register($src1$$reg),
11052 as_Register($src2$$reg),
11053 Assembler::ROR,
11054 $src3$$constant & 0x3f);
11055 %}
11056
11057 ins_pipe(ialu_reg_reg_shift);
11058 %}
11059
11060 // This pattern is automatically generated from aarch64_ad.m4
11061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11062 // val & (-1 ^ (val << shift)) ==> bicw
11063 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11064 iRegIorL2I src1, iRegIorL2I src2,
11065 immI src3, immI_M1 src4) %{
11066 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11067 ins_cost(1.9 * INSN_COST);
11068 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11069
11070 ins_encode %{
11071 __ bicw(as_Register($dst$$reg),
11072 as_Register($src1$$reg),
11073 as_Register($src2$$reg),
11074 Assembler::LSL,
11075 $src3$$constant & 0x1f);
11076 %}
11077
11078 ins_pipe(ialu_reg_reg_shift);
11079 %}
11080
11081 // This pattern is automatically generated from aarch64_ad.m4
11082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11083 // val & (-1 ^ (val << shift)) ==> bic
11084 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11085 iRegL src1, iRegL src2,
11086 immI src3, immL_M1 src4) %{
11087 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11088 ins_cost(1.9 * INSN_COST);
11089 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11090
11091 ins_encode %{
11092 __ bic(as_Register($dst$$reg),
11093 as_Register($src1$$reg),
11094 as_Register($src2$$reg),
11095 Assembler::LSL,
11096 $src3$$constant & 0x3f);
11097 %}
11098
11099 ins_pipe(ialu_reg_reg_shift);
11100 %}
11101
11102 // This pattern is automatically generated from aarch64_ad.m4
11103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11104 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11105 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11106 iRegIorL2I src1, iRegIorL2I src2,
11107 immI src3, immI_M1 src4) %{
11108 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11109 ins_cost(1.9 * INSN_COST);
11110 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11111
11112 ins_encode %{
11113 __ eonw(as_Register($dst$$reg),
11114 as_Register($src1$$reg),
11115 as_Register($src2$$reg),
11116 Assembler::LSR,
11117 $src3$$constant & 0x1f);
11118 %}
11119
11120 ins_pipe(ialu_reg_reg_shift);
11121 %}
11122
11123 // This pattern is automatically generated from aarch64_ad.m4
11124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11125 // val ^ (-1 ^ (val >>> shift)) ==> eon
11126 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11127 iRegL src1, iRegL src2,
11128 immI src3, immL_M1 src4) %{
11129 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11130 ins_cost(1.9 * INSN_COST);
11131 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11132
11133 ins_encode %{
11134 __ eon(as_Register($dst$$reg),
11135 as_Register($src1$$reg),
11136 as_Register($src2$$reg),
11137 Assembler::LSR,
11138 $src3$$constant & 0x3f);
11139 %}
11140
11141 ins_pipe(ialu_reg_reg_shift);
11142 %}
11143
11144 // This pattern is automatically generated from aarch64_ad.m4
11145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11146 // val ^ (-1 ^ (val >> shift)) ==> eonw
11147 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11148 iRegIorL2I src1, iRegIorL2I src2,
11149 immI src3, immI_M1 src4) %{
11150 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11151 ins_cost(1.9 * INSN_COST);
11152 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11153
11154 ins_encode %{
11155 __ eonw(as_Register($dst$$reg),
11156 as_Register($src1$$reg),
11157 as_Register($src2$$reg),
11158 Assembler::ASR,
11159 $src3$$constant & 0x1f);
11160 %}
11161
11162 ins_pipe(ialu_reg_reg_shift);
11163 %}
11164
11165 // This pattern is automatically generated from aarch64_ad.m4
11166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11167 // val ^ (-1 ^ (val >> shift)) ==> eon
11168 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11169 iRegL src1, iRegL src2,
11170 immI src3, immL_M1 src4) %{
11171 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11172 ins_cost(1.9 * INSN_COST);
11173 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11174
11175 ins_encode %{
11176 __ eon(as_Register($dst$$reg),
11177 as_Register($src1$$reg),
11178 as_Register($src2$$reg),
11179 Assembler::ASR,
11180 $src3$$constant & 0x3f);
11181 %}
11182
11183 ins_pipe(ialu_reg_reg_shift);
11184 %}
11185
11186 // This pattern is automatically generated from aarch64_ad.m4
11187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11188 // val ^ (-1 ^ (val ror shift)) ==> eonw
11189 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11190 iRegIorL2I src1, iRegIorL2I src2,
11191 immI src3, immI_M1 src4) %{
11192 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11193 ins_cost(1.9 * INSN_COST);
11194 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11195
11196 ins_encode %{
11197 __ eonw(as_Register($dst$$reg),
11198 as_Register($src1$$reg),
11199 as_Register($src2$$reg),
11200 Assembler::ROR,
11201 $src3$$constant & 0x1f);
11202 %}
11203
11204 ins_pipe(ialu_reg_reg_shift);
11205 %}
11206
11207 // This pattern is automatically generated from aarch64_ad.m4
11208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11209 // val ^ (-1 ^ (val ror shift)) ==> eon
11210 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11211 iRegL src1, iRegL src2,
11212 immI src3, immL_M1 src4) %{
11213 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11214 ins_cost(1.9 * INSN_COST);
11215 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11216
11217 ins_encode %{
11218 __ eon(as_Register($dst$$reg),
11219 as_Register($src1$$reg),
11220 as_Register($src2$$reg),
11221 Assembler::ROR,
11222 $src3$$constant & 0x3f);
11223 %}
11224
11225 ins_pipe(ialu_reg_reg_shift);
11226 %}
11227
11228 // This pattern is automatically generated from aarch64_ad.m4
11229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11230 // val ^ (-1 ^ (val << shift)) ==> eonw
11231 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11232 iRegIorL2I src1, iRegIorL2I src2,
11233 immI src3, immI_M1 src4) %{
11234 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11235 ins_cost(1.9 * INSN_COST);
11236 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11237
11238 ins_encode %{
11239 __ eonw(as_Register($dst$$reg),
11240 as_Register($src1$$reg),
11241 as_Register($src2$$reg),
11242 Assembler::LSL,
11243 $src3$$constant & 0x1f);
11244 %}
11245
11246 ins_pipe(ialu_reg_reg_shift);
11247 %}
11248
11249 // This pattern is automatically generated from aarch64_ad.m4
11250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11251 // val ^ (-1 ^ (val << shift)) ==> eon
11252 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11253 iRegL src1, iRegL src2,
11254 immI src3, immL_M1 src4) %{
11255 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11256 ins_cost(1.9 * INSN_COST);
11257 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11258
11259 ins_encode %{
11260 __ eon(as_Register($dst$$reg),
11261 as_Register($src1$$reg),
11262 as_Register($src2$$reg),
11263 Assembler::LSL,
11264 $src3$$constant & 0x3f);
11265 %}
11266
11267 ins_pipe(ialu_reg_reg_shift);
11268 %}
11269
11270 // This pattern is automatically generated from aarch64_ad.m4
11271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11272 // val | (-1 ^ (val >>> shift)) ==> ornw
11273 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11274 iRegIorL2I src1, iRegIorL2I src2,
11275 immI src3, immI_M1 src4) %{
11276 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11277 ins_cost(1.9 * INSN_COST);
11278 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11279
11280 ins_encode %{
11281 __ ornw(as_Register($dst$$reg),
11282 as_Register($src1$$reg),
11283 as_Register($src2$$reg),
11284 Assembler::LSR,
11285 $src3$$constant & 0x1f);
11286 %}
11287
11288 ins_pipe(ialu_reg_reg_shift);
11289 %}
11290
11291 // This pattern is automatically generated from aarch64_ad.m4
11292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11293 // val | (-1 ^ (val >>> shift)) ==> orn
11294 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11295 iRegL src1, iRegL src2,
11296 immI src3, immL_M1 src4) %{
11297 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11298 ins_cost(1.9 * INSN_COST);
11299 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11300
11301 ins_encode %{
11302 __ orn(as_Register($dst$$reg),
11303 as_Register($src1$$reg),
11304 as_Register($src2$$reg),
11305 Assembler::LSR,
11306 $src3$$constant & 0x3f);
11307 %}
11308
11309 ins_pipe(ialu_reg_reg_shift);
11310 %}
11311
11312 // This pattern is automatically generated from aarch64_ad.m4
11313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11314 // val | (-1 ^ (val >> shift)) ==> ornw
11315 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11316 iRegIorL2I src1, iRegIorL2I src2,
11317 immI src3, immI_M1 src4) %{
11318 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11319 ins_cost(1.9 * INSN_COST);
11320 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11321
11322 ins_encode %{
11323 __ ornw(as_Register($dst$$reg),
11324 as_Register($src1$$reg),
11325 as_Register($src2$$reg),
11326 Assembler::ASR,
11327 $src3$$constant & 0x1f);
11328 %}
11329
11330 ins_pipe(ialu_reg_reg_shift);
11331 %}
11332
11333 // This pattern is automatically generated from aarch64_ad.m4
11334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11335 // val | (-1 ^ (val >> shift)) ==> orn
11336 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11337 iRegL src1, iRegL src2,
11338 immI src3, immL_M1 src4) %{
11339 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11340 ins_cost(1.9 * INSN_COST);
11341 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11342
11343 ins_encode %{
11344 __ orn(as_Register($dst$$reg),
11345 as_Register($src1$$reg),
11346 as_Register($src2$$reg),
11347 Assembler::ASR,
11348 $src3$$constant & 0x3f);
11349 %}
11350
11351 ins_pipe(ialu_reg_reg_shift);
11352 %}
11353
11354 // This pattern is automatically generated from aarch64_ad.m4
11355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11356 // val | (-1 ^ (val ror shift)) ==> ornw
11357 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11358 iRegIorL2I src1, iRegIorL2I src2,
11359 immI src3, immI_M1 src4) %{
11360 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11361 ins_cost(1.9 * INSN_COST);
11362 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11363
11364 ins_encode %{
11365 __ ornw(as_Register($dst$$reg),
11366 as_Register($src1$$reg),
11367 as_Register($src2$$reg),
11368 Assembler::ROR,
11369 $src3$$constant & 0x1f);
11370 %}
11371
11372 ins_pipe(ialu_reg_reg_shift);
11373 %}
11374
11375 // This pattern is automatically generated from aarch64_ad.m4
11376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11377 // val | (-1 ^ (val ror shift)) ==> orn
11378 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11379 iRegL src1, iRegL src2,
11380 immI src3, immL_M1 src4) %{
11381 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11382 ins_cost(1.9 * INSN_COST);
11383 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11384
11385 ins_encode %{
11386 __ orn(as_Register($dst$$reg),
11387 as_Register($src1$$reg),
11388 as_Register($src2$$reg),
11389 Assembler::ROR,
11390 $src3$$constant & 0x3f);
11391 %}
11392
11393 ins_pipe(ialu_reg_reg_shift);
11394 %}
11395
11396 // This pattern is automatically generated from aarch64_ad.m4
11397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11398 // val | (-1 ^ (val << shift)) ==> ornw
11399 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11400 iRegIorL2I src1, iRegIorL2I src2,
11401 immI src3, immI_M1 src4) %{
11402 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11403 ins_cost(1.9 * INSN_COST);
11404 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11405
11406 ins_encode %{
11407 __ ornw(as_Register($dst$$reg),
11408 as_Register($src1$$reg),
11409 as_Register($src2$$reg),
11410 Assembler::LSL,
11411 $src3$$constant & 0x1f);
11412 %}
11413
11414 ins_pipe(ialu_reg_reg_shift);
11415 %}
11416
11417 // This pattern is automatically generated from aarch64_ad.m4
11418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11419 // val | (-1 ^ (val << shift)) ==> orn
11420 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11421 iRegL src1, iRegL src2,
11422 immI src3, immL_M1 src4) %{
11423 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11424 ins_cost(1.9 * INSN_COST);
11425 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11426
11427 ins_encode %{
11428 __ orn(as_Register($dst$$reg),
11429 as_Register($src1$$reg),
11430 as_Register($src2$$reg),
11431 Assembler::LSL,
11432 $src3$$constant & 0x3f);
11433 %}
11434
11435 ins_pipe(ialu_reg_reg_shift);
11436 %}
11437
11438 // This pattern is automatically generated from aarch64_ad.m4
11439 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11440 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11441 iRegIorL2I src1, iRegIorL2I src2,
11442 immI src3) %{
11443 match(Set dst (AndI src1 (URShiftI src2 src3)));
11444
11445 ins_cost(1.9 * INSN_COST);
11446 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11447
11448 ins_encode %{
11449 __ andw(as_Register($dst$$reg),
11450 as_Register($src1$$reg),
11451 as_Register($src2$$reg),
11452 Assembler::LSR,
11453 $src3$$constant & 0x1f);
11454 %}
11455
11456 ins_pipe(ialu_reg_reg_shift);
11457 %}
11458
11459 // This pattern is automatically generated from aarch64_ad.m4
11460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11461 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11462 iRegL src1, iRegL src2,
11463 immI src3) %{
11464 match(Set dst (AndL src1 (URShiftL src2 src3)));
11465
11466 ins_cost(1.9 * INSN_COST);
11467 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11468
11469 ins_encode %{
11470 __ andr(as_Register($dst$$reg),
11471 as_Register($src1$$reg),
11472 as_Register($src2$$reg),
11473 Assembler::LSR,
11474 $src3$$constant & 0x3f);
11475 %}
11476
11477 ins_pipe(ialu_reg_reg_shift);
11478 %}
11479
11480 // This pattern is automatically generated from aarch64_ad.m4
11481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11482 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11483 iRegIorL2I src1, iRegIorL2I src2,
11484 immI src3) %{
11485 match(Set dst (AndI src1 (RShiftI src2 src3)));
11486
11487 ins_cost(1.9 * INSN_COST);
11488 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11489
11490 ins_encode %{
11491 __ andw(as_Register($dst$$reg),
11492 as_Register($src1$$reg),
11493 as_Register($src2$$reg),
11494 Assembler::ASR,
11495 $src3$$constant & 0x1f);
11496 %}
11497
11498 ins_pipe(ialu_reg_reg_shift);
11499 %}
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11504 iRegL src1, iRegL src2,
11505 immI src3) %{
11506 match(Set dst (AndL src1 (RShiftL src2 src3)));
11507
11508 ins_cost(1.9 * INSN_COST);
11509 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11510
11511 ins_encode %{
11512 __ andr(as_Register($dst$$reg),
11513 as_Register($src1$$reg),
11514 as_Register($src2$$reg),
11515 Assembler::ASR,
11516 $src3$$constant & 0x3f);
11517 %}
11518
11519 ins_pipe(ialu_reg_reg_shift);
11520 %}
11521
11522 // This pattern is automatically generated from aarch64_ad.m4
11523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11524 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11525 iRegIorL2I src1, iRegIorL2I src2,
11526 immI src3) %{
11527 match(Set dst (AndI src1 (LShiftI src2 src3)));
11528
11529 ins_cost(1.9 * INSN_COST);
11530 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11531
11532 ins_encode %{
11533 __ andw(as_Register($dst$$reg),
11534 as_Register($src1$$reg),
11535 as_Register($src2$$reg),
11536 Assembler::LSL,
11537 $src3$$constant & 0x1f);
11538 %}
11539
11540 ins_pipe(ialu_reg_reg_shift);
11541 %}
11542
11543 // This pattern is automatically generated from aarch64_ad.m4
11544 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11545 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11546 iRegL src1, iRegL src2,
11547 immI src3) %{
11548 match(Set dst (AndL src1 (LShiftL src2 src3)));
11549
11550 ins_cost(1.9 * INSN_COST);
11551 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11552
11553 ins_encode %{
11554 __ andr(as_Register($dst$$reg),
11555 as_Register($src1$$reg),
11556 as_Register($src2$$reg),
11557 Assembler::LSL,
11558 $src3$$constant & 0x3f);
11559 %}
11560
11561 ins_pipe(ialu_reg_reg_shift);
11562 %}
11563
11564 // This pattern is automatically generated from aarch64_ad.m4
11565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11566 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11567 iRegIorL2I src1, iRegIorL2I src2,
11568 immI src3) %{
11569 match(Set dst (AndI src1 (RotateRight src2 src3)));
11570
11571 ins_cost(1.9 * INSN_COST);
11572 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11573
11574 ins_encode %{
11575 __ andw(as_Register($dst$$reg),
11576 as_Register($src1$$reg),
11577 as_Register($src2$$reg),
11578 Assembler::ROR,
11579 $src3$$constant & 0x1f);
11580 %}
11581
11582 ins_pipe(ialu_reg_reg_shift);
11583 %}
11584
11585 // This pattern is automatically generated from aarch64_ad.m4
11586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11587 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11588 iRegL src1, iRegL src2,
11589 immI src3) %{
11590 match(Set dst (AndL src1 (RotateRight src2 src3)));
11591
11592 ins_cost(1.9 * INSN_COST);
11593 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11594
11595 ins_encode %{
11596 __ andr(as_Register($dst$$reg),
11597 as_Register($src1$$reg),
11598 as_Register($src2$$reg),
11599 Assembler::ROR,
11600 $src3$$constant & 0x3f);
11601 %}
11602
11603 ins_pipe(ialu_reg_reg_shift);
11604 %}
11605
11606 // This pattern is automatically generated from aarch64_ad.m4
11607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11608 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11609 iRegIorL2I src1, iRegIorL2I src2,
11610 immI src3) %{
11611 match(Set dst (XorI src1 (URShiftI src2 src3)));
11612
11613 ins_cost(1.9 * INSN_COST);
11614 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11615
11616 ins_encode %{
11617 __ eorw(as_Register($dst$$reg),
11618 as_Register($src1$$reg),
11619 as_Register($src2$$reg),
11620 Assembler::LSR,
11621 $src3$$constant & 0x1f);
11622 %}
11623
11624 ins_pipe(ialu_reg_reg_shift);
11625 %}
11626
11627 // This pattern is automatically generated from aarch64_ad.m4
11628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11629 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11630 iRegL src1, iRegL src2,
11631 immI src3) %{
11632 match(Set dst (XorL src1 (URShiftL src2 src3)));
11633
11634 ins_cost(1.9 * INSN_COST);
11635 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11636
11637 ins_encode %{
11638 __ eor(as_Register($dst$$reg),
11639 as_Register($src1$$reg),
11640 as_Register($src2$$reg),
11641 Assembler::LSR,
11642 $src3$$constant & 0x3f);
11643 %}
11644
11645 ins_pipe(ialu_reg_reg_shift);
11646 %}
11647
11648 // This pattern is automatically generated from aarch64_ad.m4
11649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11650 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11651 iRegIorL2I src1, iRegIorL2I src2,
11652 immI src3) %{
11653 match(Set dst (XorI src1 (RShiftI src2 src3)));
11654
11655 ins_cost(1.9 * INSN_COST);
11656 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11657
11658 ins_encode %{
11659 __ eorw(as_Register($dst$$reg),
11660 as_Register($src1$$reg),
11661 as_Register($src2$$reg),
11662 Assembler::ASR,
11663 $src3$$constant & 0x1f);
11664 %}
11665
11666 ins_pipe(ialu_reg_reg_shift);
11667 %}
11668
11669 // This pattern is automatically generated from aarch64_ad.m4
11670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11671 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11672 iRegL src1, iRegL src2,
11673 immI src3) %{
11674 match(Set dst (XorL src1 (RShiftL src2 src3)));
11675
11676 ins_cost(1.9 * INSN_COST);
11677 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11678
11679 ins_encode %{
11680 __ eor(as_Register($dst$$reg),
11681 as_Register($src1$$reg),
11682 as_Register($src2$$reg),
11683 Assembler::ASR,
11684 $src3$$constant & 0x3f);
11685 %}
11686
11687 ins_pipe(ialu_reg_reg_shift);
11688 %}
11689
11690 // This pattern is automatically generated from aarch64_ad.m4
11691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11692 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11693 iRegIorL2I src1, iRegIorL2I src2,
11694 immI src3) %{
11695 match(Set dst (XorI src1 (LShiftI src2 src3)));
11696
11697 ins_cost(1.9 * INSN_COST);
11698 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11699
11700 ins_encode %{
11701 __ eorw(as_Register($dst$$reg),
11702 as_Register($src1$$reg),
11703 as_Register($src2$$reg),
11704 Assembler::LSL,
11705 $src3$$constant & 0x1f);
11706 %}
11707
11708 ins_pipe(ialu_reg_reg_shift);
11709 %}
11710
11711 // This pattern is automatically generated from aarch64_ad.m4
11712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11713 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11714 iRegL src1, iRegL src2,
11715 immI src3) %{
11716 match(Set dst (XorL src1 (LShiftL src2 src3)));
11717
11718 ins_cost(1.9 * INSN_COST);
11719 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11720
11721 ins_encode %{
11722 __ eor(as_Register($dst$$reg),
11723 as_Register($src1$$reg),
11724 as_Register($src2$$reg),
11725 Assembler::LSL,
11726 $src3$$constant & 0x3f);
11727 %}
11728
11729 ins_pipe(ialu_reg_reg_shift);
11730 %}
11731
11732 // This pattern is automatically generated from aarch64_ad.m4
11733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11734 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11735 iRegIorL2I src1, iRegIorL2I src2,
11736 immI src3) %{
11737 match(Set dst (XorI src1 (RotateRight src2 src3)));
11738
11739 ins_cost(1.9 * INSN_COST);
11740 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11741
11742 ins_encode %{
11743 __ eorw(as_Register($dst$$reg),
11744 as_Register($src1$$reg),
11745 as_Register($src2$$reg),
11746 Assembler::ROR,
11747 $src3$$constant & 0x1f);
11748 %}
11749
11750 ins_pipe(ialu_reg_reg_shift);
11751 %}
11752
11753 // This pattern is automatically generated from aarch64_ad.m4
11754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11755 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11756 iRegL src1, iRegL src2,
11757 immI src3) %{
11758 match(Set dst (XorL src1 (RotateRight src2 src3)));
11759
11760 ins_cost(1.9 * INSN_COST);
11761 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11762
11763 ins_encode %{
11764 __ eor(as_Register($dst$$reg),
11765 as_Register($src1$$reg),
11766 as_Register($src2$$reg),
11767 Assembler::ROR,
11768 $src3$$constant & 0x3f);
11769 %}
11770
11771 ins_pipe(ialu_reg_reg_shift);
11772 %}
11773
11774 // This pattern is automatically generated from aarch64_ad.m4
11775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11776 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11777 iRegIorL2I src1, iRegIorL2I src2,
11778 immI src3) %{
11779 match(Set dst (OrI src1 (URShiftI src2 src3)));
11780
11781 ins_cost(1.9 * INSN_COST);
11782 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11783
11784 ins_encode %{
11785 __ orrw(as_Register($dst$$reg),
11786 as_Register($src1$$reg),
11787 as_Register($src2$$reg),
11788 Assembler::LSR,
11789 $src3$$constant & 0x1f);
11790 %}
11791
11792 ins_pipe(ialu_reg_reg_shift);
11793 %}
11794
11795 // This pattern is automatically generated from aarch64_ad.m4
11796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11797 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11798 iRegL src1, iRegL src2,
11799 immI src3) %{
11800 match(Set dst (OrL src1 (URShiftL src2 src3)));
11801
11802 ins_cost(1.9 * INSN_COST);
11803 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11804
11805 ins_encode %{
11806 __ orr(as_Register($dst$$reg),
11807 as_Register($src1$$reg),
11808 as_Register($src2$$reg),
11809 Assembler::LSR,
11810 $src3$$constant & 0x3f);
11811 %}
11812
11813 ins_pipe(ialu_reg_reg_shift);
11814 %}
11815
11816 // This pattern is automatically generated from aarch64_ad.m4
11817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11818 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11819 iRegIorL2I src1, iRegIorL2I src2,
11820 immI src3) %{
11821 match(Set dst (OrI src1 (RShiftI src2 src3)));
11822
11823 ins_cost(1.9 * INSN_COST);
11824 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11825
11826 ins_encode %{
11827 __ orrw(as_Register($dst$$reg),
11828 as_Register($src1$$reg),
11829 as_Register($src2$$reg),
11830 Assembler::ASR,
11831 $src3$$constant & 0x1f);
11832 %}
11833
11834 ins_pipe(ialu_reg_reg_shift);
11835 %}
11836
11837 // This pattern is automatically generated from aarch64_ad.m4
11838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11839 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11840 iRegL src1, iRegL src2,
11841 immI src3) %{
11842 match(Set dst (OrL src1 (RShiftL src2 src3)));
11843
11844 ins_cost(1.9 * INSN_COST);
11845 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11846
11847 ins_encode %{
11848 __ orr(as_Register($dst$$reg),
11849 as_Register($src1$$reg),
11850 as_Register($src2$$reg),
11851 Assembler::ASR,
11852 $src3$$constant & 0x3f);
11853 %}
11854
11855 ins_pipe(ialu_reg_reg_shift);
11856 %}
11857
11858 // This pattern is automatically generated from aarch64_ad.m4
11859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11860 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11861 iRegIorL2I src1, iRegIorL2I src2,
11862 immI src3) %{
11863 match(Set dst (OrI src1 (LShiftI src2 src3)));
11864
11865 ins_cost(1.9 * INSN_COST);
11866 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11867
11868 ins_encode %{
11869 __ orrw(as_Register($dst$$reg),
11870 as_Register($src1$$reg),
11871 as_Register($src2$$reg),
11872 Assembler::LSL,
11873 $src3$$constant & 0x1f);
11874 %}
11875
11876 ins_pipe(ialu_reg_reg_shift);
11877 %}
11878
11879 // This pattern is automatically generated from aarch64_ad.m4
11880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11881 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11882 iRegL src1, iRegL src2,
11883 immI src3) %{
11884 match(Set dst (OrL src1 (LShiftL src2 src3)));
11885
11886 ins_cost(1.9 * INSN_COST);
11887 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11888
11889 ins_encode %{
11890 __ orr(as_Register($dst$$reg),
11891 as_Register($src1$$reg),
11892 as_Register($src2$$reg),
11893 Assembler::LSL,
11894 $src3$$constant & 0x3f);
11895 %}
11896
11897 ins_pipe(ialu_reg_reg_shift);
11898 %}
11899
11900 // This pattern is automatically generated from aarch64_ad.m4
11901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11902 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11903 iRegIorL2I src1, iRegIorL2I src2,
11904 immI src3) %{
11905 match(Set dst (OrI src1 (RotateRight src2 src3)));
11906
11907 ins_cost(1.9 * INSN_COST);
11908 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11909
11910 ins_encode %{
11911 __ orrw(as_Register($dst$$reg),
11912 as_Register($src1$$reg),
11913 as_Register($src2$$reg),
11914 Assembler::ROR,
11915 $src3$$constant & 0x1f);
11916 %}
11917
11918 ins_pipe(ialu_reg_reg_shift);
11919 %}
11920
11921 // This pattern is automatically generated from aarch64_ad.m4
11922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11923 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11924 iRegL src1, iRegL src2,
11925 immI src3) %{
11926 match(Set dst (OrL src1 (RotateRight src2 src3)));
11927
11928 ins_cost(1.9 * INSN_COST);
11929 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11930
11931 ins_encode %{
11932 __ orr(as_Register($dst$$reg),
11933 as_Register($src1$$reg),
11934 as_Register($src2$$reg),
11935 Assembler::ROR,
11936 $src3$$constant & 0x3f);
11937 %}
11938
11939 ins_pipe(ialu_reg_reg_shift);
11940 %}
11941
11942 // This pattern is automatically generated from aarch64_ad.m4
11943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11944 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11945 iRegIorL2I src1, iRegIorL2I src2,
11946 immI src3) %{
11947 match(Set dst (AddI src1 (URShiftI src2 src3)));
11948
11949 ins_cost(1.9 * INSN_COST);
11950 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11951
11952 ins_encode %{
11953 __ addw(as_Register($dst$$reg),
11954 as_Register($src1$$reg),
11955 as_Register($src2$$reg),
11956 Assembler::LSR,
11957 $src3$$constant & 0x1f);
11958 %}
11959
11960 ins_pipe(ialu_reg_reg_shift);
11961 %}
11962
11963 // This pattern is automatically generated from aarch64_ad.m4
11964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11965 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11966 iRegL src1, iRegL src2,
11967 immI src3) %{
11968 match(Set dst (AddL src1 (URShiftL src2 src3)));
11969
11970 ins_cost(1.9 * INSN_COST);
11971 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11972
11973 ins_encode %{
11974 __ add(as_Register($dst$$reg),
11975 as_Register($src1$$reg),
11976 as_Register($src2$$reg),
11977 Assembler::LSR,
11978 $src3$$constant & 0x3f);
11979 %}
11980
11981 ins_pipe(ialu_reg_reg_shift);
11982 %}
11983
11984 // This pattern is automatically generated from aarch64_ad.m4
11985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11986 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11987 iRegIorL2I src1, iRegIorL2I src2,
11988 immI src3) %{
11989 match(Set dst (AddI src1 (RShiftI src2 src3)));
11990
11991 ins_cost(1.9 * INSN_COST);
11992 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11993
11994 ins_encode %{
11995 __ addw(as_Register($dst$$reg),
11996 as_Register($src1$$reg),
11997 as_Register($src2$$reg),
11998 Assembler::ASR,
11999 $src3$$constant & 0x1f);
12000 %}
12001
12002 ins_pipe(ialu_reg_reg_shift);
12003 %}
12004
12005 // This pattern is automatically generated from aarch64_ad.m4
12006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12007 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12008 iRegL src1, iRegL src2,
12009 immI src3) %{
12010 match(Set dst (AddL src1 (RShiftL src2 src3)));
12011
12012 ins_cost(1.9 * INSN_COST);
12013 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12014
12015 ins_encode %{
12016 __ add(as_Register($dst$$reg),
12017 as_Register($src1$$reg),
12018 as_Register($src2$$reg),
12019 Assembler::ASR,
12020 $src3$$constant & 0x3f);
12021 %}
12022
12023 ins_pipe(ialu_reg_reg_shift);
12024 %}
12025
12026 // This pattern is automatically generated from aarch64_ad.m4
12027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12028 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12029 iRegIorL2I src1, iRegIorL2I src2,
12030 immI src3) %{
12031 match(Set dst (AddI src1 (LShiftI src2 src3)));
12032
12033 ins_cost(1.9 * INSN_COST);
12034 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12035
12036 ins_encode %{
12037 __ addw(as_Register($dst$$reg),
12038 as_Register($src1$$reg),
12039 as_Register($src2$$reg),
12040 Assembler::LSL,
12041 $src3$$constant & 0x1f);
12042 %}
12043
12044 ins_pipe(ialu_reg_reg_shift);
12045 %}
12046
12047 // This pattern is automatically generated from aarch64_ad.m4
12048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12049 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12050 iRegL src1, iRegL src2,
12051 immI src3) %{
12052 match(Set dst (AddL src1 (LShiftL src2 src3)));
12053
12054 ins_cost(1.9 * INSN_COST);
12055 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12056
12057 ins_encode %{
12058 __ add(as_Register($dst$$reg),
12059 as_Register($src1$$reg),
12060 as_Register($src2$$reg),
12061 Assembler::LSL,
12062 $src3$$constant & 0x3f);
12063 %}
12064
12065 ins_pipe(ialu_reg_reg_shift);
12066 %}
12067
12068 // This pattern is automatically generated from aarch64_ad.m4
12069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12070 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12071 iRegIorL2I src1, iRegIorL2I src2,
12072 immI src3) %{
12073 match(Set dst (SubI src1 (URShiftI src2 src3)));
12074
12075 ins_cost(1.9 * INSN_COST);
12076 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12077
12078 ins_encode %{
12079 __ subw(as_Register($dst$$reg),
12080 as_Register($src1$$reg),
12081 as_Register($src2$$reg),
12082 Assembler::LSR,
12083 $src3$$constant & 0x1f);
12084 %}
12085
12086 ins_pipe(ialu_reg_reg_shift);
12087 %}
12088
12089 // This pattern is automatically generated from aarch64_ad.m4
12090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12091 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12092 iRegL src1, iRegL src2,
12093 immI src3) %{
12094 match(Set dst (SubL src1 (URShiftL src2 src3)));
12095
12096 ins_cost(1.9 * INSN_COST);
12097 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12098
12099 ins_encode %{
12100 __ sub(as_Register($dst$$reg),
12101 as_Register($src1$$reg),
12102 as_Register($src2$$reg),
12103 Assembler::LSR,
12104 $src3$$constant & 0x3f);
12105 %}
12106
12107 ins_pipe(ialu_reg_reg_shift);
12108 %}
12109
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12113 iRegIorL2I src1, iRegIorL2I src2,
12114 immI src3) %{
12115 match(Set dst (SubI src1 (RShiftI src2 src3)));
12116
12117 ins_cost(1.9 * INSN_COST);
12118 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12119
12120 ins_encode %{
12121 __ subw(as_Register($dst$$reg),
12122 as_Register($src1$$reg),
12123 as_Register($src2$$reg),
12124 Assembler::ASR,
12125 $src3$$constant & 0x1f);
12126 %}
12127
12128 ins_pipe(ialu_reg_reg_shift);
12129 %}
12130
12131 // This pattern is automatically generated from aarch64_ad.m4
12132 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12133 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12134 iRegL src1, iRegL src2,
12135 immI src3) %{
12136 match(Set dst (SubL src1 (RShiftL src2 src3)));
12137
12138 ins_cost(1.9 * INSN_COST);
12139 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12140
12141 ins_encode %{
12142 __ sub(as_Register($dst$$reg),
12143 as_Register($src1$$reg),
12144 as_Register($src2$$reg),
12145 Assembler::ASR,
12146 $src3$$constant & 0x3f);
12147 %}
12148
12149 ins_pipe(ialu_reg_reg_shift);
12150 %}
12151
12152 // This pattern is automatically generated from aarch64_ad.m4
12153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12154 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12155 iRegIorL2I src1, iRegIorL2I src2,
12156 immI src3) %{
12157 match(Set dst (SubI src1 (LShiftI src2 src3)));
12158
12159 ins_cost(1.9 * INSN_COST);
12160 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12161
12162 ins_encode %{
12163 __ subw(as_Register($dst$$reg),
12164 as_Register($src1$$reg),
12165 as_Register($src2$$reg),
12166 Assembler::LSL,
12167 $src3$$constant & 0x1f);
12168 %}
12169
12170 ins_pipe(ialu_reg_reg_shift);
12171 %}
12172
12173 // This pattern is automatically generated from aarch64_ad.m4
12174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12175 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12176 iRegL src1, iRegL src2,
12177 immI src3) %{
12178 match(Set dst (SubL src1 (LShiftL src2 src3)));
12179
12180 ins_cost(1.9 * INSN_COST);
12181 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12182
12183 ins_encode %{
12184 __ sub(as_Register($dst$$reg),
12185 as_Register($src1$$reg),
12186 as_Register($src2$$reg),
12187 Assembler::LSL,
12188 $src3$$constant & 0x3f);
12189 %}
12190
12191 ins_pipe(ialu_reg_reg_shift);
12192 %}
12193
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196
12197 // Shift Left followed by Shift Right.
12198 // This idiom is used by the compiler for the i2b bytecode etc.
12199 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12200 %{
12201 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12202 ins_cost(INSN_COST * 2);
12203 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12204 ins_encode %{
12205 int lshift = $lshift_count$$constant & 63;
12206 int rshift = $rshift_count$$constant & 63;
12207 int s = 63 - lshift;
12208 int r = (rshift - lshift) & 63;
12209 __ sbfm(as_Register($dst$$reg),
12210 as_Register($src$$reg),
12211 r, s);
12212 %}
12213
12214 ins_pipe(ialu_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219
12220 // Shift Left followed by Shift Right.
12221 // This idiom is used by the compiler for the i2b bytecode etc.
12222 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12223 %{
12224 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12225 ins_cost(INSN_COST * 2);
12226 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12227 ins_encode %{
12228 int lshift = $lshift_count$$constant & 31;
12229 int rshift = $rshift_count$$constant & 31;
12230 int s = 31 - lshift;
12231 int r = (rshift - lshift) & 31;
12232 __ sbfmw(as_Register($dst$$reg),
12233 as_Register($src$$reg),
12234 r, s);
12235 %}
12236
12237 ins_pipe(ialu_reg_shift);
12238 %}
12239
12240 // This pattern is automatically generated from aarch64_ad.m4
12241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12242
12243 // Shift Left followed by Shift Right.
12244 // This idiom is used by the compiler for the i2b bytecode etc.
12245 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12246 %{
12247 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12248 ins_cost(INSN_COST * 2);
12249 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12250 ins_encode %{
12251 int lshift = $lshift_count$$constant & 63;
12252 int rshift = $rshift_count$$constant & 63;
12253 int s = 63 - lshift;
12254 int r = (rshift - lshift) & 63;
12255 __ ubfm(as_Register($dst$$reg),
12256 as_Register($src$$reg),
12257 r, s);
12258 %}
12259
12260 ins_pipe(ialu_reg_shift);
12261 %}
12262
12263 // This pattern is automatically generated from aarch64_ad.m4
12264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12265
12266 // Shift Left followed by Shift Right.
12267 // This idiom is used by the compiler for the i2b bytecode etc.
12268 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12269 %{
12270 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12271 ins_cost(INSN_COST * 2);
12272 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12273 ins_encode %{
12274 int lshift = $lshift_count$$constant & 31;
12275 int rshift = $rshift_count$$constant & 31;
12276 int s = 31 - lshift;
12277 int r = (rshift - lshift) & 31;
12278 __ ubfmw(as_Register($dst$$reg),
12279 as_Register($src$$reg),
12280 r, s);
12281 %}
12282
12283 ins_pipe(ialu_reg_shift);
12284 %}
12285
12286 // Bitfield extract with shift & mask
12287
12288 // This pattern is automatically generated from aarch64_ad.m4
12289 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12290 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12291 %{
12292 match(Set dst (AndI (URShiftI src rshift) mask));
12293 // Make sure we are not going to exceed what ubfxw can do.
12294 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12295
12296 ins_cost(INSN_COST);
12297 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12298 ins_encode %{
12299 int rshift = $rshift$$constant & 31;
12300 intptr_t mask = $mask$$constant;
12301 int width = exact_log2(mask+1);
12302 __ ubfxw(as_Register($dst$$reg),
12303 as_Register($src$$reg), rshift, width);
12304 %}
12305 ins_pipe(ialu_reg_shift);
12306 %}
12307
12308 // This pattern is automatically generated from aarch64_ad.m4
12309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12310 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12311 %{
12312 match(Set dst (AndL (URShiftL src rshift) mask));
12313 // Make sure we are not going to exceed what ubfx can do.
12314 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12315
12316 ins_cost(INSN_COST);
12317 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12318 ins_encode %{
12319 int rshift = $rshift$$constant & 63;
12320 intptr_t mask = $mask$$constant;
12321 int width = exact_log2_long(mask+1);
12322 __ ubfx(as_Register($dst$$reg),
12323 as_Register($src$$reg), rshift, width);
12324 %}
12325 ins_pipe(ialu_reg_shift);
12326 %}
12327
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331
12332 // We can use ubfx when extending an And with a mask when we know mask
12333 // is positive. We know that because immI_bitmask guarantees it.
12334 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12335 %{
12336 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12337 // Make sure we are not going to exceed what ubfxw can do.
12338 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12339
12340 ins_cost(INSN_COST * 2);
12341 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12342 ins_encode %{
12343 int rshift = $rshift$$constant & 31;
12344 intptr_t mask = $mask$$constant;
12345 int width = exact_log2(mask+1);
12346 __ ubfx(as_Register($dst$$reg),
12347 as_Register($src$$reg), rshift, width);
12348 %}
12349 ins_pipe(ialu_reg_shift);
12350 %}
12351
12352
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355
12356 // We can use ubfiz when masking by a positive number and then left shifting the result.
12357 // We know that the mask is positive because immI_bitmask guarantees it.
12358 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12359 %{
12360 match(Set dst (LShiftI (AndI src mask) lshift));
12361 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12362
12363 ins_cost(INSN_COST);
12364 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12365 ins_encode %{
12366 int lshift = $lshift$$constant & 31;
12367 intptr_t mask = $mask$$constant;
12368 int width = exact_log2(mask+1);
12369 __ ubfizw(as_Register($dst$$reg),
12370 as_Register($src$$reg), lshift, width);
12371 %}
12372 ins_pipe(ialu_reg_shift);
12373 %}
12374
12375 // This pattern is automatically generated from aarch64_ad.m4
12376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12377
12378 // We can use ubfiz when masking by a positive number and then left shifting the result.
12379 // We know that the mask is positive because immL_bitmask guarantees it.
12380 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12381 %{
12382 match(Set dst (LShiftL (AndL src mask) lshift));
12383 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12384
12385 ins_cost(INSN_COST);
12386 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12387 ins_encode %{
12388 int lshift = $lshift$$constant & 63;
12389 intptr_t mask = $mask$$constant;
12390 int width = exact_log2_long(mask+1);
12391 __ ubfiz(as_Register($dst$$reg),
12392 as_Register($src$$reg), lshift, width);
12393 %}
12394 ins_pipe(ialu_reg_shift);
12395 %}
12396
12397 // This pattern is automatically generated from aarch64_ad.m4
12398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12399
12400 // We can use ubfiz when masking by a positive number and then left shifting the result.
12401 // We know that the mask is positive because immI_bitmask guarantees it.
12402 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12403 %{
12404 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12405 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12406
12407 ins_cost(INSN_COST);
12408 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12409 ins_encode %{
12410 int lshift = $lshift$$constant & 31;
12411 intptr_t mask = $mask$$constant;
12412 int width = exact_log2(mask+1);
12413 __ ubfizw(as_Register($dst$$reg),
12414 as_Register($src$$reg), lshift, width);
12415 %}
12416 ins_pipe(ialu_reg_shift);
12417 %}
12418
12419 // This pattern is automatically generated from aarch64_ad.m4
12420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12421
12422 // We can use ubfiz when masking by a positive number and then left shifting the result.
12423 // We know that the mask is positive because immL_bitmask guarantees it.
12424 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12425 %{
12426 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12427 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12428
12429 ins_cost(INSN_COST);
12430 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12431 ins_encode %{
12432 int lshift = $lshift$$constant & 63;
12433 intptr_t mask = $mask$$constant;
12434 int width = exact_log2_long(mask+1);
12435 __ ubfiz(as_Register($dst$$reg),
12436 as_Register($src$$reg), lshift, width);
12437 %}
12438 ins_pipe(ialu_reg_shift);
12439 %}
12440
12441
12442 // This pattern is automatically generated from aarch64_ad.m4
12443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12444
12445 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12446 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12447 %{
12448 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12449 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12450
12451 ins_cost(INSN_COST);
12452 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12453 ins_encode %{
12454 int lshift = $lshift$$constant & 63;
12455 intptr_t mask = $mask$$constant;
12456 int width = exact_log2(mask+1);
12457 __ ubfiz(as_Register($dst$$reg),
12458 as_Register($src$$reg), lshift, width);
12459 %}
12460 ins_pipe(ialu_reg_shift);
12461 %}
12462
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465
12466 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12467 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12468 %{
12469 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12470 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12471
12472 ins_cost(INSN_COST);
12473 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12474 ins_encode %{
12475 int lshift = $lshift$$constant & 31;
12476 intptr_t mask = $mask$$constant;
12477 int width = exact_log2(mask+1);
12478 __ ubfiz(as_Register($dst$$reg),
12479 as_Register($src$$reg), lshift, width);
12480 %}
12481 ins_pipe(ialu_reg_shift);
12482 %}
12483
12484 // This pattern is automatically generated from aarch64_ad.m4
12485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12486
12487 // Can skip int2long conversions after AND with small bitmask
12488 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12489 %{
12490 match(Set dst (ConvI2L (AndI src msk)));
12491 ins_cost(INSN_COST);
12492 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12493 ins_encode %{
12494 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12495 %}
12496 ins_pipe(ialu_reg_shift);
12497 %}
12498
12499
12500 // Rotations
12501
12502 // This pattern is automatically generated from aarch64_ad.m4
12503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12504 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12505 %{
12506 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12507 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12508
12509 ins_cost(INSN_COST);
12510 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12511
12512 ins_encode %{
12513 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12514 $rshift$$constant & 63);
12515 %}
12516 ins_pipe(ialu_reg_reg_extr);
12517 %}
12518
12519
12520 // This pattern is automatically generated from aarch64_ad.m4
12521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12522 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12523 %{
12524 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12525 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12526
12527 ins_cost(INSN_COST);
12528 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12529
12530 ins_encode %{
12531 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12532 $rshift$$constant & 31);
12533 %}
12534 ins_pipe(ialu_reg_reg_extr);
12535 %}
12536
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12541 %{
12542 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12543 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12544
12545 ins_cost(INSN_COST);
12546 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12547
12548 ins_encode %{
12549 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12550 $rshift$$constant & 63);
12551 %}
12552 ins_pipe(ialu_reg_reg_extr);
12553 %}
12554
12555
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12559 %{
12560 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12561 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12562
12563 ins_cost(INSN_COST);
12564 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12565
12566 ins_encode %{
12567 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12568 $rshift$$constant & 31);
12569 %}
12570 ins_pipe(ialu_reg_reg_extr);
12571 %}
12572
12573 // This pattern is automatically generated from aarch64_ad.m4
12574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12575 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12576 %{
12577 match(Set dst (RotateRight src shift));
12578
12579 ins_cost(INSN_COST);
12580 format %{ "ror $dst, $src, $shift" %}
12581
12582 ins_encode %{
12583 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12584 $shift$$constant & 0x1f);
12585 %}
12586 ins_pipe(ialu_reg_reg_vshift);
12587 %}
12588
12589 // This pattern is automatically generated from aarch64_ad.m4
12590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12591 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12592 %{
12593 match(Set dst (RotateRight src shift));
12594
12595 ins_cost(INSN_COST);
12596 format %{ "ror $dst, $src, $shift" %}
12597
12598 ins_encode %{
12599 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12600 $shift$$constant & 0x3f);
12601 %}
12602 ins_pipe(ialu_reg_reg_vshift);
12603 %}
12604
12605 // This pattern is automatically generated from aarch64_ad.m4
12606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12607 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12608 %{
12609 match(Set dst (RotateRight src shift));
12610
12611 ins_cost(INSN_COST);
12612 format %{ "ror $dst, $src, $shift" %}
12613
12614 ins_encode %{
12615 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12616 %}
12617 ins_pipe(ialu_reg_reg_vshift);
12618 %}
12619
12620 // This pattern is automatically generated from aarch64_ad.m4
12621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12622 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12623 %{
12624 match(Set dst (RotateRight src shift));
12625
12626 ins_cost(INSN_COST);
12627 format %{ "ror $dst, $src, $shift" %}
12628
12629 ins_encode %{
12630 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12631 %}
12632 ins_pipe(ialu_reg_reg_vshift);
12633 %}
12634
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12638 %{
12639 match(Set dst (RotateLeft src shift));
12640
12641 ins_cost(INSN_COST);
12642 format %{ "rol $dst, $src, $shift" %}
12643
12644 ins_encode %{
12645 __ subw(rscratch1, zr, as_Register($shift$$reg));
12646 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12647 %}
12648 ins_pipe(ialu_reg_reg_vshift);
12649 %}
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12654 %{
12655 match(Set dst (RotateLeft src shift));
12656
12657 ins_cost(INSN_COST);
12658 format %{ "rol $dst, $src, $shift" %}
12659
12660 ins_encode %{
12661 __ subw(rscratch1, zr, as_Register($shift$$reg));
12662 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12663 %}
12664 ins_pipe(ialu_reg_reg_vshift);
12665 %}
12666
12667
12668 // Add/subtract (extended)
12669
12670 // This pattern is automatically generated from aarch64_ad.m4
12671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12672 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12673 %{
12674 match(Set dst (AddL src1 (ConvI2L src2)));
12675 ins_cost(INSN_COST);
12676 format %{ "add $dst, $src1, $src2, sxtw" %}
12677
12678 ins_encode %{
12679 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12680 as_Register($src2$$reg), ext::sxtw);
12681 %}
12682 ins_pipe(ialu_reg_reg);
12683 %}
12684
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12688 %{
12689 match(Set dst (SubL src1 (ConvI2L src2)));
12690 ins_cost(INSN_COST);
12691 format %{ "sub $dst, $src1, $src2, sxtw" %}
12692
12693 ins_encode %{
12694 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12695 as_Register($src2$$reg), ext::sxtw);
12696 %}
12697 ins_pipe(ialu_reg_reg);
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12703 %{
12704 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12705 ins_cost(INSN_COST);
12706 format %{ "add $dst, $src1, $src2, sxth" %}
12707
12708 ins_encode %{
12709 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12710 as_Register($src2$$reg), ext::sxth);
12711 %}
12712 ins_pipe(ialu_reg_reg);
12713 %}
12714
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12718 %{
12719 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12720 ins_cost(INSN_COST);
12721 format %{ "add $dst, $src1, $src2, sxtb" %}
12722
12723 ins_encode %{
12724 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12725 as_Register($src2$$reg), ext::sxtb);
12726 %}
12727 ins_pipe(ialu_reg_reg);
12728 %}
12729
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12733 %{
12734 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12735 ins_cost(INSN_COST);
12736 format %{ "add $dst, $src1, $src2, uxtb" %}
12737
12738 ins_encode %{
12739 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12740 as_Register($src2$$reg), ext::uxtb);
12741 %}
12742 ins_pipe(ialu_reg_reg);
12743 %}
12744
12745 // This pattern is automatically generated from aarch64_ad.m4
12746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12747 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12748 %{
12749 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12750 ins_cost(INSN_COST);
12751 format %{ "add $dst, $src1, $src2, sxth" %}
12752
12753 ins_encode %{
12754 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12755 as_Register($src2$$reg), ext::sxth);
12756 %}
12757 ins_pipe(ialu_reg_reg);
12758 %}
12759
12760 // This pattern is automatically generated from aarch64_ad.m4
12761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12762 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12763 %{
12764 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12765 ins_cost(INSN_COST);
12766 format %{ "add $dst, $src1, $src2, sxtw" %}
12767
12768 ins_encode %{
12769 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12770 as_Register($src2$$reg), ext::sxtw);
12771 %}
12772 ins_pipe(ialu_reg_reg);
12773 %}
12774
12775 // This pattern is automatically generated from aarch64_ad.m4
12776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12777 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12778 %{
12779 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12780 ins_cost(INSN_COST);
12781 format %{ "add $dst, $src1, $src2, sxtb" %}
12782
12783 ins_encode %{
12784 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12785 as_Register($src2$$reg), ext::sxtb);
12786 %}
12787 ins_pipe(ialu_reg_reg);
12788 %}
12789
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12793 %{
12794 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12795 ins_cost(INSN_COST);
12796 format %{ "add $dst, $src1, $src2, uxtb" %}
12797
12798 ins_encode %{
12799 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12800 as_Register($src2$$reg), ext::uxtb);
12801 %}
12802 ins_pipe(ialu_reg_reg);
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12808 %{
12809 match(Set dst (AddI src1 (AndI src2 mask)));
12810 ins_cost(INSN_COST);
12811 format %{ "addw $dst, $src1, $src2, uxtb" %}
12812
12813 ins_encode %{
12814 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12815 as_Register($src2$$reg), ext::uxtb);
12816 %}
12817 ins_pipe(ialu_reg_reg);
12818 %}
12819
12820 // This pattern is automatically generated from aarch64_ad.m4
12821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12822 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12823 %{
12824 match(Set dst (AddI src1 (AndI src2 mask)));
12825 ins_cost(INSN_COST);
12826 format %{ "addw $dst, $src1, $src2, uxth" %}
12827
12828 ins_encode %{
12829 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12830 as_Register($src2$$reg), ext::uxth);
12831 %}
12832 ins_pipe(ialu_reg_reg);
12833 %}
12834
12835 // This pattern is automatically generated from aarch64_ad.m4
12836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12837 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12838 %{
12839 match(Set dst (AddL src1 (AndL src2 mask)));
12840 ins_cost(INSN_COST);
12841 format %{ "add $dst, $src1, $src2, uxtb" %}
12842
12843 ins_encode %{
12844 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12845 as_Register($src2$$reg), ext::uxtb);
12846 %}
12847 ins_pipe(ialu_reg_reg);
12848 %}
12849
12850 // This pattern is automatically generated from aarch64_ad.m4
12851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12852 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12853 %{
12854 match(Set dst (AddL src1 (AndL src2 mask)));
12855 ins_cost(INSN_COST);
12856 format %{ "add $dst, $src1, $src2, uxth" %}
12857
12858 ins_encode %{
12859 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12860 as_Register($src2$$reg), ext::uxth);
12861 %}
12862 ins_pipe(ialu_reg_reg);
12863 %}
12864
12865 // This pattern is automatically generated from aarch64_ad.m4
12866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12867 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12868 %{
12869 match(Set dst (AddL src1 (AndL src2 mask)));
12870 ins_cost(INSN_COST);
12871 format %{ "add $dst, $src1, $src2, uxtw" %}
12872
12873 ins_encode %{
12874 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12875 as_Register($src2$$reg), ext::uxtw);
12876 %}
12877 ins_pipe(ialu_reg_reg);
12878 %}
12879
12880 // This pattern is automatically generated from aarch64_ad.m4
12881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12882 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12883 %{
12884 match(Set dst (SubI src1 (AndI src2 mask)));
12885 ins_cost(INSN_COST);
12886 format %{ "subw $dst, $src1, $src2, uxtb" %}
12887
12888 ins_encode %{
12889 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12890 as_Register($src2$$reg), ext::uxtb);
12891 %}
12892 ins_pipe(ialu_reg_reg);
12893 %}
12894
12895 // This pattern is automatically generated from aarch64_ad.m4
12896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12897 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12898 %{
12899 match(Set dst (SubI src1 (AndI src2 mask)));
12900 ins_cost(INSN_COST);
12901 format %{ "subw $dst, $src1, $src2, uxth" %}
12902
12903 ins_encode %{
12904 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12905 as_Register($src2$$reg), ext::uxth);
12906 %}
12907 ins_pipe(ialu_reg_reg);
12908 %}
12909
12910 // This pattern is automatically generated from aarch64_ad.m4
12911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12912 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12913 %{
12914 match(Set dst (SubL src1 (AndL src2 mask)));
12915 ins_cost(INSN_COST);
12916 format %{ "sub $dst, $src1, $src2, uxtb" %}
12917
12918 ins_encode %{
12919 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12920 as_Register($src2$$reg), ext::uxtb);
12921 %}
12922 ins_pipe(ialu_reg_reg);
12923 %}
12924
12925 // This pattern is automatically generated from aarch64_ad.m4
12926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12927 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12928 %{
12929 match(Set dst (SubL src1 (AndL src2 mask)));
12930 ins_cost(INSN_COST);
12931 format %{ "sub $dst, $src1, $src2, uxth" %}
12932
12933 ins_encode %{
12934 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12935 as_Register($src2$$reg), ext::uxth);
12936 %}
12937 ins_pipe(ialu_reg_reg);
12938 %}
12939
12940 // This pattern is automatically generated from aarch64_ad.m4
12941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12942 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12943 %{
12944 match(Set dst (SubL src1 (AndL src2 mask)));
12945 ins_cost(INSN_COST);
12946 format %{ "sub $dst, $src1, $src2, uxtw" %}
12947
12948 ins_encode %{
12949 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12950 as_Register($src2$$reg), ext::uxtw);
12951 %}
12952 ins_pipe(ialu_reg_reg);
12953 %}
12954
12955
12956 // This pattern is automatically generated from aarch64_ad.m4
12957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12958 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12959 %{
12960 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12961 ins_cost(1.9 * INSN_COST);
12962 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12963
12964 ins_encode %{
12965 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12966 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12967 %}
12968 ins_pipe(ialu_reg_reg_shift);
12969 %}
12970
12971 // This pattern is automatically generated from aarch64_ad.m4
12972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12973 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12974 %{
12975 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12976 ins_cost(1.9 * INSN_COST);
12977 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12978
12979 ins_encode %{
12980 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12981 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12982 %}
12983 ins_pipe(ialu_reg_reg_shift);
12984 %}
12985
12986 // This pattern is automatically generated from aarch64_ad.m4
12987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12988 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12989 %{
12990 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12991 ins_cost(1.9 * INSN_COST);
12992 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12993
12994 ins_encode %{
12995 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12996 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12997 %}
12998 ins_pipe(ialu_reg_reg_shift);
12999 %}
13000
13001 // This pattern is automatically generated from aarch64_ad.m4
13002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13003 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13004 %{
13005 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13006 ins_cost(1.9 * INSN_COST);
13007 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13008
13009 ins_encode %{
13010 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13011 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13012 %}
13013 ins_pipe(ialu_reg_reg_shift);
13014 %}
13015
13016 // This pattern is automatically generated from aarch64_ad.m4
13017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13018 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13019 %{
13020 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13021 ins_cost(1.9 * INSN_COST);
13022 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13023
13024 ins_encode %{
13025 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13026 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13027 %}
13028 ins_pipe(ialu_reg_reg_shift);
13029 %}
13030
13031 // This pattern is automatically generated from aarch64_ad.m4
13032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13033 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13034 %{
13035 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13036 ins_cost(1.9 * INSN_COST);
13037 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13038
13039 ins_encode %{
13040 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13041 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13042 %}
13043 ins_pipe(ialu_reg_reg_shift);
13044 %}
13045
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13049 %{
13050 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13051 ins_cost(1.9 * INSN_COST);
13052 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13053
13054 ins_encode %{
13055 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13056 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13057 %}
13058 ins_pipe(ialu_reg_reg_shift);
13059 %}
13060
13061 // This pattern is automatically generated from aarch64_ad.m4
13062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13063 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13064 %{
13065 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13066 ins_cost(1.9 * INSN_COST);
13067 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13068
13069 ins_encode %{
13070 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13071 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13072 %}
13073 ins_pipe(ialu_reg_reg_shift);
13074 %}
13075
13076 // This pattern is automatically generated from aarch64_ad.m4
13077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13078 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13079 %{
13080 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13081 ins_cost(1.9 * INSN_COST);
13082 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13083
13084 ins_encode %{
13085 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13086 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13087 %}
13088 ins_pipe(ialu_reg_reg_shift);
13089 %}
13090
13091 // This pattern is automatically generated from aarch64_ad.m4
13092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13093 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13094 %{
13095 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13096 ins_cost(1.9 * INSN_COST);
13097 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13098
13099 ins_encode %{
13100 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13101 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13102 %}
13103 ins_pipe(ialu_reg_reg_shift);
13104 %}
13105
13106 // This pattern is automatically generated from aarch64_ad.m4
13107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13108 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13109 %{
13110 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13111 ins_cost(1.9 * INSN_COST);
13112 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13113
13114 ins_encode %{
13115 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13116 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13117 %}
13118 ins_pipe(ialu_reg_reg_shift);
13119 %}
13120
13121 // This pattern is automatically generated from aarch64_ad.m4
13122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13123 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13124 %{
13125 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13126 ins_cost(1.9 * INSN_COST);
13127 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13128
13129 ins_encode %{
13130 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13131 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13132 %}
13133 ins_pipe(ialu_reg_reg_shift);
13134 %}
13135
13136 // This pattern is automatically generated from aarch64_ad.m4
13137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13138 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13139 %{
13140 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13141 ins_cost(1.9 * INSN_COST);
13142 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13143
13144 ins_encode %{
13145 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13146 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13147 %}
13148 ins_pipe(ialu_reg_reg_shift);
13149 %}
13150
13151 // This pattern is automatically generated from aarch64_ad.m4
13152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13153 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13154 %{
13155 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13156 ins_cost(1.9 * INSN_COST);
13157 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13158
13159 ins_encode %{
13160 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13161 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13162 %}
13163 ins_pipe(ialu_reg_reg_shift);
13164 %}
13165
13166 // This pattern is automatically generated from aarch64_ad.m4
13167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13168 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13169 %{
13170 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13171 ins_cost(1.9 * INSN_COST);
13172 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13173
13174 ins_encode %{
13175 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13176 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13177 %}
13178 ins_pipe(ialu_reg_reg_shift);
13179 %}
13180
13181 // This pattern is automatically generated from aarch64_ad.m4
13182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13183 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13184 %{
13185 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13186 ins_cost(1.9 * INSN_COST);
13187 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13188
13189 ins_encode %{
13190 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13191 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13192 %}
13193 ins_pipe(ialu_reg_reg_shift);
13194 %}
13195
13196 // This pattern is automatically generated from aarch64_ad.m4
13197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13198 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13199 %{
13200 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13201 ins_cost(1.9 * INSN_COST);
13202 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13203
13204 ins_encode %{
13205 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13206 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13207 %}
13208 ins_pipe(ialu_reg_reg_shift);
13209 %}
13210
13211 // This pattern is automatically generated from aarch64_ad.m4
13212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13213 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13214 %{
13215 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13216 ins_cost(1.9 * INSN_COST);
13217 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13218
13219 ins_encode %{
13220 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13221 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13222 %}
13223 ins_pipe(ialu_reg_reg_shift);
13224 %}
13225
13226 // This pattern is automatically generated from aarch64_ad.m4
13227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13228 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13229 %{
13230 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13231 ins_cost(1.9 * INSN_COST);
13232 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13233
13234 ins_encode %{
13235 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13236 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13237 %}
13238 ins_pipe(ialu_reg_reg_shift);
13239 %}
13240
13241 // This pattern is automatically generated from aarch64_ad.m4
13242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13243 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13244 %{
13245 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13246 ins_cost(1.9 * INSN_COST);
13247 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13248
13249 ins_encode %{
13250 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13251 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13252 %}
13253 ins_pipe(ialu_reg_reg_shift);
13254 %}
13255
13256 // This pattern is automatically generated from aarch64_ad.m4
13257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13258 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13259 %{
13260 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13261 ins_cost(1.9 * INSN_COST);
13262 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13263
13264 ins_encode %{
13265 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13266 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13267 %}
13268 ins_pipe(ialu_reg_reg_shift);
13269 %}
13270
13271 // This pattern is automatically generated from aarch64_ad.m4
13272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13273 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13274 %{
13275 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13276 ins_cost(1.9 * INSN_COST);
13277 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13278
13279 ins_encode %{
13280 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13281 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13282 %}
13283 ins_pipe(ialu_reg_reg_shift);
13284 %}
13285
13286 // This pattern is automatically generated from aarch64_ad.m4
13287 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13288 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13289 %{
13290 effect(DEF dst, USE src1, USE src2, USE cr);
13291 ins_cost(INSN_COST * 2);
13292 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13293
13294 ins_encode %{
13295 __ cselw($dst$$Register,
13296 $src1$$Register,
13297 $src2$$Register,
13298 Assembler::LT);
13299 %}
13300 ins_pipe(icond_reg_reg);
13301 %}
13302
13303 // This pattern is automatically generated from aarch64_ad.m4
13304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13305 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13306 %{
13307 effect(DEF dst, USE src1, USE src2, USE cr);
13308 ins_cost(INSN_COST * 2);
13309 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13310
13311 ins_encode %{
13312 __ cselw($dst$$Register,
13313 $src1$$Register,
13314 $src2$$Register,
13315 Assembler::GT);
13316 %}
13317 ins_pipe(icond_reg_reg);
13318 %}
13319
13320 // This pattern is automatically generated from aarch64_ad.m4
13321 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13322 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13323 %{
13324 effect(DEF dst, USE src1, USE cr);
13325 ins_cost(INSN_COST * 2);
13326 format %{ "cselw $dst, $src1, zr lt\t" %}
13327
13328 ins_encode %{
13329 __ cselw($dst$$Register,
13330 $src1$$Register,
13331 zr,
13332 Assembler::LT);
13333 %}
13334 ins_pipe(icond_reg);
13335 %}
13336
13337 // This pattern is automatically generated from aarch64_ad.m4
13338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13339 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13340 %{
13341 effect(DEF dst, USE src1, USE cr);
13342 ins_cost(INSN_COST * 2);
13343 format %{ "cselw $dst, $src1, zr gt\t" %}
13344
13345 ins_encode %{
13346 __ cselw($dst$$Register,
13347 $src1$$Register,
13348 zr,
13349 Assembler::GT);
13350 %}
13351 ins_pipe(icond_reg);
13352 %}
13353
13354 // This pattern is automatically generated from aarch64_ad.m4
13355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13356 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13357 %{
13358 effect(DEF dst, USE src1, USE cr);
13359 ins_cost(INSN_COST * 2);
13360 format %{ "csincw $dst, $src1, zr le\t" %}
13361
13362 ins_encode %{
13363 __ csincw($dst$$Register,
13364 $src1$$Register,
13365 zr,
13366 Assembler::LE);
13367 %}
13368 ins_pipe(icond_reg);
13369 %}
13370
13371 // This pattern is automatically generated from aarch64_ad.m4
13372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13373 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13374 %{
13375 effect(DEF dst, USE src1, USE cr);
13376 ins_cost(INSN_COST * 2);
13377 format %{ "csincw $dst, $src1, zr gt\t" %}
13378
13379 ins_encode %{
13380 __ csincw($dst$$Register,
13381 $src1$$Register,
13382 zr,
13383 Assembler::GT);
13384 %}
13385 ins_pipe(icond_reg);
13386 %}
13387
13388 // This pattern is automatically generated from aarch64_ad.m4
13389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13390 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13391 %{
13392 effect(DEF dst, USE src1, USE cr);
13393 ins_cost(INSN_COST * 2);
13394 format %{ "csinvw $dst, $src1, zr lt\t" %}
13395
13396 ins_encode %{
13397 __ csinvw($dst$$Register,
13398 $src1$$Register,
13399 zr,
13400 Assembler::LT);
13401 %}
13402 ins_pipe(icond_reg);
13403 %}
13404
13405 // This pattern is automatically generated from aarch64_ad.m4
13406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13407 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13408 %{
13409 effect(DEF dst, USE src1, USE cr);
13410 ins_cost(INSN_COST * 2);
13411 format %{ "csinvw $dst, $src1, zr ge\t" %}
13412
13413 ins_encode %{
13414 __ csinvw($dst$$Register,
13415 $src1$$Register,
13416 zr,
13417 Assembler::GE);
13418 %}
13419 ins_pipe(icond_reg);
13420 %}
13421
13422 // This pattern is automatically generated from aarch64_ad.m4
13423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13424 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13425 %{
13426 match(Set dst (MinI src imm));
13427 ins_cost(INSN_COST * 3);
13428 expand %{
13429 rFlagsReg cr;
13430 compI_reg_imm0(cr, src);
13431 cmovI_reg_imm0_lt(dst, src, cr);
13432 %}
13433 %}
13434
13435 // This pattern is automatically generated from aarch64_ad.m4
13436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13437 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13438 %{
13439 match(Set dst (MinI imm src));
13440 ins_cost(INSN_COST * 3);
13441 expand %{
13442 rFlagsReg cr;
13443 compI_reg_imm0(cr, src);
13444 cmovI_reg_imm0_lt(dst, src, cr);
13445 %}
13446 %}
13447
13448 // This pattern is automatically generated from aarch64_ad.m4
13449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13450 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13451 %{
13452 match(Set dst (MinI src imm));
13453 ins_cost(INSN_COST * 3);
13454 expand %{
13455 rFlagsReg cr;
13456 compI_reg_imm0(cr, src);
13457 cmovI_reg_imm1_le(dst, src, cr);
13458 %}
13459 %}
13460
13461 // This pattern is automatically generated from aarch64_ad.m4
13462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13463 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13464 %{
13465 match(Set dst (MinI imm src));
13466 ins_cost(INSN_COST * 3);
13467 expand %{
13468 rFlagsReg cr;
13469 compI_reg_imm0(cr, src);
13470 cmovI_reg_imm1_le(dst, src, cr);
13471 %}
13472 %}
13473
13474 // This pattern is automatically generated from aarch64_ad.m4
13475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13476 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13477 %{
13478 match(Set dst (MinI src imm));
13479 ins_cost(INSN_COST * 3);
13480 expand %{
13481 rFlagsReg cr;
13482 compI_reg_imm0(cr, src);
13483 cmovI_reg_immM1_lt(dst, src, cr);
13484 %}
13485 %}
13486
13487 // This pattern is automatically generated from aarch64_ad.m4
13488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13489 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13490 %{
13491 match(Set dst (MinI imm src));
13492 ins_cost(INSN_COST * 3);
13493 expand %{
13494 rFlagsReg cr;
13495 compI_reg_imm0(cr, src);
13496 cmovI_reg_immM1_lt(dst, src, cr);
13497 %}
13498 %}
13499
13500 // This pattern is automatically generated from aarch64_ad.m4
13501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13502 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13503 %{
13504 match(Set dst (MaxI src imm));
13505 ins_cost(INSN_COST * 3);
13506 expand %{
13507 rFlagsReg cr;
13508 compI_reg_imm0(cr, src);
13509 cmovI_reg_imm0_gt(dst, src, cr);
13510 %}
13511 %}
13512
13513 // This pattern is automatically generated from aarch64_ad.m4
13514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13515 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13516 %{
13517 match(Set dst (MaxI imm src));
13518 ins_cost(INSN_COST * 3);
13519 expand %{
13520 rFlagsReg cr;
13521 compI_reg_imm0(cr, src);
13522 cmovI_reg_imm0_gt(dst, src, cr);
13523 %}
13524 %}
13525
13526 // This pattern is automatically generated from aarch64_ad.m4
13527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13528 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13529 %{
13530 match(Set dst (MaxI src imm));
13531 ins_cost(INSN_COST * 3);
13532 expand %{
13533 rFlagsReg cr;
13534 compI_reg_imm0(cr, src);
13535 cmovI_reg_imm1_gt(dst, src, cr);
13536 %}
13537 %}
13538
13539 // This pattern is automatically generated from aarch64_ad.m4
13540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13541 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13542 %{
13543 match(Set dst (MaxI imm src));
13544 ins_cost(INSN_COST * 3);
13545 expand %{
13546 rFlagsReg cr;
13547 compI_reg_imm0(cr, src);
13548 cmovI_reg_imm1_gt(dst, src, cr);
13549 %}
13550 %}
13551
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13555 %{
13556 match(Set dst (MaxI src imm));
13557 ins_cost(INSN_COST * 3);
13558 expand %{
13559 rFlagsReg cr;
13560 compI_reg_imm0(cr, src);
13561 cmovI_reg_immM1_ge(dst, src, cr);
13562 %}
13563 %}
13564
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13568 %{
13569 match(Set dst (MaxI imm src));
13570 ins_cost(INSN_COST * 3);
13571 expand %{
13572 rFlagsReg cr;
13573 compI_reg_imm0(cr, src);
13574 cmovI_reg_immM1_ge(dst, src, cr);
13575 %}
13576 %}
13577
13578 // This pattern is automatically generated from aarch64_ad.m4
13579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13580 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13581 %{
13582 match(Set dst (ReverseI src));
13583 ins_cost(INSN_COST);
13584 format %{ "rbitw $dst, $src" %}
13585 ins_encode %{
13586 __ rbitw($dst$$Register, $src$$Register);
13587 %}
13588 ins_pipe(ialu_reg);
13589 %}
13590
13591 // This pattern is automatically generated from aarch64_ad.m4
13592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13593 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13594 %{
13595 match(Set dst (ReverseL src));
13596 ins_cost(INSN_COST);
13597 format %{ "rbit $dst, $src" %}
13598 ins_encode %{
13599 __ rbit($dst$$Register, $src$$Register);
13600 %}
13601 ins_pipe(ialu_reg);
13602 %}
13603
13604
13605 // END This section of the file is automatically generated. Do not edit --------------
13606
13607
13608 // ============================================================================
13609 // Floating Point Arithmetic Instructions
13610
13611 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13612 match(Set dst (AddF src1 src2));
13613
13614 ins_cost(INSN_COST * 5);
13615 format %{ "fadds $dst, $src1, $src2" %}
13616
13617 ins_encode %{
13618 __ fadds(as_FloatRegister($dst$$reg),
13619 as_FloatRegister($src1$$reg),
13620 as_FloatRegister($src2$$reg));
13621 %}
13622
13623 ins_pipe(fp_dop_reg_reg_s);
13624 %}
13625
13626 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13627 match(Set dst (AddD src1 src2));
13628
13629 ins_cost(INSN_COST * 5);
13630 format %{ "faddd $dst, $src1, $src2" %}
13631
13632 ins_encode %{
13633 __ faddd(as_FloatRegister($dst$$reg),
13634 as_FloatRegister($src1$$reg),
13635 as_FloatRegister($src2$$reg));
13636 %}
13637
13638 ins_pipe(fp_dop_reg_reg_d);
13639 %}
13640
13641 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13642 match(Set dst (SubF src1 src2));
13643
13644 ins_cost(INSN_COST * 5);
13645 format %{ "fsubs $dst, $src1, $src2" %}
13646
13647 ins_encode %{
13648 __ fsubs(as_FloatRegister($dst$$reg),
13649 as_FloatRegister($src1$$reg),
13650 as_FloatRegister($src2$$reg));
13651 %}
13652
13653 ins_pipe(fp_dop_reg_reg_s);
13654 %}
13655
13656 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13657 match(Set dst (SubD src1 src2));
13658
13659 ins_cost(INSN_COST * 5);
13660 format %{ "fsubd $dst, $src1, $src2" %}
13661
13662 ins_encode %{
13663 __ fsubd(as_FloatRegister($dst$$reg),
13664 as_FloatRegister($src1$$reg),
13665 as_FloatRegister($src2$$reg));
13666 %}
13667
13668 ins_pipe(fp_dop_reg_reg_d);
13669 %}
13670
13671 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13672 match(Set dst (MulF src1 src2));
13673
13674 ins_cost(INSN_COST * 6);
13675 format %{ "fmuls $dst, $src1, $src2" %}
13676
13677 ins_encode %{
13678 __ fmuls(as_FloatRegister($dst$$reg),
13679 as_FloatRegister($src1$$reg),
13680 as_FloatRegister($src2$$reg));
13681 %}
13682
13683 ins_pipe(fp_dop_reg_reg_s);
13684 %}
13685
13686 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13687 match(Set dst (MulD src1 src2));
13688
13689 ins_cost(INSN_COST * 6);
13690 format %{ "fmuld $dst, $src1, $src2" %}
13691
13692 ins_encode %{
13693 __ fmuld(as_FloatRegister($dst$$reg),
13694 as_FloatRegister($src1$$reg),
13695 as_FloatRegister($src2$$reg));
13696 %}
13697
13698 ins_pipe(fp_dop_reg_reg_d);
13699 %}
13700
13701 // src1 * src2 + src3
13702 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13703 match(Set dst (FmaF src3 (Binary src1 src2)));
13704
13705 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13706
13707 ins_encode %{
13708 assert(UseFMA, "Needs FMA instructions support.");
13709 __ fmadds(as_FloatRegister($dst$$reg),
13710 as_FloatRegister($src1$$reg),
13711 as_FloatRegister($src2$$reg),
13712 as_FloatRegister($src3$$reg));
13713 %}
13714
13715 ins_pipe(pipe_class_default);
13716 %}
13717
13718 // src1 * src2 + src3
13719 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13720 match(Set dst (FmaD src3 (Binary src1 src2)));
13721
13722 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13723
13724 ins_encode %{
13725 assert(UseFMA, "Needs FMA instructions support.");
13726 __ fmaddd(as_FloatRegister($dst$$reg),
13727 as_FloatRegister($src1$$reg),
13728 as_FloatRegister($src2$$reg),
13729 as_FloatRegister($src3$$reg));
13730 %}
13731
13732 ins_pipe(pipe_class_default);
13733 %}
13734
13735 // src1 * (-src2) + src3
13736 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13737 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13738 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13739
13740 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13741
13742 ins_encode %{
13743 assert(UseFMA, "Needs FMA instructions support.");
13744 __ fmsubs(as_FloatRegister($dst$$reg),
13745 as_FloatRegister($src1$$reg),
13746 as_FloatRegister($src2$$reg),
13747 as_FloatRegister($src3$$reg));
13748 %}
13749
13750 ins_pipe(pipe_class_default);
13751 %}
13752
13753 // src1 * (-src2) + src3
13754 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13755 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13756 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13757
13758 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13759
13760 ins_encode %{
13761 assert(UseFMA, "Needs FMA instructions support.");
13762 __ fmsubd(as_FloatRegister($dst$$reg),
13763 as_FloatRegister($src1$$reg),
13764 as_FloatRegister($src2$$reg),
13765 as_FloatRegister($src3$$reg));
13766 %}
13767
13768 ins_pipe(pipe_class_default);
13769 %}
13770
13771 // src1 * (-src2) - src3
13772 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13773 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13774 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13775
13776 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13777
13778 ins_encode %{
13779 assert(UseFMA, "Needs FMA instructions support.");
13780 __ fnmadds(as_FloatRegister($dst$$reg),
13781 as_FloatRegister($src1$$reg),
13782 as_FloatRegister($src2$$reg),
13783 as_FloatRegister($src3$$reg));
13784 %}
13785
13786 ins_pipe(pipe_class_default);
13787 %}
13788
13789 // src1 * (-src2) - src3
13790 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13791 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13792 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13793
13794 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13795
13796 ins_encode %{
13797 assert(UseFMA, "Needs FMA instructions support.");
13798 __ fnmaddd(as_FloatRegister($dst$$reg),
13799 as_FloatRegister($src1$$reg),
13800 as_FloatRegister($src2$$reg),
13801 as_FloatRegister($src3$$reg));
13802 %}
13803
13804 ins_pipe(pipe_class_default);
13805 %}
13806
13807 // src1 * src2 - src3
13808 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13809 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13810
13811 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13812
13813 ins_encode %{
13814 assert(UseFMA, "Needs FMA instructions support.");
13815 __ fnmsubs(as_FloatRegister($dst$$reg),
13816 as_FloatRegister($src1$$reg),
13817 as_FloatRegister($src2$$reg),
13818 as_FloatRegister($src3$$reg));
13819 %}
13820
13821 ins_pipe(pipe_class_default);
13822 %}
13823
13824 // src1 * src2 - src3
13825 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13826 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13827
13828 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13829
13830 ins_encode %{
13831 assert(UseFMA, "Needs FMA instructions support.");
13832 // n.b. insn name should be fnmsubd
13833 __ fnmsub(as_FloatRegister($dst$$reg),
13834 as_FloatRegister($src1$$reg),
13835 as_FloatRegister($src2$$reg),
13836 as_FloatRegister($src3$$reg));
13837 %}
13838
13839 ins_pipe(pipe_class_default);
13840 %}
13841
13842
13843 // Math.max(FF)F
13844 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13845 match(Set dst (MaxF src1 src2));
13846
13847 format %{ "fmaxs $dst, $src1, $src2" %}
13848 ins_encode %{
13849 __ fmaxs(as_FloatRegister($dst$$reg),
13850 as_FloatRegister($src1$$reg),
13851 as_FloatRegister($src2$$reg));
13852 %}
13853
13854 ins_pipe(fp_dop_reg_reg_s);
13855 %}
13856
13857 // Math.min(FF)F
13858 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13859 match(Set dst (MinF src1 src2));
13860
13861 format %{ "fmins $dst, $src1, $src2" %}
13862 ins_encode %{
13863 __ fmins(as_FloatRegister($dst$$reg),
13864 as_FloatRegister($src1$$reg),
13865 as_FloatRegister($src2$$reg));
13866 %}
13867
13868 ins_pipe(fp_dop_reg_reg_s);
13869 %}
13870
13871 // Math.max(DD)D
13872 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13873 match(Set dst (MaxD src1 src2));
13874
13875 format %{ "fmaxd $dst, $src1, $src2" %}
13876 ins_encode %{
13877 __ fmaxd(as_FloatRegister($dst$$reg),
13878 as_FloatRegister($src1$$reg),
13879 as_FloatRegister($src2$$reg));
13880 %}
13881
13882 ins_pipe(fp_dop_reg_reg_d);
13883 %}
13884
13885 // Math.min(DD)D
13886 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13887 match(Set dst (MinD src1 src2));
13888
13889 format %{ "fmind $dst, $src1, $src2" %}
13890 ins_encode %{
13891 __ fmind(as_FloatRegister($dst$$reg),
13892 as_FloatRegister($src1$$reg),
13893 as_FloatRegister($src2$$reg));
13894 %}
13895
13896 ins_pipe(fp_dop_reg_reg_d);
13897 %}
13898
13899
13900 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13901 match(Set dst (DivF src1 src2));
13902
13903 ins_cost(INSN_COST * 18);
13904 format %{ "fdivs $dst, $src1, $src2" %}
13905
13906 ins_encode %{
13907 __ fdivs(as_FloatRegister($dst$$reg),
13908 as_FloatRegister($src1$$reg),
13909 as_FloatRegister($src2$$reg));
13910 %}
13911
13912 ins_pipe(fp_div_s);
13913 %}
13914
13915 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13916 match(Set dst (DivD src1 src2));
13917
13918 ins_cost(INSN_COST * 32);
13919 format %{ "fdivd $dst, $src1, $src2" %}
13920
13921 ins_encode %{
13922 __ fdivd(as_FloatRegister($dst$$reg),
13923 as_FloatRegister($src1$$reg),
13924 as_FloatRegister($src2$$reg));
13925 %}
13926
13927 ins_pipe(fp_div_d);
13928 %}
13929
13930 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13931 match(Set dst (NegF src));
13932
13933 ins_cost(INSN_COST * 3);
13934 format %{ "fneg $dst, $src" %}
13935
13936 ins_encode %{
13937 __ fnegs(as_FloatRegister($dst$$reg),
13938 as_FloatRegister($src$$reg));
13939 %}
13940
13941 ins_pipe(fp_uop_s);
13942 %}
13943
13944 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13945 match(Set dst (NegD src));
13946
13947 ins_cost(INSN_COST * 3);
13948 format %{ "fnegd $dst, $src" %}
13949
13950 ins_encode %{
13951 __ fnegd(as_FloatRegister($dst$$reg),
13952 as_FloatRegister($src$$reg));
13953 %}
13954
13955 ins_pipe(fp_uop_d);
13956 %}
13957
13958 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13959 %{
13960 match(Set dst (AbsI src));
13961
13962 effect(KILL cr);
13963 ins_cost(INSN_COST * 2);
13964 format %{ "cmpw $src, zr\n\t"
13965 "cnegw $dst, $src, Assembler::LT\t# int abs"
13966 %}
13967
13968 ins_encode %{
13969 __ cmpw(as_Register($src$$reg), zr);
13970 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13971 %}
13972 ins_pipe(pipe_class_default);
13973 %}
13974
13975 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13976 %{
13977 match(Set dst (AbsL src));
13978
13979 effect(KILL cr);
13980 ins_cost(INSN_COST * 2);
13981 format %{ "cmp $src, zr\n\t"
13982 "cneg $dst, $src, Assembler::LT\t# long abs"
13983 %}
13984
13985 ins_encode %{
13986 __ cmp(as_Register($src$$reg), zr);
13987 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13988 %}
13989 ins_pipe(pipe_class_default);
13990 %}
13991
13992 instruct absF_reg(vRegF dst, vRegF src) %{
13993 match(Set dst (AbsF src));
13994
13995 ins_cost(INSN_COST * 3);
13996 format %{ "fabss $dst, $src" %}
13997 ins_encode %{
13998 __ fabss(as_FloatRegister($dst$$reg),
13999 as_FloatRegister($src$$reg));
14000 %}
14001
14002 ins_pipe(fp_uop_s);
14003 %}
14004
14005 instruct absD_reg(vRegD dst, vRegD src) %{
14006 match(Set dst (AbsD src));
14007
14008 ins_cost(INSN_COST * 3);
14009 format %{ "fabsd $dst, $src" %}
14010 ins_encode %{
14011 __ fabsd(as_FloatRegister($dst$$reg),
14012 as_FloatRegister($src$$reg));
14013 %}
14014
14015 ins_pipe(fp_uop_d);
14016 %}
14017
14018 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14019 match(Set dst (AbsF (SubF src1 src2)));
14020
14021 ins_cost(INSN_COST * 3);
14022 format %{ "fabds $dst, $src1, $src2" %}
14023 ins_encode %{
14024 __ fabds(as_FloatRegister($dst$$reg),
14025 as_FloatRegister($src1$$reg),
14026 as_FloatRegister($src2$$reg));
14027 %}
14028
14029 ins_pipe(fp_uop_s);
14030 %}
14031
14032 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14033 match(Set dst (AbsD (SubD src1 src2)));
14034
14035 ins_cost(INSN_COST * 3);
14036 format %{ "fabdd $dst, $src1, $src2" %}
14037 ins_encode %{
14038 __ fabdd(as_FloatRegister($dst$$reg),
14039 as_FloatRegister($src1$$reg),
14040 as_FloatRegister($src2$$reg));
14041 %}
14042
14043 ins_pipe(fp_uop_d);
14044 %}
14045
14046 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14047 match(Set dst (SqrtD src));
14048
14049 ins_cost(INSN_COST * 50);
14050 format %{ "fsqrtd $dst, $src" %}
14051 ins_encode %{
14052 __ fsqrtd(as_FloatRegister($dst$$reg),
14053 as_FloatRegister($src$$reg));
14054 %}
14055
14056 ins_pipe(fp_div_s);
14057 %}
14058
14059 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14060 match(Set dst (SqrtF src));
14061
14062 ins_cost(INSN_COST * 50);
14063 format %{ "fsqrts $dst, $src" %}
14064 ins_encode %{
14065 __ fsqrts(as_FloatRegister($dst$$reg),
14066 as_FloatRegister($src$$reg));
14067 %}
14068
14069 ins_pipe(fp_div_d);
14070 %}
14071
14072 // Math.rint, floor, ceil
14073 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14074 match(Set dst (RoundDoubleMode src rmode));
14075 format %{ "frint $dst, $src, $rmode" %}
14076 ins_encode %{
14077 switch ($rmode$$constant) {
14078 case RoundDoubleModeNode::rmode_rint:
14079 __ frintnd(as_FloatRegister($dst$$reg),
14080 as_FloatRegister($src$$reg));
14081 break;
14082 case RoundDoubleModeNode::rmode_floor:
14083 __ frintmd(as_FloatRegister($dst$$reg),
14084 as_FloatRegister($src$$reg));
14085 break;
14086 case RoundDoubleModeNode::rmode_ceil:
14087 __ frintpd(as_FloatRegister($dst$$reg),
14088 as_FloatRegister($src$$reg));
14089 break;
14090 }
14091 %}
14092 ins_pipe(fp_uop_d);
14093 %}
14094
14095 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14096 match(Set dst (CopySignD src1 (Binary src2 zero)));
14097 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14098 format %{ "CopySignD $dst $src1 $src2" %}
14099 ins_encode %{
14100 FloatRegister dst = as_FloatRegister($dst$$reg),
14101 src1 = as_FloatRegister($src1$$reg),
14102 src2 = as_FloatRegister($src2$$reg),
14103 zero = as_FloatRegister($zero$$reg);
14104 __ fnegd(dst, zero);
14105 __ bsl(dst, __ T8B, src2, src1);
14106 %}
14107 ins_pipe(fp_uop_d);
14108 %}
14109
14110 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14111 match(Set dst (CopySignF src1 src2));
14112 effect(TEMP_DEF dst, USE src1, USE src2);
14113 format %{ "CopySignF $dst $src1 $src2" %}
14114 ins_encode %{
14115 FloatRegister dst = as_FloatRegister($dst$$reg),
14116 src1 = as_FloatRegister($src1$$reg),
14117 src2 = as_FloatRegister($src2$$reg);
14118 __ movi(dst, __ T2S, 0x80, 24);
14119 __ bsl(dst, __ T8B, src2, src1);
14120 %}
14121 ins_pipe(fp_uop_d);
14122 %}
14123
14124 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14125 match(Set dst (SignumD src (Binary zero one)));
14126 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14127 format %{ "signumD $dst, $src" %}
14128 ins_encode %{
14129 FloatRegister src = as_FloatRegister($src$$reg),
14130 dst = as_FloatRegister($dst$$reg),
14131 zero = as_FloatRegister($zero$$reg),
14132 one = as_FloatRegister($one$$reg);
14133 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14134 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14135 // Bit selection instruction gets bit from "one" for each enabled bit in
14136 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14137 // NaN the whole "src" will be copied because "dst" is zero. For all other
14138 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14139 // from "src", and all other bits are copied from 1.0.
14140 __ bsl(dst, __ T8B, one, src);
14141 %}
14142 ins_pipe(fp_uop_d);
14143 %}
14144
14145 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14146 match(Set dst (SignumF src (Binary zero one)));
14147 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14148 format %{ "signumF $dst, $src" %}
14149 ins_encode %{
14150 FloatRegister src = as_FloatRegister($src$$reg),
14151 dst = as_FloatRegister($dst$$reg),
14152 zero = as_FloatRegister($zero$$reg),
14153 one = as_FloatRegister($one$$reg);
14154 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14155 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14156 // Bit selection instruction gets bit from "one" for each enabled bit in
14157 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14158 // NaN the whole "src" will be copied because "dst" is zero. For all other
14159 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14160 // from "src", and all other bits are copied from 1.0.
14161 __ bsl(dst, __ T8B, one, src);
14162 %}
14163 ins_pipe(fp_uop_d);
14164 %}
14165
14166 instruct onspinwait() %{
14167 match(OnSpinWait);
14168 ins_cost(INSN_COST);
14169
14170 format %{ "onspinwait" %}
14171
14172 ins_encode %{
14173 __ spin_wait();
14174 %}
14175 ins_pipe(pipe_class_empty);
14176 %}
14177
14178 // ============================================================================
14179 // Logical Instructions
14180
14181 // Integer Logical Instructions
14182
14183 // And Instructions
14184
14185
14186 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14187 match(Set dst (AndI src1 src2));
14188
14189 format %{ "andw $dst, $src1, $src2\t# int" %}
14190
14191 ins_cost(INSN_COST);
14192 ins_encode %{
14193 __ andw(as_Register($dst$$reg),
14194 as_Register($src1$$reg),
14195 as_Register($src2$$reg));
14196 %}
14197
14198 ins_pipe(ialu_reg_reg);
14199 %}
14200
14201 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14202 match(Set dst (AndI src1 src2));
14203
14204 format %{ "andsw $dst, $src1, $src2\t# int" %}
14205
14206 ins_cost(INSN_COST);
14207 ins_encode %{
14208 __ andw(as_Register($dst$$reg),
14209 as_Register($src1$$reg),
14210 (uint64_t)($src2$$constant));
14211 %}
14212
14213 ins_pipe(ialu_reg_imm);
14214 %}
14215
14216 // Or Instructions
14217
14218 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14219 match(Set dst (OrI src1 src2));
14220
14221 format %{ "orrw $dst, $src1, $src2\t# int" %}
14222
14223 ins_cost(INSN_COST);
14224 ins_encode %{
14225 __ orrw(as_Register($dst$$reg),
14226 as_Register($src1$$reg),
14227 as_Register($src2$$reg));
14228 %}
14229
14230 ins_pipe(ialu_reg_reg);
14231 %}
14232
14233 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14234 match(Set dst (OrI src1 src2));
14235
14236 format %{ "orrw $dst, $src1, $src2\t# int" %}
14237
14238 ins_cost(INSN_COST);
14239 ins_encode %{
14240 __ orrw(as_Register($dst$$reg),
14241 as_Register($src1$$reg),
14242 (uint64_t)($src2$$constant));
14243 %}
14244
14245 ins_pipe(ialu_reg_imm);
14246 %}
14247
14248 // Xor Instructions
14249
14250 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14251 match(Set dst (XorI src1 src2));
14252
14253 format %{ "eorw $dst, $src1, $src2\t# int" %}
14254
14255 ins_cost(INSN_COST);
14256 ins_encode %{
14257 __ eorw(as_Register($dst$$reg),
14258 as_Register($src1$$reg),
14259 as_Register($src2$$reg));
14260 %}
14261
14262 ins_pipe(ialu_reg_reg);
14263 %}
14264
14265 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14266 match(Set dst (XorI src1 src2));
14267
14268 format %{ "eorw $dst, $src1, $src2\t# int" %}
14269
14270 ins_cost(INSN_COST);
14271 ins_encode %{
14272 __ eorw(as_Register($dst$$reg),
14273 as_Register($src1$$reg),
14274 (uint64_t)($src2$$constant));
14275 %}
14276
14277 ins_pipe(ialu_reg_imm);
14278 %}
14279
14280 // Long Logical Instructions
14281 // TODO
14282
14283 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14284 match(Set dst (AndL src1 src2));
14285
14286 format %{ "and $dst, $src1, $src2\t# int" %}
14287
14288 ins_cost(INSN_COST);
14289 ins_encode %{
14290 __ andr(as_Register($dst$$reg),
14291 as_Register($src1$$reg),
14292 as_Register($src2$$reg));
14293 %}
14294
14295 ins_pipe(ialu_reg_reg);
14296 %}
14297
14298 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14299 match(Set dst (AndL src1 src2));
14300
14301 format %{ "and $dst, $src1, $src2\t# int" %}
14302
14303 ins_cost(INSN_COST);
14304 ins_encode %{
14305 __ andr(as_Register($dst$$reg),
14306 as_Register($src1$$reg),
14307 (uint64_t)($src2$$constant));
14308 %}
14309
14310 ins_pipe(ialu_reg_imm);
14311 %}
14312
14313 // Or Instructions
14314
14315 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14316 match(Set dst (OrL src1 src2));
14317
14318 format %{ "orr $dst, $src1, $src2\t# int" %}
14319
14320 ins_cost(INSN_COST);
14321 ins_encode %{
14322 __ orr(as_Register($dst$$reg),
14323 as_Register($src1$$reg),
14324 as_Register($src2$$reg));
14325 %}
14326
14327 ins_pipe(ialu_reg_reg);
14328 %}
14329
14330 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14331 match(Set dst (OrL src1 src2));
14332
14333 format %{ "orr $dst, $src1, $src2\t# int" %}
14334
14335 ins_cost(INSN_COST);
14336 ins_encode %{
14337 __ orr(as_Register($dst$$reg),
14338 as_Register($src1$$reg),
14339 (uint64_t)($src2$$constant));
14340 %}
14341
14342 ins_pipe(ialu_reg_imm);
14343 %}
14344
14345 // Xor Instructions
14346
14347 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14348 match(Set dst (XorL src1 src2));
14349
14350 format %{ "eor $dst, $src1, $src2\t# int" %}
14351
14352 ins_cost(INSN_COST);
14353 ins_encode %{
14354 __ eor(as_Register($dst$$reg),
14355 as_Register($src1$$reg),
14356 as_Register($src2$$reg));
14357 %}
14358
14359 ins_pipe(ialu_reg_reg);
14360 %}
14361
14362 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14363 match(Set dst (XorL src1 src2));
14364
14365 ins_cost(INSN_COST);
14366 format %{ "eor $dst, $src1, $src2\t# int" %}
14367
14368 ins_encode %{
14369 __ eor(as_Register($dst$$reg),
14370 as_Register($src1$$reg),
14371 (uint64_t)($src2$$constant));
14372 %}
14373
14374 ins_pipe(ialu_reg_imm);
14375 %}
14376
14377 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14378 %{
14379 match(Set dst (ConvI2L src));
14380
14381 ins_cost(INSN_COST);
14382 format %{ "sxtw $dst, $src\t# i2l" %}
14383 ins_encode %{
14384 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14385 %}
14386 ins_pipe(ialu_reg_shift);
14387 %}
14388
14389 // this pattern occurs in bigmath arithmetic
14390 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14391 %{
14392 match(Set dst (AndL (ConvI2L src) mask));
14393
14394 ins_cost(INSN_COST);
14395 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14396 ins_encode %{
14397 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14398 %}
14399
14400 ins_pipe(ialu_reg_shift);
14401 %}
14402
14403 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14404 match(Set dst (ConvL2I src));
14405
14406 ins_cost(INSN_COST);
14407 format %{ "movw $dst, $src \t// l2i" %}
14408
14409 ins_encode %{
14410 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14411 %}
14412
14413 ins_pipe(ialu_reg);
14414 %}
14415
14416 instruct convD2F_reg(vRegF dst, vRegD src) %{
14417 match(Set dst (ConvD2F src));
14418
14419 ins_cost(INSN_COST * 5);
14420 format %{ "fcvtd $dst, $src \t// d2f" %}
14421
14422 ins_encode %{
14423 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14424 %}
14425
14426 ins_pipe(fp_d2f);
14427 %}
14428
14429 instruct convF2D_reg(vRegD dst, vRegF src) %{
14430 match(Set dst (ConvF2D src));
14431
14432 ins_cost(INSN_COST * 5);
14433 format %{ "fcvts $dst, $src \t// f2d" %}
14434
14435 ins_encode %{
14436 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14437 %}
14438
14439 ins_pipe(fp_f2d);
14440 %}
14441
14442 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14443 match(Set dst (ConvF2I src));
14444
14445 ins_cost(INSN_COST * 5);
14446 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14447
14448 ins_encode %{
14449 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14450 %}
14451
14452 ins_pipe(fp_f2i);
14453 %}
14454
14455 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14456 match(Set dst (ConvF2L src));
14457
14458 ins_cost(INSN_COST * 5);
14459 format %{ "fcvtzs $dst, $src \t// f2l" %}
14460
14461 ins_encode %{
14462 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14463 %}
14464
14465 ins_pipe(fp_f2l);
14466 %}
14467
14468 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14469 match(Set dst (ConvF2HF src));
14470 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14471 "smov $dst, $tmp\t# move result from $tmp to $dst"
14472 %}
14473 effect(TEMP tmp);
14474 ins_encode %{
14475 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14476 %}
14477 ins_pipe(pipe_slow);
14478 %}
14479
14480 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14481 match(Set dst (ConvHF2F src));
14482 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14483 "fcvt $dst, $tmp\t# convert half to single precision"
14484 %}
14485 effect(TEMP tmp);
14486 ins_encode %{
14487 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14488 %}
14489 ins_pipe(pipe_slow);
14490 %}
14491
14492 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14493 match(Set dst (ConvI2F src));
14494
14495 ins_cost(INSN_COST * 5);
14496 format %{ "scvtfws $dst, $src \t// i2f" %}
14497
14498 ins_encode %{
14499 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14500 %}
14501
14502 ins_pipe(fp_i2f);
14503 %}
14504
14505 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14506 match(Set dst (ConvL2F src));
14507
14508 ins_cost(INSN_COST * 5);
14509 format %{ "scvtfs $dst, $src \t// l2f" %}
14510
14511 ins_encode %{
14512 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14513 %}
14514
14515 ins_pipe(fp_l2f);
14516 %}
14517
14518 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14519 match(Set dst (ConvD2I src));
14520
14521 ins_cost(INSN_COST * 5);
14522 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14523
14524 ins_encode %{
14525 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14526 %}
14527
14528 ins_pipe(fp_d2i);
14529 %}
14530
14531 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14532 match(Set dst (ConvD2L src));
14533
14534 ins_cost(INSN_COST * 5);
14535 format %{ "fcvtzd $dst, $src \t// d2l" %}
14536
14537 ins_encode %{
14538 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14539 %}
14540
14541 ins_pipe(fp_d2l);
14542 %}
14543
14544 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14545 match(Set dst (ConvI2D src));
14546
14547 ins_cost(INSN_COST * 5);
14548 format %{ "scvtfwd $dst, $src \t// i2d" %}
14549
14550 ins_encode %{
14551 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14552 %}
14553
14554 ins_pipe(fp_i2d);
14555 %}
14556
14557 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14558 match(Set dst (ConvL2D src));
14559
14560 ins_cost(INSN_COST * 5);
14561 format %{ "scvtfd $dst, $src \t// l2d" %}
14562
14563 ins_encode %{
14564 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14565 %}
14566
14567 ins_pipe(fp_l2d);
14568 %}
14569
14570 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14571 %{
14572 match(Set dst (RoundD src));
14573 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14574 format %{ "java_round_double $dst,$src"%}
14575 ins_encode %{
14576 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14577 as_FloatRegister($ftmp$$reg));
14578 %}
14579 ins_pipe(pipe_slow);
14580 %}
14581
14582 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14583 %{
14584 match(Set dst (RoundF src));
14585 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14586 format %{ "java_round_float $dst,$src"%}
14587 ins_encode %{
14588 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14589 as_FloatRegister($ftmp$$reg));
14590 %}
14591 ins_pipe(pipe_slow);
14592 %}
14593
14594 // stack <-> reg and reg <-> reg shuffles with no conversion
14595
14596 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14597
14598 match(Set dst (MoveF2I src));
14599
14600 effect(DEF dst, USE src);
14601
14602 ins_cost(4 * INSN_COST);
14603
14604 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14605
14606 ins_encode %{
14607 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14608 %}
14609
14610 ins_pipe(iload_reg_reg);
14611
14612 %}
14613
14614 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14615
14616 match(Set dst (MoveI2F src));
14617
14618 effect(DEF dst, USE src);
14619
14620 ins_cost(4 * INSN_COST);
14621
14622 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14623
14624 ins_encode %{
14625 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14626 %}
14627
14628 ins_pipe(pipe_class_memory);
14629
14630 %}
14631
14632 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14633
14634 match(Set dst (MoveD2L src));
14635
14636 effect(DEF dst, USE src);
14637
14638 ins_cost(4 * INSN_COST);
14639
14640 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14641
14642 ins_encode %{
14643 __ ldr($dst$$Register, Address(sp, $src$$disp));
14644 %}
14645
14646 ins_pipe(iload_reg_reg);
14647
14648 %}
14649
14650 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14651
14652 match(Set dst (MoveL2D src));
14653
14654 effect(DEF dst, USE src);
14655
14656 ins_cost(4 * INSN_COST);
14657
14658 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14659
14660 ins_encode %{
14661 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14662 %}
14663
14664 ins_pipe(pipe_class_memory);
14665
14666 %}
14667
14668 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14669
14670 match(Set dst (MoveF2I src));
14671
14672 effect(DEF dst, USE src);
14673
14674 ins_cost(INSN_COST);
14675
14676 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14677
14678 ins_encode %{
14679 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14680 %}
14681
14682 ins_pipe(pipe_class_memory);
14683
14684 %}
14685
14686 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14687
14688 match(Set dst (MoveI2F src));
14689
14690 effect(DEF dst, USE src);
14691
14692 ins_cost(INSN_COST);
14693
14694 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14695
14696 ins_encode %{
14697 __ strw($src$$Register, Address(sp, $dst$$disp));
14698 %}
14699
14700 ins_pipe(istore_reg_reg);
14701
14702 %}
14703
14704 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14705
14706 match(Set dst (MoveD2L src));
14707
14708 effect(DEF dst, USE src);
14709
14710 ins_cost(INSN_COST);
14711
14712 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14713
14714 ins_encode %{
14715 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14716 %}
14717
14718 ins_pipe(pipe_class_memory);
14719
14720 %}
14721
14722 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14723
14724 match(Set dst (MoveL2D src));
14725
14726 effect(DEF dst, USE src);
14727
14728 ins_cost(INSN_COST);
14729
14730 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14731
14732 ins_encode %{
14733 __ str($src$$Register, Address(sp, $dst$$disp));
14734 %}
14735
14736 ins_pipe(istore_reg_reg);
14737
14738 %}
14739
14740 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14741
14742 match(Set dst (MoveF2I src));
14743
14744 effect(DEF dst, USE src);
14745
14746 ins_cost(INSN_COST);
14747
14748 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14749
14750 ins_encode %{
14751 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14752 %}
14753
14754 ins_pipe(fp_f2i);
14755
14756 %}
14757
14758 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14759
14760 match(Set dst (MoveI2F src));
14761
14762 effect(DEF dst, USE src);
14763
14764 ins_cost(INSN_COST);
14765
14766 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14767
14768 ins_encode %{
14769 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14770 %}
14771
14772 ins_pipe(fp_i2f);
14773
14774 %}
14775
14776 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14777
14778 match(Set dst (MoveD2L src));
14779
14780 effect(DEF dst, USE src);
14781
14782 ins_cost(INSN_COST);
14783
14784 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14785
14786 ins_encode %{
14787 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14788 %}
14789
14790 ins_pipe(fp_d2l);
14791
14792 %}
14793
14794 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14795
14796 match(Set dst (MoveL2D src));
14797
14798 effect(DEF dst, USE src);
14799
14800 ins_cost(INSN_COST);
14801
14802 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14803
14804 ins_encode %{
14805 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14806 %}
14807
14808 ins_pipe(fp_l2d);
14809
14810 %}
14811
14812 // ============================================================================
14813 // clearing of an array
14814
14815 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14816 %{
14817 match(Set dummy (ClearArray cnt base));
14818 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14819
14820 ins_cost(4 * INSN_COST);
14821 format %{ "ClearArray $cnt, $base" %}
14822
14823 ins_encode %{
14824 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14825 if (tpc == nullptr) {
14826 ciEnv::current()->record_failure("CodeCache is full");
14827 return;
14828 }
14829 %}
14830
14831 ins_pipe(pipe_class_memory);
14832 %}
14833
14834 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14835 %{
14836 predicate((uint64_t)n->in(2)->get_long()
14837 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14838 match(Set dummy (ClearArray cnt base));
14839 effect(TEMP temp, USE_KILL base, KILL cr);
14840
14841 ins_cost(4 * INSN_COST);
14842 format %{ "ClearArray $cnt, $base" %}
14843
14844 ins_encode %{
14845 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14846 if (tpc == nullptr) {
14847 ciEnv::current()->record_failure("CodeCache is full");
14848 return;
14849 }
14850 %}
14851
14852 ins_pipe(pipe_class_memory);
14853 %}
14854
14855 // ============================================================================
14856 // Overflow Math Instructions
14857
14858 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14859 %{
14860 match(Set cr (OverflowAddI op1 op2));
14861
14862 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14863 ins_cost(INSN_COST);
14864 ins_encode %{
14865 __ cmnw($op1$$Register, $op2$$Register);
14866 %}
14867
14868 ins_pipe(icmp_reg_reg);
14869 %}
14870
14871 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14872 %{
14873 match(Set cr (OverflowAddI op1 op2));
14874
14875 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14876 ins_cost(INSN_COST);
14877 ins_encode %{
14878 __ cmnw($op1$$Register, $op2$$constant);
14879 %}
14880
14881 ins_pipe(icmp_reg_imm);
14882 %}
14883
14884 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14885 %{
14886 match(Set cr (OverflowAddL op1 op2));
14887
14888 format %{ "cmn $op1, $op2\t# overflow check long" %}
14889 ins_cost(INSN_COST);
14890 ins_encode %{
14891 __ cmn($op1$$Register, $op2$$Register);
14892 %}
14893
14894 ins_pipe(icmp_reg_reg);
14895 %}
14896
14897 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14898 %{
14899 match(Set cr (OverflowAddL op1 op2));
14900
14901 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14902 ins_cost(INSN_COST);
14903 ins_encode %{
14904 __ adds(zr, $op1$$Register, $op2$$constant);
14905 %}
14906
14907 ins_pipe(icmp_reg_imm);
14908 %}
14909
14910 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14911 %{
14912 match(Set cr (OverflowSubI op1 op2));
14913
14914 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14915 ins_cost(INSN_COST);
14916 ins_encode %{
14917 __ cmpw($op1$$Register, $op2$$Register);
14918 %}
14919
14920 ins_pipe(icmp_reg_reg);
14921 %}
14922
14923 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14924 %{
14925 match(Set cr (OverflowSubI op1 op2));
14926
14927 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14928 ins_cost(INSN_COST);
14929 ins_encode %{
14930 __ cmpw($op1$$Register, $op2$$constant);
14931 %}
14932
14933 ins_pipe(icmp_reg_imm);
14934 %}
14935
14936 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14937 %{
14938 match(Set cr (OverflowSubL op1 op2));
14939
14940 format %{ "cmp $op1, $op2\t# overflow check long" %}
14941 ins_cost(INSN_COST);
14942 ins_encode %{
14943 __ cmp($op1$$Register, $op2$$Register);
14944 %}
14945
14946 ins_pipe(icmp_reg_reg);
14947 %}
14948
14949 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14950 %{
14951 match(Set cr (OverflowSubL op1 op2));
14952
14953 format %{ "cmp $op1, $op2\t# overflow check long" %}
14954 ins_cost(INSN_COST);
14955 ins_encode %{
14956 __ subs(zr, $op1$$Register, $op2$$constant);
14957 %}
14958
14959 ins_pipe(icmp_reg_imm);
14960 %}
14961
14962 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14963 %{
14964 match(Set cr (OverflowSubI zero op1));
14965
14966 format %{ "cmpw zr, $op1\t# overflow check int" %}
14967 ins_cost(INSN_COST);
14968 ins_encode %{
14969 __ cmpw(zr, $op1$$Register);
14970 %}
14971
14972 ins_pipe(icmp_reg_imm);
14973 %}
14974
14975 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14976 %{
14977 match(Set cr (OverflowSubL zero op1));
14978
14979 format %{ "cmp zr, $op1\t# overflow check long" %}
14980 ins_cost(INSN_COST);
14981 ins_encode %{
14982 __ cmp(zr, $op1$$Register);
14983 %}
14984
14985 ins_pipe(icmp_reg_imm);
14986 %}
14987
14988 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14989 %{
14990 match(Set cr (OverflowMulI op1 op2));
14991
14992 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14993 "cmp rscratch1, rscratch1, sxtw\n\t"
14994 "movw rscratch1, #0x80000000\n\t"
14995 "cselw rscratch1, rscratch1, zr, NE\n\t"
14996 "cmpw rscratch1, #1" %}
14997 ins_cost(5 * INSN_COST);
14998 ins_encode %{
14999 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15000 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15001 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15002 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15003 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15004 %}
15005
15006 ins_pipe(pipe_slow);
15007 %}
15008
15009 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15010 %{
15011 match(If cmp (OverflowMulI op1 op2));
15012 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15013 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15014 effect(USE labl, KILL cr);
15015
15016 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15017 "cmp rscratch1, rscratch1, sxtw\n\t"
15018 "b$cmp $labl" %}
15019 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15020 ins_encode %{
15021 Label* L = $labl$$label;
15022 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15023 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15024 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15025 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15026 %}
15027
15028 ins_pipe(pipe_serial);
15029 %}
15030
15031 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15032 %{
15033 match(Set cr (OverflowMulL op1 op2));
15034
15035 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15036 "smulh rscratch2, $op1, $op2\n\t"
15037 "cmp rscratch2, rscratch1, ASR #63\n\t"
15038 "movw rscratch1, #0x80000000\n\t"
15039 "cselw rscratch1, rscratch1, zr, NE\n\t"
15040 "cmpw rscratch1, #1" %}
15041 ins_cost(6 * INSN_COST);
15042 ins_encode %{
15043 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15044 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15045 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15046 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15047 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15048 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15049 %}
15050
15051 ins_pipe(pipe_slow);
15052 %}
15053
15054 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15055 %{
15056 match(If cmp (OverflowMulL op1 op2));
15057 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15058 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15059 effect(USE labl, KILL cr);
15060
15061 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15062 "smulh rscratch2, $op1, $op2\n\t"
15063 "cmp rscratch2, rscratch1, ASR #63\n\t"
15064 "b$cmp $labl" %}
15065 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15066 ins_encode %{
15067 Label* L = $labl$$label;
15068 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15069 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15070 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15071 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15072 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15073 %}
15074
15075 ins_pipe(pipe_serial);
15076 %}
15077
15078 // ============================================================================
15079 // Compare Instructions
15080
15081 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15082 %{
15083 match(Set cr (CmpI op1 op2));
15084
15085 effect(DEF cr, USE op1, USE op2);
15086
15087 ins_cost(INSN_COST);
15088 format %{ "cmpw $op1, $op2" %}
15089
15090 ins_encode(aarch64_enc_cmpw(op1, op2));
15091
15092 ins_pipe(icmp_reg_reg);
15093 %}
15094
15095 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15096 %{
15097 match(Set cr (CmpI op1 zero));
15098
15099 effect(DEF cr, USE op1);
15100
15101 ins_cost(INSN_COST);
15102 format %{ "cmpw $op1, 0" %}
15103
15104 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15105
15106 ins_pipe(icmp_reg_imm);
15107 %}
15108
15109 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15110 %{
15111 match(Set cr (CmpI op1 op2));
15112
15113 effect(DEF cr, USE op1);
15114
15115 ins_cost(INSN_COST);
15116 format %{ "cmpw $op1, $op2" %}
15117
15118 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15119
15120 ins_pipe(icmp_reg_imm);
15121 %}
15122
15123 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15124 %{
15125 match(Set cr (CmpI op1 op2));
15126
15127 effect(DEF cr, USE op1);
15128
15129 ins_cost(INSN_COST * 2);
15130 format %{ "cmpw $op1, $op2" %}
15131
15132 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15133
15134 ins_pipe(icmp_reg_imm);
15135 %}
15136
15137 // Unsigned compare Instructions; really, same as signed compare
15138 // except it should only be used to feed an If or a CMovI which takes a
15139 // cmpOpU.
15140
15141 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15142 %{
15143 match(Set cr (CmpU op1 op2));
15144
15145 effect(DEF cr, USE op1, USE op2);
15146
15147 ins_cost(INSN_COST);
15148 format %{ "cmpw $op1, $op2\t# unsigned" %}
15149
15150 ins_encode(aarch64_enc_cmpw(op1, op2));
15151
15152 ins_pipe(icmp_reg_reg);
15153 %}
15154
15155 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15156 %{
15157 match(Set cr (CmpU op1 zero));
15158
15159 effect(DEF cr, USE op1);
15160
15161 ins_cost(INSN_COST);
15162 format %{ "cmpw $op1, #0\t# unsigned" %}
15163
15164 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15165
15166 ins_pipe(icmp_reg_imm);
15167 %}
15168
15169 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15170 %{
15171 match(Set cr (CmpU op1 op2));
15172
15173 effect(DEF cr, USE op1);
15174
15175 ins_cost(INSN_COST);
15176 format %{ "cmpw $op1, $op2\t# unsigned" %}
15177
15178 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15179
15180 ins_pipe(icmp_reg_imm);
15181 %}
15182
15183 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15184 %{
15185 match(Set cr (CmpU op1 op2));
15186
15187 effect(DEF cr, USE op1);
15188
15189 ins_cost(INSN_COST * 2);
15190 format %{ "cmpw $op1, $op2\t# unsigned" %}
15191
15192 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15193
15194 ins_pipe(icmp_reg_imm);
15195 %}
15196
15197 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15198 %{
15199 match(Set cr (CmpL op1 op2));
15200
15201 effect(DEF cr, USE op1, USE op2);
15202
15203 ins_cost(INSN_COST);
15204 format %{ "cmp $op1, $op2" %}
15205
15206 ins_encode(aarch64_enc_cmp(op1, op2));
15207
15208 ins_pipe(icmp_reg_reg);
15209 %}
15210
15211 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15212 %{
15213 match(Set cr (CmpL op1 zero));
15214
15215 effect(DEF cr, USE op1);
15216
15217 ins_cost(INSN_COST);
15218 format %{ "tst $op1" %}
15219
15220 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15221
15222 ins_pipe(icmp_reg_imm);
15223 %}
15224
15225 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15226 %{
15227 match(Set cr (CmpL op1 op2));
15228
15229 effect(DEF cr, USE op1);
15230
15231 ins_cost(INSN_COST);
15232 format %{ "cmp $op1, $op2" %}
15233
15234 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15235
15236 ins_pipe(icmp_reg_imm);
15237 %}
15238
15239 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15240 %{
15241 match(Set cr (CmpL op1 op2));
15242
15243 effect(DEF cr, USE op1);
15244
15245 ins_cost(INSN_COST * 2);
15246 format %{ "cmp $op1, $op2" %}
15247
15248 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15249
15250 ins_pipe(icmp_reg_imm);
15251 %}
15252
15253 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15254 %{
15255 match(Set cr (CmpUL op1 op2));
15256
15257 effect(DEF cr, USE op1, USE op2);
15258
15259 ins_cost(INSN_COST);
15260 format %{ "cmp $op1, $op2" %}
15261
15262 ins_encode(aarch64_enc_cmp(op1, op2));
15263
15264 ins_pipe(icmp_reg_reg);
15265 %}
15266
15267 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15268 %{
15269 match(Set cr (CmpUL op1 zero));
15270
15271 effect(DEF cr, USE op1);
15272
15273 ins_cost(INSN_COST);
15274 format %{ "tst $op1" %}
15275
15276 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15277
15278 ins_pipe(icmp_reg_imm);
15279 %}
15280
15281 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15282 %{
15283 match(Set cr (CmpUL op1 op2));
15284
15285 effect(DEF cr, USE op1);
15286
15287 ins_cost(INSN_COST);
15288 format %{ "cmp $op1, $op2" %}
15289
15290 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15291
15292 ins_pipe(icmp_reg_imm);
15293 %}
15294
15295 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15296 %{
15297 match(Set cr (CmpUL op1 op2));
15298
15299 effect(DEF cr, USE op1);
15300
15301 ins_cost(INSN_COST * 2);
15302 format %{ "cmp $op1, $op2" %}
15303
15304 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15305
15306 ins_pipe(icmp_reg_imm);
15307 %}
15308
15309 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15310 %{
15311 match(Set cr (CmpP op1 op2));
15312
15313 effect(DEF cr, USE op1, USE op2);
15314
15315 ins_cost(INSN_COST);
15316 format %{ "cmp $op1, $op2\t // ptr" %}
15317
15318 ins_encode(aarch64_enc_cmpp(op1, op2));
15319
15320 ins_pipe(icmp_reg_reg);
15321 %}
15322
15323 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15324 %{
15325 match(Set cr (CmpN op1 op2));
15326
15327 effect(DEF cr, USE op1, USE op2);
15328
15329 ins_cost(INSN_COST);
15330 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15331
15332 ins_encode(aarch64_enc_cmpn(op1, op2));
15333
15334 ins_pipe(icmp_reg_reg);
15335 %}
15336
15337 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15338 %{
15339 match(Set cr (CmpP op1 zero));
15340
15341 effect(DEF cr, USE op1, USE zero);
15342
15343 ins_cost(INSN_COST);
15344 format %{ "cmp $op1, 0\t // ptr" %}
15345
15346 ins_encode(aarch64_enc_testp(op1));
15347
15348 ins_pipe(icmp_reg_imm);
15349 %}
15350
15351 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15352 %{
15353 match(Set cr (CmpN op1 zero));
15354
15355 effect(DEF cr, USE op1, USE zero);
15356
15357 ins_cost(INSN_COST);
15358 format %{ "cmp $op1, 0\t // compressed ptr" %}
15359
15360 ins_encode(aarch64_enc_testn(op1));
15361
15362 ins_pipe(icmp_reg_imm);
15363 %}
15364
15365 // FP comparisons
15366 //
15367 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15368 // using normal cmpOp. See declaration of rFlagsReg for details.
15369
15370 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15371 %{
15372 match(Set cr (CmpF src1 src2));
15373
15374 ins_cost(3 * INSN_COST);
15375 format %{ "fcmps $src1, $src2" %}
15376
15377 ins_encode %{
15378 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15379 %}
15380
15381 ins_pipe(pipe_class_compare);
15382 %}
15383
15384 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15385 %{
15386 match(Set cr (CmpF src1 src2));
15387
15388 ins_cost(3 * INSN_COST);
15389 format %{ "fcmps $src1, 0.0" %}
15390
15391 ins_encode %{
15392 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15393 %}
15394
15395 ins_pipe(pipe_class_compare);
15396 %}
15397 // FROM HERE
15398
15399 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15400 %{
15401 match(Set cr (CmpD src1 src2));
15402
15403 ins_cost(3 * INSN_COST);
15404 format %{ "fcmpd $src1, $src2" %}
15405
15406 ins_encode %{
15407 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15408 %}
15409
15410 ins_pipe(pipe_class_compare);
15411 %}
15412
15413 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15414 %{
15415 match(Set cr (CmpD src1 src2));
15416
15417 ins_cost(3 * INSN_COST);
15418 format %{ "fcmpd $src1, 0.0" %}
15419
15420 ins_encode %{
15421 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15422 %}
15423
15424 ins_pipe(pipe_class_compare);
15425 %}
15426
15427 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15428 %{
15429 match(Set dst (CmpF3 src1 src2));
15430 effect(KILL cr);
15431
15432 ins_cost(5 * INSN_COST);
15433 format %{ "fcmps $src1, $src2\n\t"
15434 "csinvw($dst, zr, zr, eq\n\t"
15435 "csnegw($dst, $dst, $dst, lt)"
15436 %}
15437
15438 ins_encode %{
15439 Label done;
15440 FloatRegister s1 = as_FloatRegister($src1$$reg);
15441 FloatRegister s2 = as_FloatRegister($src2$$reg);
15442 Register d = as_Register($dst$$reg);
15443 __ fcmps(s1, s2);
15444 // installs 0 if EQ else -1
15445 __ csinvw(d, zr, zr, Assembler::EQ);
15446 // keeps -1 if less or unordered else installs 1
15447 __ csnegw(d, d, d, Assembler::LT);
15448 __ bind(done);
15449 %}
15450
15451 ins_pipe(pipe_class_default);
15452
15453 %}
15454
15455 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15456 %{
15457 match(Set dst (CmpD3 src1 src2));
15458 effect(KILL cr);
15459
15460 ins_cost(5 * INSN_COST);
15461 format %{ "fcmpd $src1, $src2\n\t"
15462 "csinvw($dst, zr, zr, eq\n\t"
15463 "csnegw($dst, $dst, $dst, lt)"
15464 %}
15465
15466 ins_encode %{
15467 Label done;
15468 FloatRegister s1 = as_FloatRegister($src1$$reg);
15469 FloatRegister s2 = as_FloatRegister($src2$$reg);
15470 Register d = as_Register($dst$$reg);
15471 __ fcmpd(s1, s2);
15472 // installs 0 if EQ else -1
15473 __ csinvw(d, zr, zr, Assembler::EQ);
15474 // keeps -1 if less or unordered else installs 1
15475 __ csnegw(d, d, d, Assembler::LT);
15476 __ bind(done);
15477 %}
15478 ins_pipe(pipe_class_default);
15479
15480 %}
15481
15482 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15483 %{
15484 match(Set dst (CmpF3 src1 zero));
15485 effect(KILL cr);
15486
15487 ins_cost(5 * INSN_COST);
15488 format %{ "fcmps $src1, 0.0\n\t"
15489 "csinvw($dst, zr, zr, eq\n\t"
15490 "csnegw($dst, $dst, $dst, lt)"
15491 %}
15492
15493 ins_encode %{
15494 Label done;
15495 FloatRegister s1 = as_FloatRegister($src1$$reg);
15496 Register d = as_Register($dst$$reg);
15497 __ fcmps(s1, 0.0);
15498 // installs 0 if EQ else -1
15499 __ csinvw(d, zr, zr, Assembler::EQ);
15500 // keeps -1 if less or unordered else installs 1
15501 __ csnegw(d, d, d, Assembler::LT);
15502 __ bind(done);
15503 %}
15504
15505 ins_pipe(pipe_class_default);
15506
15507 %}
15508
15509 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15510 %{
15511 match(Set dst (CmpD3 src1 zero));
15512 effect(KILL cr);
15513
15514 ins_cost(5 * INSN_COST);
15515 format %{ "fcmpd $src1, 0.0\n\t"
15516 "csinvw($dst, zr, zr, eq\n\t"
15517 "csnegw($dst, $dst, $dst, lt)"
15518 %}
15519
15520 ins_encode %{
15521 Label done;
15522 FloatRegister s1 = as_FloatRegister($src1$$reg);
15523 Register d = as_Register($dst$$reg);
15524 __ fcmpd(s1, 0.0);
15525 // installs 0 if EQ else -1
15526 __ csinvw(d, zr, zr, Assembler::EQ);
15527 // keeps -1 if less or unordered else installs 1
15528 __ csnegw(d, d, d, Assembler::LT);
15529 __ bind(done);
15530 %}
15531 ins_pipe(pipe_class_default);
15532
15533 %}
15534
15535 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15536 %{
15537 match(Set dst (CmpLTMask p q));
15538 effect(KILL cr);
15539
15540 ins_cost(3 * INSN_COST);
15541
15542 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15543 "csetw $dst, lt\n\t"
15544 "subw $dst, zr, $dst"
15545 %}
15546
15547 ins_encode %{
15548 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15549 __ csetw(as_Register($dst$$reg), Assembler::LT);
15550 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15551 %}
15552
15553 ins_pipe(ialu_reg_reg);
15554 %}
15555
15556 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15557 %{
15558 match(Set dst (CmpLTMask src zero));
15559 effect(KILL cr);
15560
15561 ins_cost(INSN_COST);
15562
15563 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15564
15565 ins_encode %{
15566 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15567 %}
15568
15569 ins_pipe(ialu_reg_shift);
15570 %}
15571
15572 // ============================================================================
15573 // Max and Min
15574
15575 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15576
15577 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15578 %{
15579 effect(DEF cr, USE src);
15580 ins_cost(INSN_COST);
15581 format %{ "cmpw $src, 0" %}
15582
15583 ins_encode %{
15584 __ cmpw($src$$Register, 0);
15585 %}
15586 ins_pipe(icmp_reg_imm);
15587 %}
15588
15589 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15590 %{
15591 match(Set dst (MinI src1 src2));
15592 ins_cost(INSN_COST * 3);
15593
15594 expand %{
15595 rFlagsReg cr;
15596 compI_reg_reg(cr, src1, src2);
15597 cmovI_reg_reg_lt(dst, src1, src2, cr);
15598 %}
15599 %}
15600
15601 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15602 %{
15603 match(Set dst (MaxI src1 src2));
15604 ins_cost(INSN_COST * 3);
15605
15606 expand %{
15607 rFlagsReg cr;
15608 compI_reg_reg(cr, src1, src2);
15609 cmovI_reg_reg_gt(dst, src1, src2, cr);
15610 %}
15611 %}
15612
15613
15614 // ============================================================================
15615 // Branch Instructions
15616
15617 // Direct Branch.
15618 instruct branch(label lbl)
15619 %{
15620 match(Goto);
15621
15622 effect(USE lbl);
15623
15624 ins_cost(BRANCH_COST);
15625 format %{ "b $lbl" %}
15626
15627 ins_encode(aarch64_enc_b(lbl));
15628
15629 ins_pipe(pipe_branch);
15630 %}
15631
15632 // Conditional Near Branch
15633 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15634 %{
15635 // Same match rule as `branchConFar'.
15636 match(If cmp cr);
15637
15638 effect(USE lbl);
15639
15640 ins_cost(BRANCH_COST);
15641 // If set to 1 this indicates that the current instruction is a
15642 // short variant of a long branch. This avoids using this
15643 // instruction in first-pass matching. It will then only be used in
15644 // the `Shorten_branches' pass.
15645 // ins_short_branch(1);
15646 format %{ "b$cmp $lbl" %}
15647
15648 ins_encode(aarch64_enc_br_con(cmp, lbl));
15649
15650 ins_pipe(pipe_branch_cond);
15651 %}
15652
15653 // Conditional Near Branch Unsigned
15654 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15655 %{
15656 // Same match rule as `branchConFar'.
15657 match(If cmp cr);
15658
15659 effect(USE lbl);
15660
15661 ins_cost(BRANCH_COST);
15662 // If set to 1 this indicates that the current instruction is a
15663 // short variant of a long branch. This avoids using this
15664 // instruction in first-pass matching. It will then only be used in
15665 // the `Shorten_branches' pass.
15666 // ins_short_branch(1);
15667 format %{ "b$cmp $lbl\t# unsigned" %}
15668
15669 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15670
15671 ins_pipe(pipe_branch_cond);
15672 %}
15673
15674 // Make use of CBZ and CBNZ. These instructions, as well as being
15675 // shorter than (cmp; branch), have the additional benefit of not
15676 // killing the flags.
15677
15678 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15679 match(If cmp (CmpI op1 op2));
15680 effect(USE labl);
15681
15682 ins_cost(BRANCH_COST);
15683 format %{ "cbw$cmp $op1, $labl" %}
15684 ins_encode %{
15685 Label* L = $labl$$label;
15686 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15687 if (cond == Assembler::EQ)
15688 __ cbzw($op1$$Register, *L);
15689 else
15690 __ cbnzw($op1$$Register, *L);
15691 %}
15692 ins_pipe(pipe_cmp_branch);
15693 %}
15694
15695 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15696 match(If cmp (CmpL op1 op2));
15697 effect(USE labl);
15698
15699 ins_cost(BRANCH_COST);
15700 format %{ "cb$cmp $op1, $labl" %}
15701 ins_encode %{
15702 Label* L = $labl$$label;
15703 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15704 if (cond == Assembler::EQ)
15705 __ cbz($op1$$Register, *L);
15706 else
15707 __ cbnz($op1$$Register, *L);
15708 %}
15709 ins_pipe(pipe_cmp_branch);
15710 %}
15711
15712 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15713 match(If cmp (CmpP op1 op2));
15714 effect(USE labl);
15715
15716 ins_cost(BRANCH_COST);
15717 format %{ "cb$cmp $op1, $labl" %}
15718 ins_encode %{
15719 Label* L = $labl$$label;
15720 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15721 if (cond == Assembler::EQ)
15722 __ cbz($op1$$Register, *L);
15723 else
15724 __ cbnz($op1$$Register, *L);
15725 %}
15726 ins_pipe(pipe_cmp_branch);
15727 %}
15728
15729 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15730 match(If cmp (CmpN op1 op2));
15731 effect(USE labl);
15732
15733 ins_cost(BRANCH_COST);
15734 format %{ "cbw$cmp $op1, $labl" %}
15735 ins_encode %{
15736 Label* L = $labl$$label;
15737 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15738 if (cond == Assembler::EQ)
15739 __ cbzw($op1$$Register, *L);
15740 else
15741 __ cbnzw($op1$$Register, *L);
15742 %}
15743 ins_pipe(pipe_cmp_branch);
15744 %}
15745
15746 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15747 match(If cmp (CmpP (DecodeN oop) zero));
15748 effect(USE labl);
15749
15750 ins_cost(BRANCH_COST);
15751 format %{ "cb$cmp $oop, $labl" %}
15752 ins_encode %{
15753 Label* L = $labl$$label;
15754 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15755 if (cond == Assembler::EQ)
15756 __ cbzw($oop$$Register, *L);
15757 else
15758 __ cbnzw($oop$$Register, *L);
15759 %}
15760 ins_pipe(pipe_cmp_branch);
15761 %}
15762
15763 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15764 match(If cmp (CmpU op1 op2));
15765 effect(USE labl);
15766
15767 ins_cost(BRANCH_COST);
15768 format %{ "cbw$cmp $op1, $labl" %}
15769 ins_encode %{
15770 Label* L = $labl$$label;
15771 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15772 if (cond == Assembler::EQ || cond == Assembler::LS) {
15773 __ cbzw($op1$$Register, *L);
15774 } else {
15775 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15776 __ cbnzw($op1$$Register, *L);
15777 }
15778 %}
15779 ins_pipe(pipe_cmp_branch);
15780 %}
15781
15782 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15783 match(If cmp (CmpUL op1 op2));
15784 effect(USE labl);
15785
15786 ins_cost(BRANCH_COST);
15787 format %{ "cb$cmp $op1, $labl" %}
15788 ins_encode %{
15789 Label* L = $labl$$label;
15790 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15791 if (cond == Assembler::EQ || cond == Assembler::LS) {
15792 __ cbz($op1$$Register, *L);
15793 } else {
15794 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15795 __ cbnz($op1$$Register, *L);
15796 }
15797 %}
15798 ins_pipe(pipe_cmp_branch);
15799 %}
15800
15801 // Test bit and Branch
15802
15803 // Patterns for short (< 32KiB) variants
15804 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15805 match(If cmp (CmpL op1 op2));
15806 effect(USE labl);
15807
15808 ins_cost(BRANCH_COST);
15809 format %{ "cb$cmp $op1, $labl # long" %}
15810 ins_encode %{
15811 Label* L = $labl$$label;
15812 Assembler::Condition cond =
15813 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15814 __ tbr(cond, $op1$$Register, 63, *L);
15815 %}
15816 ins_pipe(pipe_cmp_branch);
15817 ins_short_branch(1);
15818 %}
15819
15820 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15821 match(If cmp (CmpI op1 op2));
15822 effect(USE labl);
15823
15824 ins_cost(BRANCH_COST);
15825 format %{ "cb$cmp $op1, $labl # int" %}
15826 ins_encode %{
15827 Label* L = $labl$$label;
15828 Assembler::Condition cond =
15829 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15830 __ tbr(cond, $op1$$Register, 31, *L);
15831 %}
15832 ins_pipe(pipe_cmp_branch);
15833 ins_short_branch(1);
15834 %}
15835
15836 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15837 match(If cmp (CmpL (AndL op1 op2) op3));
15838 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15839 effect(USE labl);
15840
15841 ins_cost(BRANCH_COST);
15842 format %{ "tb$cmp $op1, $op2, $labl" %}
15843 ins_encode %{
15844 Label* L = $labl$$label;
15845 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15846 int bit = exact_log2_long($op2$$constant);
15847 __ tbr(cond, $op1$$Register, bit, *L);
15848 %}
15849 ins_pipe(pipe_cmp_branch);
15850 ins_short_branch(1);
15851 %}
15852
15853 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15854 match(If cmp (CmpI (AndI op1 op2) op3));
15855 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15856 effect(USE labl);
15857
15858 ins_cost(BRANCH_COST);
15859 format %{ "tb$cmp $op1, $op2, $labl" %}
15860 ins_encode %{
15861 Label* L = $labl$$label;
15862 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15863 int bit = exact_log2((juint)$op2$$constant);
15864 __ tbr(cond, $op1$$Register, bit, *L);
15865 %}
15866 ins_pipe(pipe_cmp_branch);
15867 ins_short_branch(1);
15868 %}
15869
15870 // And far variants
15871 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15872 match(If cmp (CmpL op1 op2));
15873 effect(USE labl);
15874
15875 ins_cost(BRANCH_COST);
15876 format %{ "cb$cmp $op1, $labl # long" %}
15877 ins_encode %{
15878 Label* L = $labl$$label;
15879 Assembler::Condition cond =
15880 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15881 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15882 %}
15883 ins_pipe(pipe_cmp_branch);
15884 %}
15885
15886 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15887 match(If cmp (CmpI op1 op2));
15888 effect(USE labl);
15889
15890 ins_cost(BRANCH_COST);
15891 format %{ "cb$cmp $op1, $labl # int" %}
15892 ins_encode %{
15893 Label* L = $labl$$label;
15894 Assembler::Condition cond =
15895 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15896 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15897 %}
15898 ins_pipe(pipe_cmp_branch);
15899 %}
15900
15901 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15902 match(If cmp (CmpL (AndL op1 op2) op3));
15903 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15904 effect(USE labl);
15905
15906 ins_cost(BRANCH_COST);
15907 format %{ "tb$cmp $op1, $op2, $labl" %}
15908 ins_encode %{
15909 Label* L = $labl$$label;
15910 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15911 int bit = exact_log2_long($op2$$constant);
15912 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15913 %}
15914 ins_pipe(pipe_cmp_branch);
15915 %}
15916
15917 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15918 match(If cmp (CmpI (AndI op1 op2) op3));
15919 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15920 effect(USE labl);
15921
15922 ins_cost(BRANCH_COST);
15923 format %{ "tb$cmp $op1, $op2, $labl" %}
15924 ins_encode %{
15925 Label* L = $labl$$label;
15926 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15927 int bit = exact_log2((juint)$op2$$constant);
15928 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15929 %}
15930 ins_pipe(pipe_cmp_branch);
15931 %}
15932
15933 // Test bits
15934
15935 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15936 match(Set cr (CmpL (AndL op1 op2) op3));
15937 predicate(Assembler::operand_valid_for_logical_immediate
15938 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15939
15940 ins_cost(INSN_COST);
15941 format %{ "tst $op1, $op2 # long" %}
15942 ins_encode %{
15943 __ tst($op1$$Register, $op2$$constant);
15944 %}
15945 ins_pipe(ialu_reg_reg);
15946 %}
15947
15948 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15949 match(Set cr (CmpI (AndI op1 op2) op3));
15950 predicate(Assembler::operand_valid_for_logical_immediate
15951 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15952
15953 ins_cost(INSN_COST);
15954 format %{ "tst $op1, $op2 # int" %}
15955 ins_encode %{
15956 __ tstw($op1$$Register, $op2$$constant);
15957 %}
15958 ins_pipe(ialu_reg_reg);
15959 %}
15960
15961 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15962 match(Set cr (CmpL (AndL op1 op2) op3));
15963
15964 ins_cost(INSN_COST);
15965 format %{ "tst $op1, $op2 # long" %}
15966 ins_encode %{
15967 __ tst($op1$$Register, $op2$$Register);
15968 %}
15969 ins_pipe(ialu_reg_reg);
15970 %}
15971
15972 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15973 match(Set cr (CmpI (AndI op1 op2) op3));
15974
15975 ins_cost(INSN_COST);
15976 format %{ "tstw $op1, $op2 # int" %}
15977 ins_encode %{
15978 __ tstw($op1$$Register, $op2$$Register);
15979 %}
15980 ins_pipe(ialu_reg_reg);
15981 %}
15982
15983
15984 // Conditional Far Branch
15985 // Conditional Far Branch Unsigned
15986 // TODO: fixme
15987
15988 // counted loop end branch near
15989 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15990 %{
15991 match(CountedLoopEnd cmp cr);
15992
15993 effect(USE lbl);
15994
15995 ins_cost(BRANCH_COST);
15996 // short variant.
15997 // ins_short_branch(1);
15998 format %{ "b$cmp $lbl \t// counted loop end" %}
15999
16000 ins_encode(aarch64_enc_br_con(cmp, lbl));
16001
16002 ins_pipe(pipe_branch);
16003 %}
16004
16005 // counted loop end branch far
16006 // TODO: fixme
16007
16008 // ============================================================================
16009 // inlined locking and unlocking
16010
16011 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16012 %{
16013 predicate(LockingMode != LM_LIGHTWEIGHT);
16014 match(Set cr (FastLock object box));
16015 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16016
16017 ins_cost(5 * INSN_COST);
16018 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16019
16020 ins_encode %{
16021 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16022 %}
16023
16024 ins_pipe(pipe_serial);
16025 %}
16026
16027 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16028 %{
16029 predicate(LockingMode != LM_LIGHTWEIGHT);
16030 match(Set cr (FastUnlock object box));
16031 effect(TEMP tmp, TEMP tmp2);
16032
16033 ins_cost(5 * INSN_COST);
16034 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16035
16036 ins_encode %{
16037 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
16038 %}
16039
16040 ins_pipe(pipe_serial);
16041 %}
16042
16043 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16044 %{
16045 predicate(LockingMode == LM_LIGHTWEIGHT);
16046 match(Set cr (FastLock object box));
16047 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16048
16049 ins_cost(5 * INSN_COST);
16050 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16051
16052 ins_encode %{
16053 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16054 %}
16055
16056 ins_pipe(pipe_serial);
16057 %}
16058
16059 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16060 %{
16061 predicate(LockingMode == LM_LIGHTWEIGHT);
16062 match(Set cr (FastUnlock object box));
16063 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16064
16065 ins_cost(5 * INSN_COST);
16066 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16067
16068 ins_encode %{
16069 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16070 %}
16071
16072 ins_pipe(pipe_serial);
16073 %}
16074
16075 // ============================================================================
16076 // Safepoint Instructions
16077
16078 // TODO
16079 // provide a near and far version of this code
16080
16081 instruct safePoint(rFlagsReg cr, iRegP poll)
16082 %{
16083 match(SafePoint poll);
16084 effect(KILL cr);
16085
16086 format %{
16087 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16088 %}
16089 ins_encode %{
16090 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16091 %}
16092 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16093 %}
16094
16095
16096 // ============================================================================
16097 // Procedure Call/Return Instructions
16098
16099 // Call Java Static Instruction
16100
16101 instruct CallStaticJavaDirect(method meth)
16102 %{
16103 match(CallStaticJava);
16104
16105 effect(USE meth);
16106
16107 ins_cost(CALL_COST);
16108
16109 format %{ "call,static $meth \t// ==> " %}
16110
16111 ins_encode(aarch64_enc_java_static_call(meth),
16112 aarch64_enc_call_epilog);
16113
16114 ins_pipe(pipe_class_call);
16115 %}
16116
16117 // TO HERE
16118
16119 // Call Java Dynamic Instruction
16120 instruct CallDynamicJavaDirect(method meth)
16121 %{
16122 match(CallDynamicJava);
16123
16124 effect(USE meth);
16125
16126 ins_cost(CALL_COST);
16127
16128 format %{ "CALL,dynamic $meth \t// ==> " %}
16129
16130 ins_encode(aarch64_enc_java_dynamic_call(meth),
16131 aarch64_enc_call_epilog);
16132
16133 ins_pipe(pipe_class_call);
16134 %}
16135
16136 // Call Runtime Instruction
16137
16138 instruct CallRuntimeDirect(method meth)
16139 %{
16140 match(CallRuntime);
16141
16142 effect(USE meth);
16143
16144 ins_cost(CALL_COST);
16145
16146 format %{ "CALL, runtime $meth" %}
16147
16148 ins_encode( aarch64_enc_java_to_runtime(meth) );
16149
16150 ins_pipe(pipe_class_call);
16151 %}
16152
16153 // Call Runtime Instruction
16154
16155 instruct CallLeafDirect(method meth)
16156 %{
16157 match(CallLeaf);
16158
16159 effect(USE meth);
16160
16161 ins_cost(CALL_COST);
16162
16163 format %{ "CALL, runtime leaf $meth" %}
16164
16165 ins_encode( aarch64_enc_java_to_runtime(meth) );
16166
16167 ins_pipe(pipe_class_call);
16168 %}
16169
16170 // Call Runtime Instruction
16171
16172 instruct CallLeafNoFPDirect(method meth)
16173 %{
16174 match(CallLeafNoFP);
16175
16176 effect(USE meth);
16177
16178 ins_cost(CALL_COST);
16179
16180 format %{ "CALL, runtime leaf nofp $meth" %}
16181
16182 ins_encode( aarch64_enc_java_to_runtime(meth) );
16183
16184 ins_pipe(pipe_class_call);
16185 %}
16186
16187 // Tail Call; Jump from runtime stub to Java code.
16188 // Also known as an 'interprocedural jump'.
16189 // Target of jump will eventually return to caller.
16190 // TailJump below removes the return address.
16191 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16192 // emitted just above the TailCall which has reset rfp to the caller state.
16193 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16194 %{
16195 match(TailCall jump_target method_ptr);
16196
16197 ins_cost(CALL_COST);
16198
16199 format %{ "br $jump_target\t# $method_ptr holds method" %}
16200
16201 ins_encode(aarch64_enc_tail_call(jump_target));
16202
16203 ins_pipe(pipe_class_call);
16204 %}
16205
16206 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16207 %{
16208 match(TailJump jump_target ex_oop);
16209
16210 ins_cost(CALL_COST);
16211
16212 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16213
16214 ins_encode(aarch64_enc_tail_jmp(jump_target));
16215
16216 ins_pipe(pipe_class_call);
16217 %}
16218
16219 // Forward exception.
16220 instruct ForwardExceptionjmp()
16221 %{
16222 match(ForwardException);
16223 ins_cost(CALL_COST);
16224
16225 format %{ "b forward_exception_stub" %}
16226 ins_encode %{
16227 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16228 %}
16229 ins_pipe(pipe_class_call);
16230 %}
16231
16232 // Create exception oop: created by stack-crawling runtime code.
16233 // Created exception is now available to this handler, and is setup
16234 // just prior to jumping to this handler. No code emitted.
16235 // TODO check
16236 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16237 instruct CreateException(iRegP_R0 ex_oop)
16238 %{
16239 match(Set ex_oop (CreateEx));
16240
16241 format %{ " -- \t// exception oop; no code emitted" %}
16242
16243 size(0);
16244
16245 ins_encode( /*empty*/ );
16246
16247 ins_pipe(pipe_class_empty);
16248 %}
16249
16250 // Rethrow exception: The exception oop will come in the first
16251 // argument position. Then JUMP (not call) to the rethrow stub code.
16252 instruct RethrowException() %{
16253 match(Rethrow);
16254 ins_cost(CALL_COST);
16255
16256 format %{ "b rethrow_stub" %}
16257
16258 ins_encode( aarch64_enc_rethrow() );
16259
16260 ins_pipe(pipe_class_call);
16261 %}
16262
16263
16264 // Return Instruction
16265 // epilog node loads ret address into lr as part of frame pop
16266 instruct Ret()
16267 %{
16268 match(Return);
16269
16270 format %{ "ret\t// return register" %}
16271
16272 ins_encode( aarch64_enc_ret() );
16273
16274 ins_pipe(pipe_branch);
16275 %}
16276
16277 // Die now.
16278 instruct ShouldNotReachHere() %{
16279 match(Halt);
16280
16281 ins_cost(CALL_COST);
16282 format %{ "ShouldNotReachHere" %}
16283
16284 ins_encode %{
16285 if (is_reachable()) {
16286 __ stop(_halt_reason);
16287 }
16288 %}
16289
16290 ins_pipe(pipe_class_default);
16291 %}
16292
16293 // ============================================================================
16294 // Partial Subtype Check
16295 //
16296 // superklass array for an instance of the superklass. Set a hidden
16297 // internal cache on a hit (cache is checked with exposed code in
16298 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16299 // encoding ALSO sets flags.
16300
16301 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16302 %{
16303 match(Set result (PartialSubtypeCheck sub super));
16304 effect(KILL cr, KILL temp);
16305
16306 ins_cost(1100); // slightly larger than the next version
16307 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16308
16309 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16310
16311 opcode(0x1); // Force zero of result reg on hit
16312
16313 ins_pipe(pipe_class_memory);
16314 %}
16315
16316 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16317 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16318 rFlagsReg cr)
16319 %{
16320 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16321 predicate(UseSecondarySupersTable);
16322 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16323
16324 ins_cost(700); // smaller than the next version
16325 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16326
16327 ins_encode %{
16328 bool success = false;
16329 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16330 if (InlineSecondarySupersTest) {
16331 success = __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register,
16332 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16333 $vtemp$$FloatRegister,
16334 $result$$Register,
16335 super_klass_slot);
16336 } else {
16337 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16338 success = (call != nullptr);
16339 }
16340 if (!success) {
16341 ciEnv::current()->record_failure("CodeCache is full");
16342 return;
16343 }
16344 %}
16345
16346 ins_pipe(pipe_class_memory);
16347 %}
16348
16349 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16350 %{
16351 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16352 effect(KILL temp, KILL result);
16353
16354 ins_cost(1100); // slightly larger than the next version
16355 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16356
16357 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16358
16359 opcode(0x0); // Don't zero result reg on hit
16360
16361 ins_pipe(pipe_class_memory);
16362 %}
16363
16364 // Intrisics for String.compareTo()
16365
16366 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16367 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16368 %{
16369 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16370 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16371 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16372
16373 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16374 ins_encode %{
16375 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16376 __ string_compare($str1$$Register, $str2$$Register,
16377 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16378 $tmp1$$Register, $tmp2$$Register,
16379 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16380 %}
16381 ins_pipe(pipe_class_memory);
16382 %}
16383
16384 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16385 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16386 %{
16387 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16388 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16389 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16390
16391 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16392 ins_encode %{
16393 __ string_compare($str1$$Register, $str2$$Register,
16394 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16395 $tmp1$$Register, $tmp2$$Register,
16396 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16397 %}
16398 ins_pipe(pipe_class_memory);
16399 %}
16400
16401 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16402 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16403 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16404 %{
16405 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16406 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16407 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16408 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16409
16410 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16411 ins_encode %{
16412 __ string_compare($str1$$Register, $str2$$Register,
16413 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16414 $tmp1$$Register, $tmp2$$Register,
16415 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16416 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16417 %}
16418 ins_pipe(pipe_class_memory);
16419 %}
16420
16421 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16422 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16423 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16424 %{
16425 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16426 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16427 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16428 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16429
16430 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16431 ins_encode %{
16432 __ string_compare($str1$$Register, $str2$$Register,
16433 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16434 $tmp1$$Register, $tmp2$$Register,
16435 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16436 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16437 %}
16438 ins_pipe(pipe_class_memory);
16439 %}
16440
16441 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16442 // these string_compare variants as NEON register type for convenience so that the prototype of
16443 // string_compare can be shared with all variants.
16444
16445 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16446 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16447 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16448 pRegGov_P1 pgtmp2, rFlagsReg cr)
16449 %{
16450 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16451 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16452 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16453 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16454
16455 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16456 ins_encode %{
16457 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16458 __ string_compare($str1$$Register, $str2$$Register,
16459 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16460 $tmp1$$Register, $tmp2$$Register,
16461 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16462 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16463 StrIntrinsicNode::LL);
16464 %}
16465 ins_pipe(pipe_class_memory);
16466 %}
16467
16468 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16469 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16470 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16471 pRegGov_P1 pgtmp2, rFlagsReg cr)
16472 %{
16473 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16474 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16475 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16476 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16477
16478 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16479 ins_encode %{
16480 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16481 __ string_compare($str1$$Register, $str2$$Register,
16482 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16483 $tmp1$$Register, $tmp2$$Register,
16484 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16485 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16486 StrIntrinsicNode::LU);
16487 %}
16488 ins_pipe(pipe_class_memory);
16489 %}
16490
16491 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16492 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16493 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16494 pRegGov_P1 pgtmp2, rFlagsReg cr)
16495 %{
16496 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16497 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16498 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16499 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16500
16501 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16502 ins_encode %{
16503 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16504 __ string_compare($str1$$Register, $str2$$Register,
16505 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16506 $tmp1$$Register, $tmp2$$Register,
16507 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16508 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16509 StrIntrinsicNode::UL);
16510 %}
16511 ins_pipe(pipe_class_memory);
16512 %}
16513
16514 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16515 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16516 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16517 pRegGov_P1 pgtmp2, rFlagsReg cr)
16518 %{
16519 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16520 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16521 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16522 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16523
16524 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16525 ins_encode %{
16526 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16527 __ string_compare($str1$$Register, $str2$$Register,
16528 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16529 $tmp1$$Register, $tmp2$$Register,
16530 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16531 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16532 StrIntrinsicNode::UU);
16533 %}
16534 ins_pipe(pipe_class_memory);
16535 %}
16536
16537 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16538 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16539 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16540 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16541 %{
16542 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16543 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16544 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16545 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16546 TEMP vtmp0, TEMP vtmp1, KILL cr);
16547 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16548 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16549
16550 ins_encode %{
16551 __ string_indexof($str1$$Register, $str2$$Register,
16552 $cnt1$$Register, $cnt2$$Register,
16553 $tmp1$$Register, $tmp2$$Register,
16554 $tmp3$$Register, $tmp4$$Register,
16555 $tmp5$$Register, $tmp6$$Register,
16556 -1, $result$$Register, StrIntrinsicNode::UU);
16557 %}
16558 ins_pipe(pipe_class_memory);
16559 %}
16560
16561 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16562 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16563 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16564 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16565 %{
16566 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16567 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16568 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16569 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16570 TEMP vtmp0, TEMP vtmp1, KILL cr);
16571 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16572 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16573
16574 ins_encode %{
16575 __ string_indexof($str1$$Register, $str2$$Register,
16576 $cnt1$$Register, $cnt2$$Register,
16577 $tmp1$$Register, $tmp2$$Register,
16578 $tmp3$$Register, $tmp4$$Register,
16579 $tmp5$$Register, $tmp6$$Register,
16580 -1, $result$$Register, StrIntrinsicNode::LL);
16581 %}
16582 ins_pipe(pipe_class_memory);
16583 %}
16584
16585 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16586 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16587 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16588 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16589 %{
16590 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16591 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16592 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16593 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16594 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16595 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16596 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16597
16598 ins_encode %{
16599 __ string_indexof($str1$$Register, $str2$$Register,
16600 $cnt1$$Register, $cnt2$$Register,
16601 $tmp1$$Register, $tmp2$$Register,
16602 $tmp3$$Register, $tmp4$$Register,
16603 $tmp5$$Register, $tmp6$$Register,
16604 -1, $result$$Register, StrIntrinsicNode::UL);
16605 %}
16606 ins_pipe(pipe_class_memory);
16607 %}
16608
16609 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16610 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16611 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16612 %{
16613 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16614 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16615 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16616 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16617 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16618 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16619
16620 ins_encode %{
16621 int icnt2 = (int)$int_cnt2$$constant;
16622 __ string_indexof($str1$$Register, $str2$$Register,
16623 $cnt1$$Register, zr,
16624 $tmp1$$Register, $tmp2$$Register,
16625 $tmp3$$Register, $tmp4$$Register, zr, zr,
16626 icnt2, $result$$Register, StrIntrinsicNode::UU);
16627 %}
16628 ins_pipe(pipe_class_memory);
16629 %}
16630
16631 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16632 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16633 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16634 %{
16635 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16636 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16637 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16638 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16639 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16640 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16641
16642 ins_encode %{
16643 int icnt2 = (int)$int_cnt2$$constant;
16644 __ string_indexof($str1$$Register, $str2$$Register,
16645 $cnt1$$Register, zr,
16646 $tmp1$$Register, $tmp2$$Register,
16647 $tmp3$$Register, $tmp4$$Register, zr, zr,
16648 icnt2, $result$$Register, StrIntrinsicNode::LL);
16649 %}
16650 ins_pipe(pipe_class_memory);
16651 %}
16652
16653 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16654 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16655 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16656 %{
16657 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16658 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16659 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16660 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16661 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16662 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16663
16664 ins_encode %{
16665 int icnt2 = (int)$int_cnt2$$constant;
16666 __ string_indexof($str1$$Register, $str2$$Register,
16667 $cnt1$$Register, zr,
16668 $tmp1$$Register, $tmp2$$Register,
16669 $tmp3$$Register, $tmp4$$Register, zr, zr,
16670 icnt2, $result$$Register, StrIntrinsicNode::UL);
16671 %}
16672 ins_pipe(pipe_class_memory);
16673 %}
16674
16675 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16676 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16677 iRegINoSp tmp3, rFlagsReg cr)
16678 %{
16679 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16680 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16681 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16682 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16683
16684 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16685
16686 ins_encode %{
16687 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16688 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16689 $tmp3$$Register);
16690 %}
16691 ins_pipe(pipe_class_memory);
16692 %}
16693
16694 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16695 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16696 iRegINoSp tmp3, rFlagsReg cr)
16697 %{
16698 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16699 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16700 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16701 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16702
16703 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16704
16705 ins_encode %{
16706 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16707 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16708 $tmp3$$Register);
16709 %}
16710 ins_pipe(pipe_class_memory);
16711 %}
16712
16713 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16714 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16715 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16716 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16717 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16718 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16719 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16720 ins_encode %{
16721 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16722 $result$$Register, $ztmp1$$FloatRegister,
16723 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16724 $ptmp$$PRegister, true /* isL */);
16725 %}
16726 ins_pipe(pipe_class_memory);
16727 %}
16728
16729 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16730 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16731 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16732 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16733 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16734 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16735 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16736 ins_encode %{
16737 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16738 $result$$Register, $ztmp1$$FloatRegister,
16739 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16740 $ptmp$$PRegister, false /* isL */);
16741 %}
16742 ins_pipe(pipe_class_memory);
16743 %}
16744
16745 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16746 iRegI_R0 result, rFlagsReg cr)
16747 %{
16748 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16749 match(Set result (StrEquals (Binary str1 str2) cnt));
16750 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16751
16752 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16753 ins_encode %{
16754 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16755 __ string_equals($str1$$Register, $str2$$Register,
16756 $result$$Register, $cnt$$Register);
16757 %}
16758 ins_pipe(pipe_class_memory);
16759 %}
16760
16761 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16762 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16763 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16764 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16765 iRegP_R10 tmp, rFlagsReg cr)
16766 %{
16767 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16768 match(Set result (AryEq ary1 ary2));
16769 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16770 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16771 TEMP vtmp6, TEMP vtmp7, KILL cr);
16772
16773 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16774 ins_encode %{
16775 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16776 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16777 $result$$Register, $tmp$$Register, 1);
16778 if (tpc == nullptr) {
16779 ciEnv::current()->record_failure("CodeCache is full");
16780 return;
16781 }
16782 %}
16783 ins_pipe(pipe_class_memory);
16784 %}
16785
16786 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16787 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16788 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16789 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16790 iRegP_R10 tmp, rFlagsReg cr)
16791 %{
16792 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16793 match(Set result (AryEq ary1 ary2));
16794 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16795 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16796 TEMP vtmp6, TEMP vtmp7, KILL cr);
16797
16798 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16799 ins_encode %{
16800 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16801 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16802 $result$$Register, $tmp$$Register, 2);
16803 if (tpc == nullptr) {
16804 ciEnv::current()->record_failure("CodeCache is full");
16805 return;
16806 }
16807 %}
16808 ins_pipe(pipe_class_memory);
16809 %}
16810
16811 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16812 %{
16813 match(Set result (CountPositives ary1 len));
16814 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16815 format %{ "count positives byte[] $ary1,$len -> $result" %}
16816 ins_encode %{
16817 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16818 if (tpc == nullptr) {
16819 ciEnv::current()->record_failure("CodeCache is full");
16820 return;
16821 }
16822 %}
16823 ins_pipe( pipe_slow );
16824 %}
16825
16826 // fast char[] to byte[] compression
16827 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16828 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16829 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16830 iRegI_R0 result, rFlagsReg cr)
16831 %{
16832 match(Set result (StrCompressedCopy src (Binary dst len)));
16833 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16834 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16835
16836 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16837 ins_encode %{
16838 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16839 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16840 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16841 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16842 %}
16843 ins_pipe(pipe_slow);
16844 %}
16845
16846 // fast byte[] to char[] inflation
16847 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16848 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16849 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16850 %{
16851 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16852 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16853 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16854 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16855
16856 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16857 ins_encode %{
16858 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16859 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16860 $vtmp2$$FloatRegister, $tmp$$Register);
16861 if (tpc == nullptr) {
16862 ciEnv::current()->record_failure("CodeCache is full");
16863 return;
16864 }
16865 %}
16866 ins_pipe(pipe_class_memory);
16867 %}
16868
16869 // encode char[] to byte[] in ISO_8859_1
16870 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16871 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16872 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16873 iRegI_R0 result, rFlagsReg cr)
16874 %{
16875 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16876 match(Set result (EncodeISOArray src (Binary dst len)));
16877 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16878 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16879
16880 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16881 ins_encode %{
16882 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16883 $result$$Register, false,
16884 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16885 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16886 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16887 %}
16888 ins_pipe(pipe_class_memory);
16889 %}
16890
16891 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16892 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16893 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16894 iRegI_R0 result, rFlagsReg cr)
16895 %{
16896 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16897 match(Set result (EncodeISOArray src (Binary dst len)));
16898 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16899 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16900
16901 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16902 ins_encode %{
16903 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16904 $result$$Register, true,
16905 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16906 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16907 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16908 %}
16909 ins_pipe(pipe_class_memory);
16910 %}
16911
16912 //----------------------------- CompressBits/ExpandBits ------------------------
16913
16914 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16915 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16916 match(Set dst (CompressBits src mask));
16917 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16918 format %{ "mov $tsrc, $src\n\t"
16919 "mov $tmask, $mask\n\t"
16920 "bext $tdst, $tsrc, $tmask\n\t"
16921 "mov $dst, $tdst"
16922 %}
16923 ins_encode %{
16924 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16925 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16926 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16927 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16928 %}
16929 ins_pipe(pipe_slow);
16930 %}
16931
16932 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16933 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16934 match(Set dst (CompressBits (LoadI mem) mask));
16935 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16936 format %{ "ldrs $tsrc, $mem\n\t"
16937 "ldrs $tmask, $mask\n\t"
16938 "bext $tdst, $tsrc, $tmask\n\t"
16939 "mov $dst, $tdst"
16940 %}
16941 ins_encode %{
16942 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16944 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16945 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16946 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16947 %}
16948 ins_pipe(pipe_slow);
16949 %}
16950
16951 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16952 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16953 match(Set dst (CompressBits src mask));
16954 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16955 format %{ "mov $tsrc, $src\n\t"
16956 "mov $tmask, $mask\n\t"
16957 "bext $tdst, $tsrc, $tmask\n\t"
16958 "mov $dst, $tdst"
16959 %}
16960 ins_encode %{
16961 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16962 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16963 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16964 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16965 %}
16966 ins_pipe(pipe_slow);
16967 %}
16968
16969 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16970 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16971 match(Set dst (CompressBits (LoadL mem) mask));
16972 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16973 format %{ "ldrd $tsrc, $mem\n\t"
16974 "ldrd $tmask, $mask\n\t"
16975 "bext $tdst, $tsrc, $tmask\n\t"
16976 "mov $dst, $tdst"
16977 %}
16978 ins_encode %{
16979 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16981 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16982 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16983 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16984 %}
16985 ins_pipe(pipe_slow);
16986 %}
16987
16988 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16989 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16990 match(Set dst (ExpandBits src mask));
16991 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16992 format %{ "mov $tsrc, $src\n\t"
16993 "mov $tmask, $mask\n\t"
16994 "bdep $tdst, $tsrc, $tmask\n\t"
16995 "mov $dst, $tdst"
16996 %}
16997 ins_encode %{
16998 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16999 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17000 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17001 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17002 %}
17003 ins_pipe(pipe_slow);
17004 %}
17005
17006 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17007 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17008 match(Set dst (ExpandBits (LoadI mem) mask));
17009 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17010 format %{ "ldrs $tsrc, $mem\n\t"
17011 "ldrs $tmask, $mask\n\t"
17012 "bdep $tdst, $tsrc, $tmask\n\t"
17013 "mov $dst, $tdst"
17014 %}
17015 ins_encode %{
17016 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17018 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17019 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17020 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17021 %}
17022 ins_pipe(pipe_slow);
17023 %}
17024
17025 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17026 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17027 match(Set dst (ExpandBits src mask));
17028 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17029 format %{ "mov $tsrc, $src\n\t"
17030 "mov $tmask, $mask\n\t"
17031 "bdep $tdst, $tsrc, $tmask\n\t"
17032 "mov $dst, $tdst"
17033 %}
17034 ins_encode %{
17035 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17036 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17037 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17038 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17039 %}
17040 ins_pipe(pipe_slow);
17041 %}
17042
17043
17044 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17045 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17046 match(Set dst (ExpandBits (LoadL mem) mask));
17047 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17048 format %{ "ldrd $tsrc, $mem\n\t"
17049 "ldrd $tmask, $mask\n\t"
17050 "bdep $tdst, $tsrc, $tmask\n\t"
17051 "mov $dst, $tdst"
17052 %}
17053 ins_encode %{
17054 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17055 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17056 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17057 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17058 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17059 %}
17060 ins_pipe(pipe_slow);
17061 %}
17062
17063 // ============================================================================
17064 // This name is KNOWN by the ADLC and cannot be changed.
17065 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17066 // for this guy.
17067 instruct tlsLoadP(thread_RegP dst)
17068 %{
17069 match(Set dst (ThreadLocal));
17070
17071 ins_cost(0);
17072
17073 format %{ " -- \t// $dst=Thread::current(), empty" %}
17074
17075 size(0);
17076
17077 ins_encode( /*empty*/ );
17078
17079 ins_pipe(pipe_class_empty);
17080 %}
17081
17082 //----------PEEPHOLE RULES-----------------------------------------------------
17083 // These must follow all instruction definitions as they use the names
17084 // defined in the instructions definitions.
17085 //
17086 // peepmatch ( root_instr_name [preceding_instruction]* );
17087 //
17088 // peepconstraint %{
17089 // (instruction_number.operand_name relational_op instruction_number.operand_name
17090 // [, ...] );
17091 // // instruction numbers are zero-based using left to right order in peepmatch
17092 //
17093 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17094 // // provide an instruction_number.operand_name for each operand that appears
17095 // // in the replacement instruction's match rule
17096 //
17097 // ---------VM FLAGS---------------------------------------------------------
17098 //
17099 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17100 //
17101 // Each peephole rule is given an identifying number starting with zero and
17102 // increasing by one in the order seen by the parser. An individual peephole
17103 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17104 // on the command-line.
17105 //
17106 // ---------CURRENT LIMITATIONS----------------------------------------------
17107 //
17108 // Only match adjacent instructions in same basic block
17109 // Only equality constraints
17110 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17111 // Only one replacement instruction
17112 //
17113 // ---------EXAMPLE----------------------------------------------------------
17114 //
17115 // // pertinent parts of existing instructions in architecture description
17116 // instruct movI(iRegINoSp dst, iRegI src)
17117 // %{
17118 // match(Set dst (CopyI src));
17119 // %}
17120 //
17121 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17122 // %{
17123 // match(Set dst (AddI dst src));
17124 // effect(KILL cr);
17125 // %}
17126 //
17127 // // Change (inc mov) to lea
17128 // peephole %{
17129 // // increment preceded by register-register move
17130 // peepmatch ( incI_iReg movI );
17131 // // require that the destination register of the increment
17132 // // match the destination register of the move
17133 // peepconstraint ( 0.dst == 1.dst );
17134 // // construct a replacement instruction that sets
17135 // // the destination to ( move's source register + one )
17136 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17137 // %}
17138 //
17139
17140 // Implementation no longer uses movX instructions since
17141 // machine-independent system no longer uses CopyX nodes.
17142 //
17143 // peephole
17144 // %{
17145 // peepmatch (incI_iReg movI);
17146 // peepconstraint (0.dst == 1.dst);
17147 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17148 // %}
17149
17150 // peephole
17151 // %{
17152 // peepmatch (decI_iReg movI);
17153 // peepconstraint (0.dst == 1.dst);
17154 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17155 // %}
17156
17157 // peephole
17158 // %{
17159 // peepmatch (addI_iReg_imm movI);
17160 // peepconstraint (0.dst == 1.dst);
17161 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17162 // %}
17163
17164 // peephole
17165 // %{
17166 // peepmatch (incL_iReg movL);
17167 // peepconstraint (0.dst == 1.dst);
17168 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17169 // %}
17170
17171 // peephole
17172 // %{
17173 // peepmatch (decL_iReg movL);
17174 // peepconstraint (0.dst == 1.dst);
17175 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17176 // %}
17177
17178 // peephole
17179 // %{
17180 // peepmatch (addL_iReg_imm movL);
17181 // peepconstraint (0.dst == 1.dst);
17182 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17183 // %}
17184
17185 // peephole
17186 // %{
17187 // peepmatch (addP_iReg_imm movP);
17188 // peepconstraint (0.dst == 1.dst);
17189 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17190 // %}
17191
17192 // // Change load of spilled value to only a spill
17193 // instruct storeI(memory mem, iRegI src)
17194 // %{
17195 // match(Set mem (StoreI mem src));
17196 // %}
17197 //
17198 // instruct loadI(iRegINoSp dst, memory mem)
17199 // %{
17200 // match(Set dst (LoadI mem));
17201 // %}
17202 //
17203
17204 //----------SMARTSPILL RULES---------------------------------------------------
17205 // These must follow all instruction definitions as they use the names
17206 // defined in the instructions definitions.
17207
17208 // Local Variables:
17209 // mode: c++
17210 // End: